diff --git a/e2e/commands.js b/e2e/commands.js index b02401e76..8e615f1b6 100644 --- a/e2e/commands.js +++ b/e2e/commands.js @@ -43,6 +43,12 @@ const viewSchemas = { "CREATE VIEW IF NOT EXISTS btc_trades_view AS SELECT * FROM btc_trades;", } +const liveViewSchemas = { + btc_trades_lv: + "CREATE LIVE VIEW IF NOT EXISTS btc_trades_lv FLUSH EVERY 1s IN MEMORY 5s START FROM BEGINNING AS " + + "SELECT timestamp, symbol, avg(price) OVER (PARTITION BY symbol ORDER BY timestamp ROWS 100 PRECEDING) AS moving_avg FROM btc_trades;", +} + Cypress.on("uncaught:exception", (err) => { // Monaco editor's word highlighter throws "Canceled" errors during rapid tab switching // when restoreViewState cancels pending async operations - this is harmless @@ -509,6 +515,14 @@ Cypress.Commands.add("dropViewIfExists", (name) => { cy.execQuery(`DROP VIEW IF EXISTS ${name};`) }) +Cypress.Commands.add("createLiveView", (name) => { + cy.execQuery(liveViewSchemas[name]) +}) + +Cypress.Commands.add("dropLiveViewIfExists", (name) => { + cy.execQuery(`DROP LIVE VIEW IF EXISTS ${name};`) +}) + Cypress.Commands.add("interceptQuery", (query, alias, response) => { cy.intercept( { @@ -662,6 +676,22 @@ Cypress.Commands.add("expandViews", () => { }) }) +Cypress.Commands.add("expandLiveViews", () => { + cy.get("body").then((body) => { + if (body.find('[data-hook="expand-live-views"]').length > 0) { + cy.get('[data-hook="expand-live-views"]').dblclick({ force: true }) + } + }) +}) + +Cypress.Commands.add("collapseLiveViews", () => { + cy.get("body").then((body) => { + if (body.find('[data-hook="collapse-live-views"]').length > 0) { + cy.get('[data-hook="collapse-live-views"]').dblclick({ force: true }) + } + }) +}) + Cypress.Commands.add("openDetailsDrawer", (name, kind = "table") => { const titleHook = `schema-${kind}-title` cy.getByDataHook(titleHook).contains(name).click() @@ -670,6 +700,22 @@ Cypress.Commands.add("openDetailsDrawer", (name, kind = "table") => { cy.getByDataHook("table-details-name").should("have.value", name) }) +// Radix arms a tooltip only on a trigger pointermove, and realHover emits a +// single move event. That one armed intent can be silently lost to the +// provider's pointer-in-transit gate or to boundary events fired when the DOM +// re-renders under the stationary pointer (e.g. right after a tab switch while +// its data fetches land). Nothing re-arms it without another pointermove, so +// nudge the pointer after hovering: the first move lets Radix clear stale +// transit state, the second re-arms the tooltip. +// Takes a factory rather than an element so every action re-queries the +// trigger: a wrapped node that detaches on re-render keeps a zeroed rect, and +// realMouseMove would then aim at the viewport corner instead of the trigger. +Cypress.Commands.add("hoverForTooltip", (getTrigger) => { + getTrigger().realHover() + getTrigger().realMouseMove(2, 2, { position: "center" }) + getTrigger().realMouseMove(0, 0, { position: "center" }) +}) + Cypress.Commands.add("getEditorTabs", () => { return cy.get(".chrome-tab") }) diff --git a/e2e/questdb b/e2e/questdb index 9b59a9211..12a33d651 160000 --- a/e2e/questdb +++ b/e2e/questdb @@ -1 +1 @@ -Subproject commit 9b59a921165af573cedd22bf8b12613de19cb8bd +Subproject commit 12a33d651e51e2682e7a448c8db5168fc72dfad3 diff --git a/e2e/tests/console/schema.spec.js b/e2e/tests/console/schema.spec.js index 6d3289f7c..a9adfd99e 100644 --- a/e2e/tests/console/schema.spec.js +++ b/e2e/tests/console/schema.spec.js @@ -16,6 +16,8 @@ const materializedViews = ["btc_trades_mv"] const views = ["btc_trades_view"] +const liveViews = ["btc_trades_lv"] + describe("questdb schema with working tables", () => { before(() => { cy.loadConsoleWithAuth() @@ -585,7 +587,7 @@ describe("materialized views", () => { cy.wait(1200) cy.getByDataHook("tooltip").should( "contain", - `Partitioned by "week", ordered on "timestamp" column.`, + `Materialized view. Partitioned by "week", ordered on "timestamp" column.`, ) }) @@ -683,6 +685,45 @@ describe("materialized views", () => { ) }) + it("should omit the reason when the view is invalidated without one", () => { + // Given a matview the server reports as invalid with a null reason + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { + query: "materialized_views()", + }, + }, + (req) => { + req.continue((res) => { + if (res.body?.dataset?.length > 0) { + const viewStatusIndex = res.body.columns.findIndex( + (c) => c.name === "view_status", + ) + const invalidationReasonIndex = res.body.columns.findIndex( + (c) => c.name === "invalidation_reason", + ) + res.body.dataset[0][viewStatusIndex] = "invalid" + res.body.dataset[0][invalidationReasonIndex] = null + } + return res + }) + }, + ) + cy.refreshSchema() + cy.expandMatViews() + + // When + cy.hoverForTooltip(() => cy.getByDataHook("schema-row-error-icon")) + cy.wait(300) + + // Then the message stops at the status, with no interpolated null + cy.getByDataHook("tooltip") + .should("contain", "Materialized view is invalid") + .and("not.contain", "null") + }) + after(() => { cy.loadConsoleWithAuth() @@ -696,6 +737,263 @@ describe("materialized views", () => { }) }) +describe("live views", () => { + before(() => { + cy.loadConsoleWithAuth() + + tables.forEach((table) => { + cy.createTable(table) + }) + liveViews.forEach((lv) => { + cy.createLiveView(lv) + }) + cy.refreshSchema() + }) + + beforeEach(() => { + cy.collapseTables() + cy.collapseMatViews() + }) + + afterEach(() => { + cy.collapseLiveViews() + }) + + it("should show live views in their own folder", () => { + // Given + cy.getByDataHook("schema-folder-title").should( + "contain", + `Live views (${liveViews.length})`, + ) + + // When + cy.expandLiveViews() + + // Then + cy.getByDataHook("schema-liveview-title").should("contain", "btc_trades_lv") + }) + + it("should show the table icon description in the tooltip for a live view", () => { + // Given + cy.expandLiveViews() + + // When + cy.hoverForTooltip(() => + cy + .getByDataHook("schema-liveview-title") + .contains("btc_trades_lv") + .closest('[data-hook="schema-row"]') + .find('[data-hook="table-icon"]'), + ) + cy.wait(1200) + + // Then: the kind prefix identifies the schema object before the shared + // partition/timestamp sentence. + cy.getByDataHook("tooltip").should( + "contain", + `Live view. Partitioned by "day", ordered on "timestamp" column.`, + ) + }) + + it("should show the base table and copy schema for a live view", () => { + // Given + cy.expandLiveViews() + + // When + cy.getByDataHook("schema-liveview-title") + .contains("btc_trades_lv") + .dblclick() + cy.getByDataHook("schema-row").contains("Base tables").dblclick() + + // Then + cy.getByDataHook("schema-detail-title") + .contains("btc_trades") + .should("exist") + + // When + cy.getByDataHook("schema-liveview-title") + .contains("btc_trades_lv") + .rightclick() + cy.getByDataHook("table-context-menu-copy-schema") + .filter(":visible") + .click() + + // Then + if (Cypress.isBrowser("electron")) { + cy.window() + .its("navigator.clipboard") + .invoke("readText") + .should("match", /^CREATE LIVE VIEW.*'btc_trades_lv'/) + } + }) + + it("should not offer creating a materialized view from a live view", () => { + // Given + cy.expandLiveViews() + + // When + cy.getByDataHook("schema-liveview-title") + .contains("btc_trades_lv") + .rightclick() + + // Then + cy.getByDataHook("table-context-menu-view-details") + .filter(":visible") + .should("exist") + cy.getByDataHook("table-context-menu-create-matview").should("not.exist") + cy.realPress("Escape") + }) + + it("should resume WAL for a suspended live view with an ALTER LIVE VIEW statement", () => { + // Given: tables() does not flag a suspended live view yet, so simulate the flag + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /tables\(\)/ }, + }, + (req) => { + req.continue((res) => { + if (res.body?.dataset?.length > 0) { + const nameIndex = res.body.columns.findIndex( + (c) => c.name === "table_name", + ) + const suspendedIndex = res.body.columns.findIndex( + (c) => c.name === "table_suspended", + ) + for (const row of res.body.dataset) { + if (row[nameIndex] === "btc_trades_lv") { + row[suspendedIndex] = true + } + } + } + return res + }) + }, + ) + cy.refreshSchema() + cy.expandLiveViews() + cy.getByDataHook("schema-row-error-icon").should("be.visible") + + // When + cy.getByDataHook("schema-liveview-title") + .contains("btc_trades_lv") + .rightclick() + cy.getByDataHook("table-context-menu-resume-wal").filter(":visible").click() + + // Then + cy.getByDataHook("schema-suspension-dialog").should( + "have.attr", + "data-table-name", + "btc_trades_lv", + ) + + // When + cy.intercept({ + method: "GET", + pathname: "/exec", + query: { query: /ALTER LIVE VIEW/ }, + }).as("resumeLiveViewWal") + cy.getByDataHook("schema-suspension-dialog-restart-transaction").click() + + // Then + cy.wait("@resumeLiveViewWal").then((interception) => { + expect(decodeURIComponent(interception.request.url)).to.contain( + "ALTER LIVE VIEW 'btc_trades_lv' RESUME WAL", + ) + }) + cy.getByDataHook("schema-suspension-dialog-dismiss").click() + cy.getByDataHook("schema-suspension-dialog").should("not.exist") + }) + + it("should show a warning icon and tooltip when the live view is invalidated", () => { + // Given + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { + query: /live_views\(\)/, + }, + }, + (req) => { + req.continue((res) => { + if (res.body && res.body.dataset && res.body.dataset.length > 0) { + const viewStatusIndex = res.body.columns.findIndex( + (c) => c.name === "view_status", + ) + const invalidationReasonIndex = res.body.columns.findIndex( + (c) => c.name === "invalidation_reason", + ) + res.body.dataset[0][viewStatusIndex] = "invalid" + res.body.dataset[0][invalidationReasonIndex] = + "this is an invalidation reason" + } + return res + }) + }, + ) + cy.refreshSchema() + cy.expandLiveViews() + + // When + cy.hoverForTooltip(() => cy.getByDataHook("schema-row-error-icon")) + cy.wait(300) + + // Then + cy.getByDataHook("tooltip").should( + "contain", + "Live view is invalid: this is an invalidation reason", + ) + }) + + it("should show a warning icon and tooltip when the live view state is unreadable", () => { + // Given + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /live_views\(\)/ }, + }, + (req) => { + req.continue((res) => { + if (res.body?.dataset?.length > 0) { + const viewStatusIndex = res.body.columns.findIndex( + (c) => c.name === "view_status", + ) + res.body.dataset[0][viewStatusIndex] = "state_unreadable" + } + return res + }) + }, + ) + cy.refreshSchema() + cy.expandLiveViews() + + // When + cy.hoverForTooltip(() => cy.getByDataHook("schema-row-error-icon")) + cy.wait(300) + + // Then + cy.getByDataHook("tooltip").should( + "contain", + "Live view state files are unreadable", + ) + }) + + after(() => { + cy.loadConsoleWithAuth() + + liveViews.forEach((lv) => { + cy.dropLiveViewIfExists(lv) + }) + + tables.forEach((table) => { + cy.dropTableIfExists(table) + }) + }) +}) + describe("create materialized view from context menu", () => { const sourceTable = "btc_trades" const nonWalTable = "btc_trades_no_wal" diff --git a/e2e/tests/console/tableDetails.spec.js b/e2e/tests/console/tableDetails.spec.js index 4bb001bad..9f8a8f300 100644 --- a/e2e/tests/console/tableDetails.spec.js +++ b/e2e/tests/console/tableDetails.spec.js @@ -8,14 +8,24 @@ const { createChatTitleResponse, isTitleRequest, } = require("../../utils/aiAssistant") - const TEST_TABLE = "btc_trades" const TEST_TABLE_NO_WAL = "btc_trades_no_wal" const TEST_MATVIEW = "btc_trades_mv" const TEST_MATVIEW_ON_MV = "btc_trades_mv_on_mv" const TEST_VIEW = "btc_trades_view" - -function interceptTablesQuery(modifications) { +const TEST_LIVE_VIEW = "btc_trades_lv" +const TEST_LIVE_VIEW_2 = "btc_trades_lv_2" +const TEST_LIVE_VIEW_BASE_2 = "btc_trades_lv_base_2" + +const TEST_LIVE_VIEW_BASE_2_DDL = + `CREATE TABLE IF NOT EXISTS ${TEST_LIVE_VIEW_BASE_2} ` + + "(symbol SYMBOL, price DOUBLE, timestamp TIMESTAMP) " + + "TIMESTAMP(timestamp) PARTITION BY DAY WAL;" +const TEST_LIVE_VIEW_2_DDL = + `CREATE LIVE VIEW IF NOT EXISTS ${TEST_LIVE_VIEW_2} FLUSH EVERY 1s IN MEMORY 5s START FROM BEGINNING AS ` + + `SELECT timestamp, symbol, avg(price) OVER (PARTITION BY symbol ORDER BY timestamp ROWS 100 PRECEDING) AS moving_avg FROM ${TEST_LIVE_VIEW_BASE_2};` + +function interceptTablesQuery(modifications, targetTable = TEST_TABLE) { cy.intercept( { method: "GET", @@ -34,7 +44,7 @@ function interceptTablesQuery(modifications) { (c) => c.name === "table_name", ) for (let i = 0; i < res.body.dataset.length; i++) { - if (res.body.dataset[i][tableNameIndex] === TEST_TABLE) { + if (res.body.dataset[i][tableNameIndex] === targetTable) { res.body.dataset[i][fieldIndex] = value } } @@ -74,6 +84,46 @@ function interceptMatViewsQuery(modifications) { ).as("matviewsQuery") } +function interceptLiveViewsQuery(modifications) { + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /live_views\(\)/ }, + }, + (req) => { + req.continue((res) => { + if (res.body?.dataset?.length > 0) { + for (const [fieldName, value] of Object.entries(modifications)) { + const fieldIndex = res.body.columns.findIndex( + (c) => c.name === fieldName, + ) + if (fieldIndex !== -1) { + for (let i = 0; i < res.body.dataset.length; i++) { + res.body.dataset[i][fieldIndex] = value + } + } + } + } + return res + }) + }, + ).as("liveViewsQuery") +} + +function mutateLiveViewResponse(res, modifications) { + if (!res.body?.dataset?.length) return + + for (const [fieldName, value] of Object.entries(modifications)) { + const fieldIndex = res.body.columns.findIndex( + (column) => column.name === fieldName, + ) + if (fieldIndex !== -1) { + res.body.dataset[0][fieldIndex] = value + } + } +} + function interceptAIRequest(responseText = "Test AI response", sql = null) { const responseData = createFinalResponseData("openai", responseText, sql) @@ -88,19 +138,6 @@ function interceptAIRequest(responseText = "Test AI response", sql = null) { }).as("openaiRequest") } -// Radix arms a tooltip only on a trigger pointermove, and realHover emits a -// single move event. That one armed intent can be silently lost to the -// provider's pointer-in-transit gate or to boundary events fired when the DOM -// re-renders under the stationary pointer (e.g. right after a tab switch while -// its data fetches land). Nothing re-arms it without another pointermove, so -// nudge the pointer after hovering: the first move lets Radix clear stale -// transit state, the second re-arms the tooltip. -function hoverForTooltip(hook) { - cy.getByDataHook(hook).realHover() - cy.getByDataHook(hook).realMouseMove(2, 2, { position: "center" }) - cy.getByDataHook(hook).realMouseMove(0, 0, { position: "center" }) -} - describe("TableDetailsDrawer", () => { beforeEach(() => { cy.intercept("POST", PROVIDERS.openai.endpoint, (req) => { @@ -188,6 +225,207 @@ describe("TableDetailsDrawer", () => { }) }) + describe("source availability", () => { + before(() => { + cy.loadConsoleWithAuth() + cy.createTable(TEST_TABLE) + cy.refreshSchema() + }) + + beforeEach(() => { + cy.loadConsoleWithAuth(false, getOpenAIConfiguredSettings()) + cy.expandTables() + }) + + it("should disable DDL actions after repeated failures", () => { + // Given + let failDDL = false + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /SHOW CREATE TABLE/ }, + }, + (req) => { + if (failDDL) { + req.reply({ + statusCode: 500, + body: { error: "DDL unavailable", position: 0 }, + }) + } else { + req.continue() + } + }, + ).as("ddlAvailability") + + // When + cy.openDetailsDrawer(TEST_TABLE) + cy.getByDataHook("table-details-tab-details").click() + cy.wait("@ddlAvailability") + + // Then + cy.getByDataHook("table-details-copy-ddl").should("not.be.disabled") + cy.getByDataHook("table-details-explain-ai").should("not.be.disabled") + + // When + cy.then(() => { + failDDL = true + }) + cy.wait("@ddlAvailability") + cy.wait("@ddlAvailability") + cy.wait("@ddlAvailability") + + // Then + cy.get('[data-hook="table-details-ddl-unavailable"]', { + timeout: 5000, + }) + .should("be.visible") + .and("contain", "Unavailable") + cy.getByDataHook("table-details-copy-ddl").should("be.disabled") + cy.getByDataHook("table-details-explain-ai").should("be.disabled") + }) + + it("should distinguish unavailable columns from an empty schema", () => { + // Given + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /SHOW COLUMNS FROM/ }, + }, + { + statusCode: 500, + body: { error: "Columns unavailable", position: 0 }, + }, + ).as("columnsUnavailable") + + // When + cy.openDetailsDrawer(TEST_TABLE) + cy.getByDataHook("table-details-tab-details").click() + cy.wait("@columnsUnavailable") + cy.wait("@columnsUnavailable") + cy.wait("@columnsUnavailable") + + // Then + cy.get('[data-hook="table-details-columns-unavailable"]', { + timeout: 5000, + }) + .should("be.visible") + .and("contain", "Unavailable") + .and("not.contain", "Columns (0)") + cy.getByDataHook("table-details-columns-toggle").should("not.exist") + }) + + it("should retain last-known-good table data after an admitted failure", () => { + // Given + cy.openDetailsDrawer(TEST_TABLE) + cy.getByDataHook("table-details-row-count-value").should("be.visible") + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: new RegExp(`tables\\(\\).*${TEST_TABLE}`) }, + }, + { + statusCode: 500, + body: { error: "Tables unavailable", position: 0 }, + }, + ).as("tablesUnavailable") + + // When + cy.wait("@tablesUnavailable") + cy.wait("@tablesUnavailable") + + // Then + cy.getByDataHook("table-details-tables-error").should("not.exist") + cy.wait("@tablesUnavailable") + cy.get('[data-hook="table-details-tables-error"]', { + timeout: 5000, + }) + .should("be.visible") + .and("contain", "last successful response") + cy.getByDataHook("table-details-row-count-value").should("be.visible") + cy.getByDataHook("table-details-health-status").should( + "have.attr", + "data-severity", + "unknown", + ) + }) + + it("should show a full drawer error when tables metadata never loads", () => { + // Given + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: new RegExp(`tables\\(\\).*${TEST_TABLE}`) }, + }, + { + statusCode: 500, + body: { error: "Tables unavailable", position: 0 }, + }, + ).as("initialTablesUnavailable") + + // When + cy.openDetailsDrawer(TEST_TABLE) + cy.wait("@initialTablesUnavailable") + cy.wait("@initialTablesUnavailable") + + // Then + cy.getByDataHook("table-details-source-error").should("not.exist") + cy.wait("@initialTablesUnavailable") + cy.get('[data-hook="table-details-source-error"]', { + timeout: 5000, + }) + .should("be.visible") + .and("have.attr", "role", "alert") + .and("contain", `Unable to load ${TEST_TABLE}`) + .and("contain", "retry automatically") + }) + + it("should clear the target immediately after a successful empty tables response", () => { + // Given + let returnEmpty = false + let emptyResponseSent = false + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: new RegExp(`tables\\(\\).*${TEST_TABLE}`) }, + }, + (req) => { + req.continue((res) => { + if (returnEmpty && !emptyResponseSent) { + res.body.dataset = [] + res.body.count = 0 + emptyResponseSent = true + } + return res + }) + }, + ) + cy.openDetailsDrawer(TEST_TABLE) + cy.getByDataHook("table-details-row-count-value").should("be.visible") + + // When + cy.then(() => { + returnEmpty = true + }) + cy.wrap(null).should(() => { + expect(emptyResponseSent).to.equal(true) + }) + + // Then + cy.getByDataHook("table-details-name").should("have.value", "") + cy.getByDataHook("table-details-empty-state").should("be.visible") + }) + + after(() => { + cy.loadConsoleWithAuth() + cy.dropTable(TEST_TABLE) + }) + }) + describe("critical health issues - WAL suspended (R1)", () => { before(() => { cy.loadConsoleWithAuth() @@ -615,7 +853,7 @@ describe("TableDetailsDrawer", () => { cy.getByDataHook("table-details-type-badge").should( "contain", - "Materialized View", + "Materialized view", ) cy.getByDataHook("table-details-view-status").should("be.visible") cy.getByDataHook("table-details-base-table-status").should("be.visible") @@ -638,12 +876,70 @@ describe("TableDetailsDrawer", () => { cy.getByDataHook("table-details-type-badge").should( "contain", - "Materialized View", + "Materialized view", ) cy.getByDataHook("table-details-name").should("have.value", TEST_MATVIEW) cy.getByDataHook("sidebar-back-button").should("be.disabled") }) + it("should keep table-backed details when matview metadata is unavailable", () => { + // Given + interceptTablesQuery({ table_memory_pressure_level: 1 }, TEST_MATVIEW) + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /materialized_views\(\) WHERE view_name/ }, + }, + (req) => { + req.continue((res) => { + res.body.dataset = [] + res.body.count = 0 + return res + }) + }, + ).as("missingMatViewMetadata") + + // When + cy.openDetailsDrawer(TEST_MATVIEW, "matview") + cy.wait("@missingMatViewMetadata") + cy.wait("@missingMatViewMetadata") + cy.wait("@missingMatViewMetadata") + + // Then + cy.get('[data-hook="table-details-kind-metadata-error"]', { + timeout: 5000, + }) + .should("be.visible") + .and("contain", "retry automatically") + cy.getByDataHook("table-details-view-status").should( + "contain", + "Unavailable", + ) + cy.getByDataHook("table-details-health-status").should( + "have.attr", + "data-severity", + "warning", + ) + cy.getByDataHook("table-details-tab-error-badge").should("not.exist") + cy.getByDataHook("table-details-tab-warning-badge").should("be.visible") + + // When + cy.getByDataHook("table-details-tab-details").click() + + // Then + cy.getByDataHook("table-details-base-table-section") + .should("be.visible") + .and("contain", "Unavailable") + cy.getByDataHook("table-details-base-table-link").should("be.disabled") + cy.getByDataHook("table-details-details-section") + .should("be.visible") + .should("contain", "Deduplication") + .should("contain", "Partitioning") + .should("contain", "Refresh Type") + .should("contain", "Unavailable") + }) + after(() => { cy.loadConsoleWithAuth() cy.dropMaterializedView(TEST_MATVIEW) @@ -670,7 +966,7 @@ describe("TableDetailsDrawer", () => { cy.getByDataHook("table-details-type-badge").should( "contain", - "Materialized View", + "Materialized view", ) cy.getByDataHook("table-details-tab-details").click() @@ -686,7 +982,7 @@ describe("TableDetailsDrawer", () => { cy.getByDataHook("table-details-name").should("have.value", TEST_MATVIEW) cy.getByDataHook("table-details-type-badge").should( "contain", - "Materialized View", + "Materialized view", ) }) @@ -733,53 +1029,835 @@ describe("TableDetailsDrawer", () => { }) }) - describe("view specific", () => { + describe("live view specific", () => { before(() => { cy.loadConsoleWithAuth() cy.createTable(TEST_TABLE) - cy.createView(TEST_VIEW) + cy.execQuery(TEST_LIVE_VIEW_BASE_2_DDL) + cy.createLiveView(TEST_LIVE_VIEW) + cy.execQuery(TEST_LIVE_VIEW_2_DDL) }) beforeEach(() => { cy.loadConsoleWithAuth() cy.refreshSchema() cy.collapseTables() - cy.collapseMatViews() - cy.expandViews() + cy.expandLiveViews() }) - it("should open view details from schema, show View badge, no tabs, only DDL and columns sections with columns expanded", () => { - cy.openDetailsDrawer(TEST_VIEW, "view") - - cy.getByDataHook("table-details-type-badge").should("contain", "View") - - cy.getByDataHook("table-details-tab-monitoring").should("not.exist") - cy.getByDataHook("table-details-tab-details").should("not.exist") + it("should show live view type badge, view status and live view monitoring sections", () => { + // When + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") - cy.getByDataHook("table-details-ddl-section").should("be.visible") + // Then + cy.getByDataHook("table-details-type-badge").should( + "contain", + "Live view", + ) + cy.getByDataHook("table-details-view-status") + .should("be.visible") + .should("contain", "Active") + cy.getByDataHook("table-details-base-table-status").should("be.visible") + cy.getByDataHook("table-details-live-view-freshness") + .should("be.visible") + .should("contain", "Unflushed Transactions") + .should("contain", "Since Last Flush") + cy.getByDataHook("table-details-live-view-freshness-grid").then( + ($grid) => { + const gridStyle = getComputedStyle($grid[0]) + const firstItemStyle = getComputedStyle($grid[0].children[0]) + + expect(gridStyle.gridTemplateColumns.split(" ")).to.have.length(2) + expect(firstItemStyle.gridColumn).to.equal("1 / -1") + }, + ) + cy.getByDataHook("table-details-live-view-memory") + .should("be.visible") + .should("contain", "Rows in Memory") + .should("contain", "Memory Footprint") + cy.getByDataHook("table-details-live-view-freshness").should( + "contain", + "Writer Stall", + ) + cy.getByDataHook("table-details-live-view-memory").should( + "not.contain", + "Dropped Below Start From", + ) + }) - cy.getByDataHook("table-details-columns-content").should("be.visible") + it("should keep live view sections mounted when metadata returns no rows", () => { + // Given + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /live_views\(\) WHERE view_name/ }, + }, + (req) => { + req.continue((res) => { + res.body.dataset = [] + res.body.count = 0 + return res + }) + }, + ).as("emptyLiveViewMetadata") - cy.getByDataHook("table-details-details-section").should("not.exist") + // When + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + cy.wait("@emptyLiveViewMetadata") + cy.wait("@emptyLiveViewMetadata") + cy.wait("@emptyLiveViewMetadata") - cy.getByDataHook("table-details-health-status") + // Then + cy.get('[data-hook="table-details-kind-metadata-error"]', { + timeout: 5000, + }) .should("be.visible") - .should("have.attr", "data-severity", "healthy") - }) - - after(() => { - cy.loadConsoleWithAuth() - cy.dropViewIfExists(TEST_VIEW) - cy.dropTable(TEST_TABLE) + .and("contain", "retry automatically") + cy.getByDataHook("table-details-name").should( + "have.value", + TEST_LIVE_VIEW, + ) + cy.getByDataHook("table-details-row-count-value").should("be.visible") + cy.getByDataHook("table-details-health-status").should( + "have.attr", + "data-severity", + "unknown", + ) + cy.getByDataHook("table-details-view-status").should( + "contain", + "Unavailable", + ) + cy.getByDataHook("table-details-live-view-freshness") + .should("be.visible") + .and("contain", "Unavailable") + cy.getByDataHook("table-details-live-view-memory") + .should("be.visible") + .and("contain", "Unavailable") + .and("contain", "Dropped Below Start From") }) - }) - describe("view invalid state (R4)", () => { - before(() => { - cy.loadConsoleWithAuth() - cy.createTable(TEST_TABLE) - cy.createView(TEST_VIEW) - }) + it("should tolerate transient failures and require two successes to recover", () => { + // Given + let failMetadata = true + + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /live_views\(\) WHERE view_name/ }, + }, + (req) => { + if (failMetadata) { + req.reply({ + statusCode: 500, + body: { + error: "live view metadata unavailable", + position: 0, + query: String(req.query.query ?? ""), + }, + }) + } else { + req.continue() + } + }, + ).as("liveViewAvailability") + + // When + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + cy.wait("@liveViewAvailability") + cy.wait("@liveViewAvailability") + + // Then + cy.getByDataHook("table-details-kind-metadata-error").should("not.exist") + + // When + cy.wait("@liveViewAvailability") + + // Then + cy.get('[data-hook="table-details-kind-metadata-error"]', { + timeout: 5000, + }) + .should("be.visible") + .should("contain", "retry automatically") + cy.getByDataHook("table-details-health-status").should( + "have.attr", + "data-severity", + "unknown", + ) + + // When + cy.then(() => { + failMetadata = false + }) + cy.wait("@liveViewAvailability") + + // Then + cy.getByDataHook("table-details-kind-metadata-error").should("be.visible") + + // When + cy.wait("@liveViewAvailability") + + // Then + cy.getByDataHook("table-details-view-status").should("contain", "Active") + cy.getByDataHook("table-details-kind-metadata-error").should("not.exist") + cy.getByDataHook("table-details-health-status").should( + "have.attr", + "data-severity", + "healthy", + ) + }) + + it("should complete metadata polling when responses exceed the poll period", () => { + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /live_views\(\) WHERE view_name/ }, + }, + (req) => { + req.continue((res) => { + res.setDelay(1500) + return res + }) + }, + ) + + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + + cy.getByDataHook("table-details-view-status") + .should("be.visible") + .should("contain", "Active") + cy.getByDataHook("table-details-live-view-memory").should("be.visible") + }) + + it("should display unsafe LONG counters without rounding", () => { + interceptLiveViewsQuery({ + lag_seqtxn: "9007199254740993", + in_mem_rows: "9007199254740993", + }) + + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + + cy.getByDataHook("table-details-live-view-freshness").should( + "contain", + "9,007,199,254,740,993 txns", + ) + cy.getByDataHook("table-details-live-view-memory").should( + "contain", + "9,007,199,254,740,993", + ) + }) + + it("should show the seeding status, writer stall and dropped rows from live view metrics", () => { + // Given + interceptLiveViewsQuery({ + view_status: "seeding", + writer_stall_micros: "6500000", + below_lower_bound_count: "5", + o3_rejected_count: "3", + }) + + // When + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + + // Then: the server holds last_processed_seqtxn equal to + // seed_target_seqtxn for the whole seed, so there is no progress to show + cy.getByDataHook("table-details-view-status").should("contain", "Seeding") + cy.getByDataHook("table-details-live-view-freshness") + .should("contain", "Writer Stall") + .should("contain", "6.5 s") + cy.getByDataHook("table-details-live-view-memory") + .should("contain", "Dropped Below Start From") + .should("contain", "5 in-order ยท 3 out-of-order") + }) + + it("should show Never and Unknown for lag values the server does not know yet", () => { + // Given: lag_micros is NULL until the first flush and lag_seqtxn is + // NULL while the base table token is unresolved + interceptLiveViewsQuery({ + lag_micros: null, + lag_seqtxn: null, + }) + + // When + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + + // Then + cy.getByDataHook("table-details-live-view-freshness") + .should("contain", "Unknown") + .should("contain", "Never") + }) + + it("should show the live view definition cards in the details tab", () => { + // Given + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + + // When + cy.getByDataHook("table-details-tab-details").click() + + // Then + cy.getByDataHook("table-details-ddl-section").should("be.visible") + cy.getByDataHook("table-details-flush-every-card") + .should("contain", "Flush Every") + .should("contain", "1 Second") + cy.getByDataHook("table-details-in-memory-card") + .should("contain", "In Memory") + .should("contain", "5 Seconds") + cy.getByDataHook("table-details-start-from-card") + .should("contain", "Start From") + .should("contain", "Beginning") + cy.getByDataHook("table-details-details-section").should( + "contain", + "Partitioning", + ) + cy.getByDataHook("table-details-details-section") + .should("not.contain", "TTL") + .should("not.contain", "Deduplication") + .should("not.contain", "Refresh Type") + cy.getByDataHook("table-details-storage-policy-section").should( + "not.exist", + ) + }) + + it("should navigate to the base table and back preserving kinds", () => { + // Given + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + cy.getByDataHook("table-details-tab-details").click() + cy.getByDataHook("table-details-base-table-section").should("be.visible") + + // When + cy.getByDataHook("table-details-base-table-link") + .should("contain", TEST_TABLE) + .click() + + // Then + cy.getByDataHook("table-details-type-badge").should("contain", "Table") + cy.getByDataHook("table-details-name").should("have.value", TEST_TABLE) + + // When + cy.getByDataHook("sidebar-back-button").click() + + // Then + cy.getByDataHook("table-details-type-badge").should( + "contain", + "Live view", + ) + cy.getByDataHook("table-details-name").should( + "have.value", + TEST_LIVE_VIEW, + ) + }) + + it("should ignore an in-flight response after selecting another live view", () => { + let delayOldTarget = false + let oldTargetRequestStarted = false + + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /live_views\(\) WHERE view_name/ }, + }, + (req) => { + const query = String(req.query.query ?? "") + if ( + delayOldTarget && + !oldTargetRequestStarted && + query.includes(TEST_LIVE_VIEW) + ) { + oldTargetRequestStarted = true + req.continue((res) => { + mutateLiveViewResponse(res, { + view_status: "invalid", + invalidation_reason: "belongs to old target", + in_mem_rows: "111", + }) + res.setDelay(1800) + return res + }) + } else if (query.includes(TEST_LIVE_VIEW_2)) { + req.continue((res) => { + mutateLiveViewResponse(res, { + view_status: "seeding", + in_mem_rows: "222", + }) + return res + }) + } else { + req.continue() + } + }, + ) + + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + cy.getByDataHook("table-details-view-status").should("contain", "Active") + + cy.then(() => { + delayOldTarget = true + }) + cy.wrap(null).should(() => { + expect(oldTargetRequestStarted).to.equal(true) + }) + + cy.getByDataHook("table-details-name").click() + cy.getByDataHook("table-details-name").clear().type(TEST_LIVE_VIEW_2) + cy.getByDataHook("table-details-name").type("{enter}") + + cy.getByDataHook("table-details-name").should( + "have.value", + TEST_LIVE_VIEW_2, + ) + cy.getByDataHook("table-details-view-status").should("contain", "Seeding") + cy.getByDataHook("table-details-live-view-memory").should( + "contain", + "222", + ) + + // The drawer aborts the in-flight stale request on target switch, so + // its delayed response may never arrive and cannot be cy.wait-ed on. + // Wait out the delay window instead: by now the stale response has + // either landed and been ignored, or its request died with the abort. + cy.wait(2000) + cy.getByDataHook("table-details-name").should( + "have.value", + TEST_LIVE_VIEW_2, + ) + cy.getByDataHook("table-details-view-status").should("contain", "Seeding") + cy.getByDataHook("table-details-live-view-memory") + .should("contain", "222") + .should("not.contain", "111") + }) + + it("should ignore an in-flight base table response after selecting another live view", () => { + // Given + let oldBaseRequestStarted = false + + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /tables\(\) where table_name/ }, + }, + (req) => { + const query = String(req.query.query ?? "") + if (!oldBaseRequestStarted && query.includes(`'${TEST_TABLE}'`)) { + oldBaseRequestStarted = true + req.continue((res) => { + res.body.dataset = [] + res.body.count = 0 + res.setDelay(1800) + return res + }) + } else { + req.continue() + } + }, + ) + + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + cy.wrap(null).should(() => { + expect(oldBaseRequestStarted).to.equal(true) + }) + + // When + cy.getByDataHook("table-details-name").click() + cy.getByDataHook("table-details-name").clear().type(TEST_LIVE_VIEW_2) + cy.getByDataHook("table-details-name").type("{enter}") + + // Then + cy.getByDataHook("table-details-name").should( + "have.value", + TEST_LIVE_VIEW_2, + ) + cy.getByDataHook("table-details-base-table-status").should( + "contain", + "Valid", + ) + cy.wait(2000) + cy.getByDataHook("table-details-base-table-status") + .should("contain", "Valid") + .and("not.contain", "Dropped") + }) + + it("should not close a newly opened sidebar for a stale empty response", () => { + let injectEmpty = false + let emptyRequestStarted = false + + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /live_views\(\) WHERE view_name/ }, + }, + (req) => { + if (injectEmpty && !emptyRequestStarted) { + emptyRequestStarted = true + req.continue((res) => { + res.body.dataset = [] + res.body.count = 0 + res.setDelay(1800) + return res + }) + } else { + req.continue() + } + }, + ) + + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + cy.getByDataHook("table-details-view-status").should("contain", "Active") + + cy.then(() => { + injectEmpty = true + }) + cy.wrap(null).should(() => { + expect(emptyRequestStarted).to.equal(true) + }) + + cy.getByDataHook("news-panel-button") + .click() + .should("have.attr", "data-selected", "true") + + // Same abort caveat as above: wait out the delay window, then assert + // the stale empty response did not close the newly opened sidebar. + cy.wait(2000) + cy.getByDataHook("news-panel-button").should( + "have.attr", + "data-selected", + "true", + ) + }) + + after(() => { + cy.loadConsoleWithAuth() + cy.dropLiveViewIfExists(TEST_LIVE_VIEW_2) + cy.dropLiveViewIfExists(TEST_LIVE_VIEW) + cy.dropTableIfExists(TEST_LIVE_VIEW_BASE_2) + cy.dropTableIfExists(TEST_TABLE) + }) + }) + + describe("live view invalid state (R5)", () => { + before(() => { + cy.loadConsoleWithAuth() + cy.createTable(TEST_TABLE) + cy.createLiveView(TEST_LIVE_VIEW) + cy.refreshSchema() + cy.getByDataHook("schema-folder-title") + .contains("Live views") + .should("exist") + }) + + it("should show critical health status and permanence guidance for an invalid live view", () => { + // Given + interceptLiveViewsQuery({ + view_status: "invalid", + invalidation_reason: "rename column operation [column=price]", + }) + cy.expandLiveViews() + cy.getByDataHook("schema-liveview-title").should( + "contain", + TEST_LIVE_VIEW, + ) + + // When + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + + // Then + cy.getByDataHook("table-details-health-status") + .should("be.visible") + .should("have.attr", "data-severity", "critical") + cy.getByDataHook("table-details-error-banner") + .should("be.visible") + .should("contain", "Live view is invalid") + .should("contain", "Invalidation is permanent") + cy.getByDataHook("table-details-view-status").should("contain", "Invalid") + cy.getByDataHook("table-details-resume-wal-button").should("not.exist") + }) + + after(() => { + cy.loadConsoleWithAuth() + cy.dropLiveViewIfExists(TEST_LIVE_VIEW) + cy.dropTableIfExists(TEST_TABLE) + }) + }) + + describe("live view load-failure states (R6/R7)", () => { + before(() => { + cy.loadConsoleWithAuth() + cy.createTable(TEST_TABLE) + cy.createLiveView(TEST_LIVE_VIEW) + cy.refreshSchema() + cy.getByDataHook("schema-folder-title") + .contains("Live views") + .should("exist") + }) + + it("should show critical health and retain metric sections for an unreadable live view", () => { + // Given: load-failure stubs report NULL for every diagnostic column + interceptLiveViewsQuery({ + view_status: "state_unreadable", + base_table_name: null, + view_sql: null, + flush_every_interval: null, + flush_every_interval_unit: null, + in_memory_interval: null, + in_memory_interval_unit: null, + view_lower_bound_timestamp: null, + lag_seqtxn: null, + lag_micros: null, + writer_stall_micros: null, + in_mem_rows: null, + in_mem_bytes: null, + below_lower_bound_count: null, + o3_rejected_count: null, + }) + cy.expandLiveViews() + cy.getByDataHook("schema-liveview-title").should( + "contain", + TEST_LIVE_VIEW, + ) + + // When + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + + // Then + cy.getByDataHook("table-details-health-status") + .should("be.visible") + .should("have.attr", "data-severity", "critical") + cy.getByDataHook("table-details-error-banner") + .should("be.visible") + .should("contain", "Live view state files are unreadable") + .should("contain", "drop and recreate the view") + cy.getByDataHook("table-details-view-status").should( + "contain", + "State unreadable", + ) + cy.getByDataHook("table-details-base-table-status") + .should("contain", "Unknown") + .and("not.contain", "Valid") + .and("not.contain", "Suspended") + .and("not.contain", "Dropped") + cy.getByDataHook("table-details-live-view-freshness") + .should("be.visible") + .and("contain", "Unavailable") + .and("not.contain", "Never") + cy.getByDataHook("table-details-live-view-memory") + .should("be.visible") + .and("contain", "Unavailable") + .and("not.contain", "Unknown") + + // When + cy.getByDataHook("table-details-tab-details").click() + + // Then + cy.getByDataHook("table-details-details-section") + .should("be.visible") + .and("contain", "Flush Every") + .and("contain", "In Memory") + .and("contain", "Partitioning") + cy.getByDataHook("table-details-flush-every-card").should( + "contain", + "Unavailable", + ) + cy.getByDataHook("table-details-in-memory-card").should( + "contain", + "Unavailable", + ) + cy.getByDataHook("table-details-start-from-card") + .should("contain", "Unavailable") + .and("not.contain", "Beginning") + }) + + it("should show the version unsupported status and retain metric sections", () => { + // Given: load-failure stubs report NULL for every diagnostic column + cy.loadConsoleWithAuth() + cy.refreshSchema() + interceptLiveViewsQuery({ + view_status: "version_unsupported", + base_table_name: null, + view_sql: null, + flush_every_interval: null, + flush_every_interval_unit: null, + in_memory_interval: null, + in_memory_interval_unit: null, + view_lower_bound_timestamp: null, + lag_seqtxn: null, + lag_micros: null, + writer_stall_micros: null, + in_mem_rows: null, + in_mem_bytes: null, + below_lower_bound_count: null, + o3_rejected_count: null, + }) + cy.expandLiveViews() + cy.getByDataHook("schema-liveview-title").should( + "contain", + TEST_LIVE_VIEW, + ) + + // When + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + + // Then + cy.getByDataHook("table-details-health-status") + .should("be.visible") + .should("have.attr", "data-severity", "critical") + cy.getByDataHook("table-details-view-status").should( + "contain", + "Version unsupported", + ) + cy.getByDataHook("table-details-live-view-freshness") + .should("be.visible") + .and("contain", "Unavailable") + .and("not.contain", "Never") + cy.getByDataHook("table-details-live-view-memory") + .should("be.visible") + .and("contain", "Unavailable") + .and("not.contain", "Unknown") + }) + + after(() => { + cy.loadConsoleWithAuth() + cy.dropLiveViewIfExists(TEST_LIVE_VIEW) + cy.dropTableIfExists(TEST_TABLE) + }) + }) + + describe("live view dropped while the drawer is open", () => { + // beforeEach so a retry starts from a re-created live view: the test + // drops it mid-flow, and a before() would poison the second attempt. + beforeEach(() => { + cy.loadConsoleWithAuth() + cy.createTable(TEST_TABLE) + cy.createLiveView(TEST_LIVE_VIEW) + cy.refreshSchema() + cy.getByDataHook("schema-folder-title") + .contains("Live views") + .should("exist") + }) + + it("should show the empty state after the live view is dropped", () => { + // Given + cy.expandLiveViews() + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + + // When + cy.dropLiveViewIfExists(TEST_LIVE_VIEW) + + // Then: the drawer clears its target; the table selector stays + // rendered and empties so the user can pick another table. + cy.getByDataHook("table-details-name").should("have.value", "") + cy.getByDataHook("table-details-toggle-button").should( + "have.attr", + "data-selected", + "true", + ) + cy.getByDataHook("table-details-empty-state").should("be.visible") + + // When the drawer is closed and reopened + cy.getByDataHook("table-details-toggle-button").click() + cy.getByDataHook("table-details-toggle-button").should( + "have.attr", + "data-selected", + "false", + ) + cy.getByDataHook("table-details-toggle-button").click() + + // Then + cy.getByDataHook("table-details-toggle-button").should( + "have.attr", + "data-selected", + "true", + ) + cy.getByDataHook("table-details-empty-state").should("be.visible") + }) + + after(() => { + cy.loadConsoleWithAuth() + cy.dropLiveViewIfExists(TEST_LIVE_VIEW) + cy.dropTableIfExists(TEST_TABLE) + }) + }) + + describe("view specific", () => { + before(() => { + cy.loadConsoleWithAuth() + cy.createTable(TEST_TABLE) + cy.createView(TEST_VIEW) + }) + + beforeEach(() => { + cy.loadConsoleWithAuth() + cy.refreshSchema() + cy.collapseTables() + cy.collapseMatViews() + cy.expandViews() + }) + + it("should open view details from schema, show View badge, no tabs, only DDL and columns sections with columns expanded", () => { + // Given + let tableMetadataRequests = 0 + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { + query: new RegExp(`tables\\(\\).*${TEST_VIEW}`), + }, + }, + (req) => { + tableMetadataRequests += 1 + req.continue() + }, + ).as("viewTableMetadata") + + // When + cy.openDetailsDrawer(TEST_VIEW, "view") + + // Then + cy.getByDataHook("table-details-type-badge").should("contain", "View") + + cy.getByDataHook("table-details-tab-monitoring").should("not.exist") + cy.getByDataHook("table-details-tab-details").should("not.exist") + + cy.getByDataHook("table-details-ddl-section").should("be.visible") + + cy.getByDataHook("table-details-columns-content").should("be.visible") + + cy.getByDataHook("table-details-details-section").should("not.exist") + + cy.getByDataHook("table-details-health-status") + .should("be.visible") + .should("have.attr", "data-severity", "healthy") + + // When + cy.wait("@viewTableMetadata") + cy.then(() => { + tableMetadataRequests = 0 + }) + cy.wait(2200) + + // Then + cy.then(() => { + expect(tableMetadataRequests).to.be.within(1, 3) + }) + }) + + after(() => { + cy.loadConsoleWithAuth() + cy.dropViewIfExists(TEST_VIEW) + cy.dropTable(TEST_TABLE) + }) + }) + + describe("view invalid state (R4)", () => { + before(() => { + cy.loadConsoleWithAuth() + cy.createTable(TEST_TABLE) + cy.createView(TEST_VIEW) + }) it("should show error banner when view becomes invalid after base table is dropped", () => { cy.loadConsoleWithAuth() @@ -839,7 +1917,7 @@ describe("TableDetailsDrawer", () => { cy.openDetailsDrawer(TEST_TABLE) cy.getByDataHook("table-details-error-ask-ai").should("be.disabled") - hoverForTooltip("table-details-error-ask-ai") + cy.hoverForTooltip(() => cy.getByDataHook("table-details-error-ask-ai")) cy.wait(200) cy.getByDataHook("tooltip").should( "contain", @@ -849,7 +1927,7 @@ describe("TableDetailsDrawer", () => { cy.wait(200) cy.getByDataHook("table-details-warning-ask-ai").should("be.disabled") - hoverForTooltip("table-details-warning-ask-ai") + cy.hoverForTooltip(() => cy.getByDataHook("table-details-warning-ask-ai")) cy.wait(200) cy.getByDataHook("tooltip").should( "contain", @@ -860,7 +1938,7 @@ describe("TableDetailsDrawer", () => { cy.getByDataHook("tooltip").should("not.exist") cy.getByDataHook("table-details-tab-details").click() cy.getByDataHook("table-details-explain-ai").should("be.disabled") - hoverForTooltip("table-details-explain-ai") + cy.hoverForTooltip(() => cy.getByDataHook("table-details-explain-ai")) cy.wait(200) cy.getByDataHook("tooltip").should( "contain", @@ -891,7 +1969,7 @@ describe("TableDetailsDrawer", () => { cy.openDetailsDrawer(TEST_TABLE) cy.getByDataHook("table-details-error-ask-ai").should("be.disabled") - hoverForTooltip("table-details-error-ask-ai") + cy.hoverForTooltip(() => cy.getByDataHook("table-details-error-ask-ai")) cy.wait(200) cy.getByDataHook("tooltip").should( "contain", @@ -901,7 +1979,7 @@ describe("TableDetailsDrawer", () => { cy.wait(200) cy.getByDataHook("table-details-warning-ask-ai").should("be.disabled") - hoverForTooltip("table-details-warning-ask-ai") + cy.hoverForTooltip(() => cy.getByDataHook("table-details-warning-ask-ai")) cy.wait(200) cy.getByDataHook("tooltip").should( "contain", @@ -913,7 +1991,7 @@ describe("TableDetailsDrawer", () => { cy.getByDataHook("table-details-tab-details").click() cy.getByDataHook("table-details-explain-ai").should("be.disabled") cy.getByDataHook("table-details-copy-ddl").should("be.visible").click() - hoverForTooltip("table-details-explain-ai") + cy.hoverForTooltip(() => cy.getByDataHook("table-details-explain-ai")) cy.wait(200) cy.getByDataHook("tooltip").should( "contain", @@ -1205,63 +2283,4 @@ describe("TableDetailsDrawer", () => { cy.dropTable(TEST_TABLE_2) }) }) - - describe("table with STORAGE POLICY", () => { - before(() => { - cy.loadConsoleWithAuth() - cy.createTable(TEST_TABLE) - cy.refreshSchema() - }) - - beforeEach(() => { - cy.intercept( - { - method: "GET", - pathname: "/exec", - query: { query: /SHOW\s+CREATE/i }, - }, - (req) => { - req.continue((res) => { - const row = res.body?.dataset?.[0] - if (row && typeof row[0] === "string") { - row[0] = row[0].replace( - /(\bPARTITION\s+BY\s+\w+)/i, - "$1 STORAGE POLICY(TO PARQUET 3 DAYS, TO REMOTE 10 DAYS, DROP LOCAL 1 YEARS)", - ) - } - }) - }, - ).as("showCreate") - - cy.loadConsoleWithAuth() - cy.expandTables() - }) - - it("hides TTL and renders the storage policy section", () => { - cy.openDetailsDrawer(TEST_TABLE) - cy.getByDataHook("table-details-tab-details").click() - - cy.getByDataHook("table-details-storage-policy-section") - .should("be.visible") - .within(() => { - cy.contains("To Parquet").should("be.visible") - cy.contains("3 Days").should("be.visible") - cy.contains("To Remote").should("be.visible") - cy.contains("10 Days").should("be.visible") - cy.contains("Drop Local").should("be.visible") - cy.contains("1 Year").should("be.visible") - }) - - cy.getByDataHook("table-details-details-section") - .should("be.visible") - .within(() => { - cy.contains("TTL").should("not.exist") - }) - }) - - after(() => { - cy.loadConsoleWithAuth() - cy.dropTable(TEST_TABLE) - }) - }) }) diff --git a/e2e/tests/enterprise/import.spec.js b/e2e/tests/enterprise/import.spec.js index d713df6ef..5caa1cb25 100644 --- a/e2e/tests/enterprise/import.spec.js +++ b/e2e/tests/enterprise/import.spec.js @@ -15,6 +15,6 @@ describe("CSV import in enterprise", () => { }) cy.getByDataHook("import-table-column-schema").should("be.visible") cy.getByDataHook("import-table-column-owner").should("be.visible") - cy.contains("option", "admin").should("exist") + cy.getByDataHook("import-table-owner-select").should("contain", "admin") }) }) diff --git a/e2e/tests/enterprise/oidc.spec.js b/e2e/tests/enterprise/oidc.spec.js index 1c50a4798..a1aa48aaf 100644 --- a/e2e/tests/enterprise/oidc.spec.js +++ b/e2e/tests/enterprise/oidc.spec.js @@ -288,8 +288,9 @@ describe("OIDC", () => { }) cy.getByDataHook("import-table-column-schema").should("be.visible") cy.getByDataHook("import-table-column-owner").should("be.visible") - cy.contains("option", "john doe").should("not.exist") - cy.contains("option", "group1").should("exist") + cy.getByDataHook("import-table-owner-select").click() + cy.contains('[role="menuitemradio"]', "group1").should("exist") + cy.contains('[role="menuitemradio"]', "john doe").should("not.exist") }) }) diff --git a/e2e/tests/enterprise/tableDetails.spec.js b/e2e/tests/enterprise/tableDetails.spec.js index f9117c6a0..425f0a715 100644 --- a/e2e/tests/enterprise/tableDetails.spec.js +++ b/e2e/tests/enterprise/tableDetails.spec.js @@ -1,6 +1,7 @@ /// const TEST_TABLE = "btc_trades" +const TEST_LIVE_VIEW = "btc_trades_lv" describe("TableDetailsDrawer in enterprise", () => { describe("without a STORAGE POLICY shows 'Not configured'", () => { @@ -10,10 +11,28 @@ describe("TableDetailsDrawer in enterprise", () => { cy.refreshSchema() }) - it("renders the section with the 'Not configured' placeholder", () => { + it("shows loading before the 'Not configured' placeholder", () => { + // Given + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /storage_policies/ }, + }, + (req) => { + req.continue((res) => { + res.setDelay(1000) + }) + }, + ).as("storagePolicy") + + // When cy.openDetailsDrawer(TEST_TABLE) cy.getByDataHook("table-details-tab-details").click() + // Then + cy.getByDataHook("table-details-storage-loading").should("be.visible") + cy.wait("@storagePolicy") cy.getByDataHook("table-details-storage-policy-section") .should("be.visible") .within(() => { @@ -29,10 +48,132 @@ describe("TableDetailsDrawer in enterprise", () => { .within(() => { cy.contains("TTL").should("not.exist") }) + + // When + cy.intercept( + { + method: "GET", + pathname: "/exec", + query: { query: /storage_policies/ }, + }, + { + statusCode: 500, + body: { error: "Storage policy unavailable", position: 0 }, + }, + ).as("storagePolicyUnavailable") + // The policy is polled every STORAGE_POLICY_POLL_MS (5s), so each retry + // and the failure threshold behind the banner need more than the 5s + // requestTimeout Cypress applies to an aliased wait by default. + cy.wait("@storagePolicyUnavailable", { requestTimeout: 15000 }) + cy.wait("@storagePolicyUnavailable", { requestTimeout: 15000 }) + cy.wait("@storagePolicyUnavailable", { requestTimeout: 15000 }) + + // Then + cy.getByDataHook("table-details-storage-unavailable", { timeout: 12000 }) + .should("be.visible") + .and("contain", "Unavailable") + }) + + after(() => { + cy.loadConsoleWithAuth() + cy.dropTable(TEST_TABLE) + }) + }) + + describe("with a STORAGE POLICY", () => { + before(() => { + cy.loadConsoleWithAuth() + cy.createTable(TEST_TABLE) + cy.execQuery( + `ALTER TABLE ${TEST_TABLE} SET STORAGE POLICY(TO PARQUET 3 DAYS, TO REMOTE 10 DAYS, DROP LOCAL 1 YEARS)`, + ) + cy.refreshSchema() + }) + + it("renders catalog policy values and disabled status", () => { + // Given + cy.intercept({ + method: "GET", + pathname: "/exec", + query: { query: /storage_policies/ }, + }).as("storagePolicies") + cy.openDetailsDrawer(TEST_TABLE) + + // When + cy.getByDataHook("table-details-tab-details").click() + cy.wait("@storagePolicies") + + // Then the catalogue is read as a bare identifier, not a table function + cy.get("@storagePolicies") + .its("request.url") + .then((url) => { + const query = decodeURIComponent(url) + expect(query).to.contain("storage_policies WHERE table_dir_name") + expect(query).not.to.contain("storage_policies(") + }) + + // Then each duration column renders through its own label + cy.getByDataHook("table-details-storage-policy-section") + .should("be.visible") + .within(() => { + cy.contains("To Parquet").should("be.visible") + cy.contains("3 Days").should("be.visible") + cy.contains("To Remote").should("be.visible") + cy.contains("10 Days").should("be.visible") + cy.contains("Drop Local").should("be.visible") + cy.contains("1 Year").should("be.visible") + // DROP REMOTE was never set, so the catalogue reports it as a zero + // duration and the stage is omitted rather than shown as "0 Hours" + cy.contains("Drop Remote").should("not.exist") + }) + + // When + cy.execQuery(`ALTER TABLE ${TEST_TABLE} DISABLE STORAGE POLICY`) + + // Then the next poll picks the change up, up to 5s away + cy.getByDataHook("table-details-storage-disabled", { timeout: 12000 }) + .should("be.visible") + .and("contain", "Disabled") + }) + + after(() => { + cy.loadConsoleWithAuth() + cy.dropTable(TEST_TABLE) + }) + }) + + describe("hides the STORAGE POLICY section for a live view", () => { + before(() => { + cy.loadConsoleWithAuth() + cy.createTable(TEST_TABLE) + cy.createLiveView(TEST_LIVE_VIEW) + cy.refreshSchema() + cy.expandLiveViews() + }) + + it("shows no storage policy section, unlike a table on the same server", () => { + // Given: enterprise renders the section for a table, so a live view that + // hides it proves the kind guard rather than the enterprise flag. + cy.openDetailsDrawer(TEST_TABLE) + cy.getByDataHook("table-details-tab-details").click() + cy.getByDataHook("table-details-storage-policy-section").should( + "be.visible", + ) + + // When + cy.openDetailsDrawer(TEST_LIVE_VIEW, "liveview") + cy.getByDataHook("table-details-tab-details").click() + + // Then + cy.getByDataHook("table-details-flush-every-card").should("be.visible") + cy.getByDataHook("table-details-storage-policy-section").should( + "not.exist", + ) }) after(() => { cy.loadConsoleWithAuth() + cy.dropLiveViewIfExists(TEST_LIVE_VIEW) cy.dropTable(TEST_TABLE) }) }) diff --git a/src/components/TableSelector/index.tsx b/src/components/TableSelector/index.tsx index 21a4115f9..89662ac63 100644 --- a/src/components/TableSelector/index.tsx +++ b/src/components/TableSelector/index.tsx @@ -4,7 +4,7 @@ import * as RadixPopover from "@radix-ui/react-popover" import Highlighter from "react-highlight-words" import { XIcon } from "@phosphor-icons/react" import { TableIcon } from "../../scenes/Schema/table-icon" -import type { PartitionBy } from "../../utils/questdb" +import type { PartitionBy, TableKind } from "../../utils/questdb" import { VirtualizedTree, type VirtualizedTreeHandle, @@ -16,7 +16,7 @@ import { floatingSurfaceStyles } from "../overlayStyles" export type TableOption = { label: string value: string - kind?: "table" | "matview" | "view" + kind?: TableKind disabled?: boolean walEnabled?: boolean partitionBy?: PartitionBy diff --git a/src/consts/shared-definitions.json b/src/consts/shared-definitions.json index 9432c14f1..16c95af3c 100644 --- a/src/consts/shared-definitions.json +++ b/src/consts/shared-definitions.json @@ -5,7 +5,7 @@ "surfaces": ["ai", "mcp"], "mutatesNotebook": false, "createsNotebook": false, - "description": "Get a list of all tables and materialized views in the QuestDB database", + "description": "Get a list of all tables, materialized views, views and live views in the QuestDB database", "inputSchema": { "type": "object", "additionalProperties": false, @@ -18,14 +18,14 @@ "surfaces": ["ai", "mcp"], "mutatesNotebook": false, "createsNotebook": false, - "description": "Get the full schema definition (DDL) for a specific table or materialized view", + "description": "Get the full schema definition (DDL) for a specific table, materialized view, view or live view", "inputSchema": { "type": "object", "additionalProperties": false, "properties": { "table_name": { "type": "string", - "description": "The name of the table or materialized view to get schema for" + "description": "The name of the table, materialized view, view or live view to get schema for" } }, "required": ["table_name"] @@ -37,14 +37,14 @@ "surfaces": ["ai", "mcp"], "mutatesNotebook": false, "createsNotebook": false, - "description": "Get the runtime details/statistics of a specific table or materialized view", + "description": "Get the runtime details/statistics of a specific table, materialized view, view or live view", "inputSchema": { "type": "object", "additionalProperties": false, "properties": { "table_name": { "type": "string", - "description": "The name of the table or materialized view to get details for" + "description": "The name of the table, materialized view, view or live view to get details for" } }, "required": ["table_name"] diff --git a/src/hooks/useAIQuickActions.ts b/src/hooks/useAIQuickActions.ts index a1b5f8070..dd1d31253 100644 --- a/src/hooks/useAIQuickActions.ts +++ b/src/hooks/useAIQuickActions.ts @@ -17,29 +17,30 @@ import type { } from "../scenes/Schema/TableDetailsDrawer/healthCheck" import * as QuestDB from "../utils/questdb" import { selectors } from "../store" -import type { PartitionBy, QueryResult, Table } from "../utils/questdb/types" +import { getTableKindLabel } from "../utils/questdb/types" +import type { + LiveView, + MaterializedView, + PartitionBy, + QueryResult, + Table, + TableKind, +} from "../utils/questdb/types" + +type HealthIssueDiagnosticContext = { + source: "materialized_views()" | "live_views()" + data: MaterializedView | LiveView + guidance?: string +} type SchemaDisplayData = { tableName: string - kind: "table" | "matview" | "view" + kind: TableKind partitionBy?: PartitionBy walEnabled?: boolean designatedTimestamp?: string } -const getTableKindLabel = (kind: "table" | "matview" | "view") => { - switch (kind) { - case "table": - return "Table" - case "matview": - return "Materialized view" - case "view": - return "View" - default: - return "" - } -} - export const useAIQuickActions = () => { const { quest } = useContext(QuestContext) const tables = useSelector(selectors.query.getTables) @@ -70,27 +71,18 @@ export const useAIQuickActions = () => { const getTableSchema = async ( tableName: string, - kind: "table" | "matview" | "view", + kind: TableKind, ): Promise => { try { - const response = - kind === "matview" - ? await quest.showMatViewDDL(tableName) - : kind === "view" - ? await quest.showViewDDL(tableName) - : await quest.showTableDDL(tableName) + const response = await quest.showDDL(tableName, kind) if (response?.type === QuestDB.Type.DQL && response.data?.[0]?.ddl) { return response.data[0].ddl } } catch (_error) { - const kindLabel = - kind === "matview" - ? "materialized view" - : kind === "view" - ? "view" - : "table" - toast.error(`Cannot fetch schema for ${kindLabel} '${tableName}'`) + toast.error( + `Cannot fetch schema for ${getTableKindLabel(kind).toLowerCase()} '${tableName}'`, + ) } return null } @@ -98,7 +90,7 @@ export const useAIQuickActions = () => { const handleExplainSchema = async ( id: number, name: string, - kind: "table" | "matview" | "view", + kind: TableKind, schemaDisplayData?: Omit, ) => { if (isBlockingAIStatus(status)) { @@ -202,6 +194,7 @@ export const useAIQuickActions = () => { tableName: string, issue: HealthIssue, trendSamples?: TimestampedSample[], + diagnosticContext?: HealthIssueDiagnosticContext, ) => { if (isBlockingAIStatus(status)) { return @@ -231,7 +224,16 @@ export const useAIQuickActions = () => { toast.error(`Cannot fetch details for table '${tableName}'`) return } - const tableDetails = JSON.stringify(tableDetailsResponse.data[0], null, 2) + const tableDetails = QuestDB.stringifyWithBigInts( + tableDetailsResponse.data[0], + 2, + ) + const diagnosticDetails = diagnosticContext + ? { + source: diagnosticContext.source, + data: QuestDB.stringifyWithBigInts(diagnosticContext.data, 2), + } + : undefined let monitoringDocs: string try { @@ -256,10 +258,12 @@ export const useAIQuickActions = () => { id: issue.id, field: issue.field, message: issue.message, - currentValue: issue.currentValue, + currentValue: issue.promptValue ?? issue.currentValue, severity: issue.severity as "critical" | "warning", }, tableDetails, + diagnosticDetails, + issueGuidance: diagnosticContext?.guidance, monitoringDocs, trendSamples, settings: { model: currentModel, apiKey }, @@ -305,10 +309,12 @@ export const useAIQuickActions = () => { id: issue.id, field: issue.field, message: issue.message, - currentValue: issue.currentValue, + currentValue: issue.promptValue ?? issue.currentValue, severity: issue.severity as "critical" | "warning", }, tableDetails, + diagnosticDetails, + issueGuidance: diagnosticContext?.guidance, monitoringDocs, trendSamples, settings: { model: currentModel, apiKey }, diff --git a/src/providers/AIConversationProvider/types.ts b/src/providers/AIConversationProvider/types.ts index a6592da7a..bd1663ea3 100644 --- a/src/providers/AIConversationProvider/types.ts +++ b/src/providers/AIConversationProvider/types.ts @@ -1,4 +1,4 @@ -import type { PartitionBy } from "../../utils/questdb" +import type { PartitionBy, TableKind } from "../../utils/questdb" import type { QueryKey } from "../../scenes/Editor/Monaco/utils" import type { Message } from "../../utils/ai/types" import type { RanStatus } from "../../utils/ai/runStatus" @@ -10,7 +10,7 @@ export type ConversationId = string export type SchemaDisplayData = { tableName: string - kind: "table" | "matview" | "view" + kind: TableKind partitionBy?: PartitionBy walEnabled?: boolean designatedTimestamp?: string diff --git a/src/providers/QuestProvider/index.tsx b/src/providers/QuestProvider/index.tsx index 115667b47..c06971be3 100644 --- a/src/providers/QuestProvider/index.tsx +++ b/src/providers/QuestProvider/index.tsx @@ -127,7 +127,7 @@ export const QuestProvider: React.FC = ({ children }) => { .then((result) => { if (result.type === QuestDB.Type.DQL && result.count === 1) { setBuildVersion(formatVersion(result.dataset[0][0] as string)) - setCommitHash(formatCommitHash(result.dataset[0][0])) + setCommitHash(formatCommitHash(result.dataset[0][0] as string)) } }) }, []) diff --git a/src/scenes/Editor/AIChatWindow/ChatMessages.tsx b/src/scenes/Editor/AIChatWindow/ChatMessages.tsx index 42e6a9d99..3b5c33cc4 100644 --- a/src/scenes/Editor/AIChatWindow/ChatMessages.tsx +++ b/src/scenes/Editor/AIChatWindow/ChatMessages.tsx @@ -534,8 +534,7 @@ export const ChatMessages: React.FC = ({ type: "tableDetails", payload: { tableName: table.table_name, - isMatView: table.table_type === "M", - isView: table.table_type === "V", + kind: getTableKind(table), }, }), ) diff --git a/src/scenes/Editor/AIChatWindow/index.tsx b/src/scenes/Editor/AIChatWindow/index.tsx index 4999bf773..7b01d8122 100644 --- a/src/scenes/Editor/AIChatWindow/index.tsx +++ b/src/scenes/Editor/AIChatWindow/index.tsx @@ -47,7 +47,7 @@ import { createFixFlowConfig, createSchemaExplainFlowConfig, } from "../../../utils/ai/executeAIFlow" -import { getTableKindLabel } from "../../Schema/VirtualTables" +import { getTableKind, getTableKindLabel } from "../../../utils/questdb/types" import * as QuestDB from "../../../utils/questdb" import { QuestContext } from "../../../providers" import { useDispatch, useSelector } from "react-redux" @@ -615,8 +615,7 @@ const AIChatWindow: React.FC = () => { type: "tableDetails", payload: { tableName: table.table_name, - isMatView: table.table_type === "M", - isView: table.table_type === "V", + kind: getTableKind(table), }, }), ) @@ -783,12 +782,10 @@ const AIChatWindow: React.FC = () => { const schemaData = userMessage.displaySchemaData try { - const ddlResult = - schemaData.kind === "matview" - ? await quest.showMatViewDDL(schemaData.tableName) - : schemaData.kind === "view" - ? await quest.showViewDDL(schemaData.tableName) - : await quest.showTableDDL(schemaData.tableName) + const ddlResult = await quest.showDDL( + schemaData.tableName, + schemaData.kind, + ) if ( ddlResult?.type !== QuestDB.Type.DQL || diff --git a/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx b/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx index f278619a0..878c65bbf 100644 --- a/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx +++ b/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx @@ -230,6 +230,7 @@ export const FilesToUpload = ({ diff --git a/src/scenes/Schema/Row/index.tsx b/src/scenes/Schema/Row/index.tsx index d9d631a09..1e962e175 100644 --- a/src/scenes/Schema/Row/index.tsx +++ b/src/scenes/Schema/Row/index.tsx @@ -56,6 +56,7 @@ import { Checkbox } from "../checkbox" import { Tooltip } from "../../../components/Tooltip" import { mapColumnTypeToUI } from "../../../scenes/Import/ImportCSVFiles/utils" import { + LIVEVIEWS_GROUP_KEY, MATVIEWS_GROUP_KEY, TABLES_GROUP_KEY, VIEWS_GROUP_KEY, @@ -68,9 +69,20 @@ export type TreeNodeKind = | "table" | "matview" | "view" + | "liveview" | "folder" | "detail" +const TABLE_NODE_KINDS: TreeNodeKind[] = [ + "table", + "matview", + "view", + "liveview", +] + +const isTableNodeKind = (kind: TreeNodeKind): kind is QuestDB.TableKind => + TABLE_NODE_KINDS.includes(kind) + type Props = Readonly<{ id: string index: number @@ -407,16 +419,17 @@ const Row = ({ const pulseTimeoutRef = useRef | null>(null) const wrapperRef = useRef(null) const isExpandable = - ["folder", "table", "matview", "view"].includes(kind) || + ["folder", "table", "matview", "view", "liveview"].includes(kind) || (kind === "column" && type === "SYMBOL") - const isTableKind = ["table", "matview", "view"].includes(kind) + const isTableKind = isTableNodeKind(kind) const isRootFolder = [ MATVIEWS_GROUP_KEY, TABLES_GROUP_KEY, VIEWS_GROUP_KEY, + LIVEVIEWS_GROUP_KEY, ].includes(id ?? "") const matchesSearch = - ["column", "table", "matview", "view"].includes(kind) && + ["column", "table", "matview", "view", "liveview"].includes(kind) && query && name.toLowerCase().includes(query.toLowerCase()) @@ -611,15 +624,18 @@ const Row = ({ designatedTimestamp={designatedTimestamp} partitionBy={partitionBy} walEnabled={walEnabled} - kind={kind as "table" | "matview" | "view"} + kind={kind} /> )} {kind === "detail" && } - {["column", "table", "matview", "view"].includes(kind) ? ( + {["column", "table", "matview", "view", "liveview"].includes( + kind, + ) ? ( ) : ( name diff --git a/src/scenes/Schema/SchemaContext.tsx b/src/scenes/Schema/SchemaContext.tsx index b00b40e5d..82a7fb6df 100644 --- a/src/scenes/Schema/SchemaContext.tsx +++ b/src/scenes/Schema/SchemaContext.tsx @@ -1,21 +1,17 @@ import React, { createContext, useContext, useState, useMemo } from "react" -import { TreeNodeKind } from "./Row" +import type { TableKind } from "../../utils/questdb/types" + +export type SelectedTable = { name: string; type: TableKind } export const SchemaContext = createContext<{ query: string setQuery: (query: string) => void selectOpen: boolean setSelectOpen: (open: boolean) => void - selectedTables: { name: string; type: TreeNodeKind }[] - setSelectedTables: (tables: { name: string; type: TreeNodeKind }[]) => void - handleSelectToggle: ({ - name, - type, - }: { - name: string - type: TreeNodeKind - }) => void - selectedTablesMap: Map + selectedTables: SelectedTable[] + setSelectedTables: (tables: SelectedTable[]) => void + handleSelectToggle: (table: SelectedTable) => void + selectedTablesMap: Map focusedIndex: number | null setFocusedIndex: (index: number | null) => void }>({ @@ -44,9 +40,7 @@ export const SchemaProvider: React.FC<{ children: React.ReactNode }> = ({ }) => { const [query, setQuery] = useState("") const [selectOpen, _setSelectOpen] = useState(false) - const [selectedTables, setSelectedTables] = useState< - { name: string; type: TreeNodeKind }[] - >([]) + const [selectedTables, setSelectedTables] = useState([]) const [focusedIndex, setFocusedIndex] = useState(null) const selectedTablesMap = useMemo( @@ -57,13 +51,7 @@ export const SchemaProvider: React.FC<{ children: React.ReactNode }> = ({ [selectedTables], ) - const handleSelectToggle = ({ - name, - type, - }: { - name: string - type: TreeNodeKind - }) => { + const handleSelectToggle = ({ name, type }: SelectedTable) => { const key = `${name}-${type}` if (selectedTablesMap.has(key)) { setSelectedTables( diff --git a/src/scenes/Schema/SuspensionDialog/index.tsx b/src/scenes/Schema/SuspensionDialog/index.tsx index 67eb0cf59..eb49450d4 100644 --- a/src/scenes/Schema/SuspensionDialog/index.tsx +++ b/src/scenes/Schema/SuspensionDialog/index.tsx @@ -10,7 +10,7 @@ import { Input, CopyButton, } from "../../../components" -import { getTableKindLabel } from "../VirtualTables" +import { getTableKindLabel } from "../../../utils/questdb/types" import { Undo, CheckCircle, Database2, Files } from "../../../components/icons" import { trackEvent } from "../../../modules/ConsoleEventTracker" import { ConsoleEvent } from "../../../modules/ConsoleEventTracker/events" @@ -88,7 +88,7 @@ const GENERIC_ERROR_TEXT = "Error restarting transaction" type Props = { tableName: string open: boolean - kind: "table" | "matview" | "view" + kind: Exclude onOpenChange: (open: boolean) => void } @@ -109,7 +109,7 @@ export const SuspensionDialog = ({ const fetchWalTableData = useCallback(async () => { try { - const escapedName = tableName.replace(/'/g, "''") + const escapedName = QuestDB.escapeSqlLiteral(tableName) const response = await quest.query( `wal_tables() WHERE name = '${escapedName}'`, ) @@ -142,8 +142,14 @@ export const SuspensionDialog = ({ void trackEvent(ConsoleEvent.SCHEMA_RESUME_WAL_SUBMIT) setIsSubmitting(true) setError(undefined) - const escapedName = tableName.replace(/'/g, "''") - const queryStart = `ALTER ${kind === "matview" ? "MATERIALIZED VIEW" : "TABLE"}` + const escapedName = QuestDB.escapeSqlLiteral(tableName) + const queryStart = `ALTER ${ + kind === "matview" + ? "MATERIALIZED VIEW" + : kind === "liveview" + ? "LIVE VIEW" + : "TABLE" + }` try { const response = await quest.query( `${queryStart} '${escapedName}' RESUME WAL${ diff --git a/src/scenes/Schema/TableDetailsDrawer/DetailsTab.tsx b/src/scenes/Schema/TableDetailsDrawer/DetailsTab.tsx index 712365d94..163762f35 100644 --- a/src/scenes/Schema/TableDetailsDrawer/DetailsTab.tsx +++ b/src/scenes/Schema/TableDetailsDrawer/DetailsTab.tsx @@ -1,4 +1,4 @@ -import React, { useMemo } from "react" +import React from "react" import styled, { useTheme } from "styled-components" import { CodeIcon, @@ -10,13 +10,14 @@ import { } from "@phosphor-icons/react" import { Box, Text, CopyButton, TextButton } from "../../../components" import { LiteEditor } from "../../../components/LiteEditor" -import type { - Table, - MaterializedView, - View, - Column, -} from "../../../utils/questdb/types" -import { formatTTL, extractStoragePolicyClauses } from "./utils" +import type { Table, Column, StoragePolicy } from "../../../utils/questdb/types" +import type { SourceState, TableKindData } from "./types" +import { + formatTTL, + formatInterval, + formatUtcTimestamp, + formatStoragePolicyClauses, +} from "./utils" import { ColumnIcon } from "../Row" import { Section, @@ -25,24 +26,24 @@ import { SectionTitleClickable, SectionTitleContainer, CaretIcon, + UnavailableValue, } from "./shared-styles" import { SchemaAIButton } from "./SchemaAIButton" import { ErrorBanner } from "./ErrorBanner" -import { ISSUE_DOCS_URLS } from "./healthCheck" +import { ISSUE_DOCS_URLS, isLiveViewLoadFailure } from "./healthCheck" import { useEditor } from "../../../providers" import { trackEvent } from "../../../modules/ConsoleEventTracker" import { ConsoleEvent } from "../../../modules/ConsoleEventTracker/events" export interface DetailsTabProps { tableData: Table - matViewData: MaterializedView | null - viewData: View | null - columns: Column[] - ddl: string - isMatView: boolean - isView: boolean + kindData: TableKindData + columnsState: SourceState + ddlState: SourceState + storagePolicyState: SourceState isEnterprise: boolean truncatedDDL: { text: string; grayedOutLines: [number, number] | null } + baseTableName: string | undefined baseTableStatus: "Valid" | "Suspended" | "Dropped" | null columnsExpanded: boolean onColumnsExpandedChange: (expanded: boolean) => void @@ -140,14 +141,13 @@ const ButtonsContainer = styled(Box).attrs({ export const DetailsTab = ({ tableData, - matViewData, - viewData, - columns, - ddl, - isMatView, - isView, + kindData, + columnsState, + ddlState, + storagePolicyState, isEnterprise, truncatedDDL, + baseTableName, baseTableStatus, columnsExpanded, onColumnsExpandedChange, @@ -157,33 +157,60 @@ export const DetailsTab = ({ }: DetailsTabProps) => { const { addBuffer } = useEditor() const theme = useTheme() + const viewState = kindData.kind === "view" ? kindData.view : null + const matViewState = kindData.kind === "matview" ? kindData.matView : null + const liveViewState = kindData.kind === "liveview" ? kindData.liveView : null + const view = viewState?.status === "ready" ? viewState.data : null + const matView = matViewState?.status === "ready" ? matViewState.data : null + const liveView = liveViewState?.status === "ready" ? liveViewState.data : null + const kindSourceUnavailable = + viewState?.status === "unavailable" || + matViewState?.status === "unavailable" || + liveViewState?.status === "unavailable" + const liveViewUnavailable = liveViewState?.status === "unavailable" + const liveViewDiagnosticsUnavailable = + liveViewUnavailable || isLiveViewLoadFailure(liveView) + const matViewUnavailable = matViewState?.status === "unavailable" + const columns = columnsState.status === "ready" ? columnsState.data : [] + const ddl = ddlState.status === "ready" ? ddlState.data : "" const baseTableExists = baseTableStatus === "Valid" || baseTableStatus === "Suspended" - const storagePolicyClauses = useMemo( - () => extractStoragePolicyClauses(ddl), - [ddl], - ) + const storagePolicy = + storagePolicyState.status === "ready" ? storagePolicyState.data : null + const storagePolicyClauses = formatStoragePolicyClauses(storagePolicy) + const storagePolicyDisabled = storagePolicy?.status === "D" const hasStoragePolicy = storagePolicyClauses.length > 0 const hasTtl = (tableData.ttlValue ?? 0) !== 0 - const showStoragePolicySection = isEnterprise || hasStoragePolicy + const showStoragePolicySection = kindData.kind === "table" && isEnterprise + const showDetailsSection = + kindData.kind === "table" || + kindData.kind === "matview" || + liveView !== null || + liveViewUnavailable return ( <> - {isMatView && matViewData && ( + {(baseTableName || + ((kindData.kind === "matview" || kindData.kind === "liveview") && + kindSourceUnavailable)) && ( Base Table - - {matViewData.base_table_name} - + {kindSourceUnavailable ? ( + + ) : ( + + {baseTableName} + + )} {baseTableExists && ( )} - {isView && viewData?.view_status === "invalid" && ( + {view?.view_status === "invalid" && (
@@ -214,12 +241,19 @@ export const DetailsTab = ({ Explain with AI - {ddl && ( + {ddlState.status === "unavailable" ? ( + + ) : ddl ? ( - )} + ) : null}
{/* Columns Section */} - {columns.length === 0 ? ( + {columnsState.status === "unavailable" ? ( +
+ + + Columns + + +
+ ) : columnsState.status === "loading" ? ( +
+ + + Columns + + Loadingโ€ฆ +
+ ) : columns.length === 0 ? (
)} - {/* Details Section - layout differs by type, hidden for views */} - {!isView && ( + {/* Details Section - layout differs by type and stays hidden for views. */} + {showDetailsSection && (
Details - {isMatView && matViewData ? ( + {liveView || liveViewUnavailable ? ( + /* Live view: 4 cards (2ร—2). TTL, dedup and refresh type do not apply. */ + + + Flush Every + + {liveViewDiagnosticsUnavailable ? ( + + ) : ( + formatInterval( + liveView?.flush_every_interval ?? null, + liveView?.flush_every_interval_unit ?? null, + ) + )} + + + + In Memory + + {liveViewDiagnosticsUnavailable ? ( + + ) : ( + formatInterval( + liveView?.in_memory_interval ?? null, + liveView?.in_memory_interval_unit ?? null, + ) + )} + + + + Start From + + {liveViewDiagnosticsUnavailable ? ( + + ) : liveView?.view_lower_bound_timestamp ? ( + formatUtcTimestamp(liveView.view_lower_bound_timestamp) + ) : ( + "Beginning" + )} + + + + Partitioning + + {tableData.partitionBy === "NONE" + ? "None" + : tableData.partitionBy.charAt(0).toUpperCase() + + tableData.partitionBy.slice(1).toLowerCase()} + + + + ) : kindData.kind === "matview" ? ( /* Matview: 4 cards (2ร—2) when TTL is configured, 3 cards (1 row) when not. */ {hasTtl && ( @@ -343,13 +463,19 @@ export const DetailsTab = ({ Refresh Type - {matViewData.refresh_type.charAt(0).toUpperCase() + - matViewData.refresh_type.slice(1).toLowerCase()} + {matViewUnavailable ? ( + + ) : matView ? ( + matView.refresh_type.charAt(0).toUpperCase() + + matView.refresh_type.slice(1).toLowerCase() + ) : ( + Loadingโ€ฆ + )} - ) : ( - /* Table: 3 cards (1 row) when TTL is configured, 2 cards (1 row) when not. */ + ) : kindData.kind === "table" ? ( + /* Table: 3 cards when TTL is configured, 2 when not. */ {hasTtl && ( @@ -375,28 +501,53 @@ export const DetailsTab = ({ - )} + ) : null}
)} - {!isView && showStoragePolicySection && ( + {showStoragePolicySection && (
Storage policy - {hasStoragePolicy ? ( - - {storagePolicyClauses.map((clause) => ( - + ) : storagePolicyState.status === "loading" ? ( + + Loadingโ€ฆ + + ) : hasStoragePolicy ? ( + + {storagePolicyDisabled && ( + - {clause.action} - {clause.duration} - - ))} - + + Disabled + + )} + + {storagePolicyClauses.map((clause) => ( + + {clause.action} + {clause.duration} + + ))} + + ) : ( void + onAskAI?: () => void docsUrl?: string showResumeButton?: boolean onResume?: () => void @@ -72,6 +72,10 @@ export const ErrorBanner = ({ showResumeButton, onResume, }: Props) => { + const hasActions = Boolean( + onAskAI || docsUrl || (showResumeButton && onResume), + ) + return ( @@ -85,31 +89,35 @@ export const ErrorBanner = ({ {description && {description}} - - {showResumeButton && onResume && ( - - Resume WAL - - )} - - Ask AI - - {docsUrl && ( - - View explanation in docs - - )} - + {hasActions && ( + + {showResumeButton && onResume && ( + + Resume WAL + + )} + {onAskAI && ( + + Ask AI + + )} + {docsUrl && ( + + View explanation in docs + + )} + + )} ) } diff --git a/src/scenes/Schema/TableDetailsDrawer/HealthStatusLabel.tsx b/src/scenes/Schema/TableDetailsDrawer/HealthStatusLabel.tsx index 2c9abb616..38c1f80e8 100644 --- a/src/scenes/Schema/TableDetailsDrawer/HealthStatusLabel.tsx +++ b/src/scenes/Schema/TableDetailsDrawer/HealthStatusLabel.tsx @@ -21,6 +21,8 @@ const LabelContainer = styled.div<{ $severity: HealthSeverity }>` return `${theme.color.statusDanger}1F` case "warning": return `${theme.color.statusWarning}1F` + case "unknown": + return `${theme.color.contentDisabled}1F` case "recovering": case "healthy": default: @@ -88,6 +90,8 @@ const StatusSquare = styled(Square)<{ $severity: HealthSeverity }>` return theme.color.statusDanger case "warning": return theme.color.statusWarning + case "unknown": + return theme.color.contentDisabled case "recovering": case "healthy": default: @@ -102,7 +106,9 @@ export const HealthStatusLabel = ({ severity }: Props) => { ? "Error" : severity === "warning" ? "Warning" - : "Healthy" + : severity === "unknown" + ? "Unknown" + : "Healthy" return ( diff --git a/src/scenes/Schema/TableDetailsDrawer/MonitoringTab.tsx b/src/scenes/Schema/TableDetailsDrawer/MonitoringTab.tsx index a9adfa986..af215d244 100644 --- a/src/scenes/Schema/TableDetailsDrawer/MonitoringTab.tsx +++ b/src/scenes/Schema/TableDetailsDrawer/MonitoringTab.tsx @@ -12,17 +12,25 @@ import { ArrowUpRightIcon, ArrowDownRightIcon, ArrowRightIcon, + TimerIcon, + MemoryIcon, } from "@phosphor-icons/react" import { SquareWithShadow } from "./HealthStatusLabel" import { Badge, Box, CopyButton, Text, Tooltip } from "../../../components" -import type { Table, MaterializedView } from "../../../utils/questdb/types" +import { type LiveView, type Table } from "../../../utils/questdb/types" +import type { TableKindData } from "./types" import { formatRelativeTimestamp, formatMemoryPressure, formatRowCount, + formatMicrosDuration, + formatBytes, + formatTxnCount, } from "./utils" import { ISSUE_DOCS_URLS, + getLiveViewIssueGuidance, + isLiveViewLoadFailure, type HealthStatus, type HealthSeverity, type HealthIssue, @@ -37,17 +45,20 @@ import { SectionTitleClickable, SectionTitleContainer, CaretIcon, + UnavailableValue, } from "./shared-styles" +const BIGINT_ZERO = BigInt(0) + export interface MonitoringTabProps { tableData: Table - matViewData: MaterializedView | null - isMatView: boolean + kindData: TableKindData healthStatus: HealthStatus | null criticalIssues: HealthIssue[] performanceWarnings: HealthIssue[] isIngestionActive: boolean isIngestionDisabled: boolean + baseTableName: string | undefined baseTableStatus: "Valid" | "Suspended" | "Dropped" | null walExpanded: boolean onWalExpandedChange: (expanded: boolean) => void @@ -55,7 +66,7 @@ export interface MonitoringTabProps { onAskAI: (issue: HealthIssue) => void } -const RowCountIndicatorInner = styled.div<{ $isMatView?: boolean }>` +const RowCountIndicatorInner = styled.div<{ $attachedToStatus?: boolean }>` display: flex; align-items: center; gap: 0.5rem; @@ -65,8 +76,8 @@ const RowCountIndicatorInner = styled.div<{ $isMatView?: boolean }>` width: 100%; font-size: ${({ theme }) => theme.fontSize.md}; color: ${({ theme }) => theme.color.contentPrimary}; - ${({ $isMatView }) => - $isMatView && + ${({ $attachedToStatus }) => + $attachedToStatus && css` border-bottom-left-radius: 0 !important; border-bottom-right-radius: 0 !important; @@ -84,15 +95,15 @@ const TimestampUnderline = styled.span` color: ${({ theme }) => theme.color.contentSecondary}; ` -const MetricsGrid = styled.div<{ $isMatView?: boolean }>` +const MetricsGrid = styled.div<{ $attachedToRowCount?: boolean }>` width: 100%; display: grid; grid-template-columns: repeat(2, 1fr); gap: 0.2rem; border-radius: 0.5rem; overflow: hidden; - ${({ $isMatView }) => - $isMatView && + ${({ $attachedToRowCount }) => + $attachedToRowCount && css` border-top-left-radius: 0 !important; border-top-right-radius: 0 !important; @@ -124,21 +135,29 @@ const MetricValue = styled(Text).attrs({ white-space: nowrap; ` -const TwoColumnGrid = styled.div` +const ConfigGrid = styled.div<{ $columns: number }>` display: grid; - grid-template-columns: repeat(2, 1fr); + grid-template-columns: repeat(${({ $columns }) => $columns}, 1fr); gap: 1rem; padding: 0 1rem; ` -const ConfigItem = styled(Box).attrs<{ $background?: string }>({ +const ConfigItem = styled(Box).attrs<{ + $background?: string + $fullWidth?: boolean +}>({ flexDirection: "column", gap: "0.5rem", align: "flex-start", -})<{ $background?: string }>` +})<{ $background?: string; $fullWidth?: boolean }>` background: ${({ $background }) => $background}; min-width: 0; overflow: hidden; + ${({ $fullWidth }) => + $fullWidth && + css` + grid-column: 1 / -1; + `} ` const RateText = styled(Text)` @@ -217,6 +236,8 @@ const getSeverityColor = ( return theme.color.statusDanger case "warning": return theme.color.statusWarning + case "unknown": + return theme.color.contentDisabled case "recovering": return theme.color.statusSuccess default: @@ -251,6 +272,37 @@ export const HELPER_TEXT = { ), + liveViewLag: ( + + Base-table transactions the view has not yet applied and flushed. This + normally rises between flushes and drops when a flush completes. + + ), + liveViewSinceLastFlush: ( + <> + + Time since the last successful flush. This measures flush activity, not + data staleness; it keeps growing while the base table is idle. + + + ), + liveViewInMemory: ( + <> + + Rows in Memory is the live row count of the in-memory tier and drops as + rows age out. Memory Footprint is a peak high-water mark that does not + shrink after a burst. + + + ), + liveViewDroppedRows: ( + <> + + Rows the START FROM boundary excluded from the view, split into in-order + and out-of-order arrivals. Counters reset on restart. + + + ), transactionLag: ( <> @@ -304,25 +356,44 @@ const getTrendAssets = ( } } -const formatRate = (rate: number, field: string): string => { - const absRate = Math.abs(rate) - const unit = field === "transactionLag" ? "transactions/s" : "rows/s" - +const formatRateMagnitude = (absRate: number): string => { if (absRate >= 1_000_000_000_000) { - return `${(absRate / 1_000_000_000_000).toFixed(1)}T ${unit}` + return `${(absRate / 1_000_000_000_000).toFixed(1)}T` } if (absRate >= 1_000_000_000) { - return `${(absRate / 1_000_000_000).toFixed(1)}B ${unit}` + return `${(absRate / 1_000_000_000).toFixed(1)}B` } if (absRate >= 1_000_000) { - return `${(absRate / 1_000_000).toFixed(1)}M ${unit}` + return `${(absRate / 1_000_000).toFixed(1)}M` } if (absRate >= 1_000) { - return `${(absRate / 1_000).toFixed(1)}K ${unit}` + return `${(absRate / 1_000).toFixed(1)}K` } - return `${Math.round(absRate)} ${unit}` + return `${Math.round(absRate)}` +} + +const formatRate = (rate: number, field: string): string => { + const unit = field === "transactionLag" ? "transactions/s" : "rows/s" + const magnitude = formatRateMagnitude(Math.abs(rate)) + const sign = magnitude === "0" ? "" : rate > 0 ? "+" : "-" + return `${sign}${magnitude} ${unit}` } +type LiveViewFailureStatus = Extract< + LiveView["view_status"], + "invalid" | "version_unsupported" | "state_unreadable" +> + +const LIVE_VIEW_FAILURE_STATUS_LABELS: Record = { + invalid: "Invalid", + version_unsupported: "Version unsupported", + state_unreadable: "State unreadable", +} + +const isLiveViewFailureStatus = ( + status: LiveView["view_status"], +): status is LiveViewFailureStatus => status in LIVE_VIEW_FAILURE_STATUS_LABELS + const ConfigItemWithHealth = ({ label, helperText, @@ -330,6 +401,8 @@ const ConfigItemWithHealth = ({ issue, showTrend, trend, + boxedValue, + fullWidth, dataHook, }: { label: string @@ -338,6 +411,8 @@ const ConfigItemWithHealth = ({ issue?: HealthIssue showTrend?: boolean trend?: TrendIndicator + boxedValue?: boolean + fullWidth?: boolean dataHook?: string }) => { const theme = useTheme() @@ -346,6 +421,7 @@ const ConfigItemWithHealth = ({ : undefined const iconColor = issue ? getSeverityColor(theme, issue.severity) : undefined + const isWarningTrend = trend?.direction === "increasing" const trendValue = ( {value} {showTrend && trend && ( - + {trendAssets?.icon} - {trend.rate > 0 ? "+" : "-"} {formatRate(trend.rate, trend.field)} @@ -377,7 +445,7 @@ const ConfigItemWithHealth = ({ ) return ( - + {label} @@ -397,6 +465,12 @@ const ConfigItemWithHealth = ({ ) : ( trendValue ) + ) : boxedValue ? ( + + + {value} + + ) : ( {value} @@ -408,13 +482,13 @@ const ConfigItemWithHealth = ({ export const MonitoringTab = ({ tableData, - matViewData, - isMatView, + kindData, healthStatus, criticalIssues, performanceWarnings, isIngestionActive, isIngestionDisabled, + baseTableName, baseTableStatus, walExpanded, onWalExpandedChange, @@ -422,28 +496,46 @@ export const MonitoringTab = ({ onAskAI, }: MonitoringTabProps) => { const theme = useTheme() + const matViewState = kindData.kind === "matview" ? kindData.matView : null + const liveViewState = kindData.kind === "liveview" ? kindData.liveView : null + const matView = matViewState?.status === "ready" ? matViewState.data : null + const liveView = liveViewState?.status === "ready" ? liveViewState.data : null + const matViewUnavailable = matViewState?.status === "unavailable" + const liveViewUnavailable = liveViewState?.status === "unavailable" + const liveViewDiagnosticsUnavailable = + liveViewUnavailable || isLiveViewLoadFailure(liveView) const lastWriteTimestamp = (() => { if (!tableData.table_last_write_timestamp) return null const date = new Date(tableData.table_last_write_timestamp) if (isNaN(date.getTime()) || date.getTime() === 0) return null return date.toISOString() })() - + const hasStatusSection = matViewState !== null || liveViewState !== null + const hasLiveViewDroppedRows = + liveView !== null && + ((liveView.below_lower_bound_count ?? BIGINT_ZERO) > BIGINT_ZERO || + (liveView.o3_rejected_count ?? BIGINT_ZERO) > BIGINT_ZERO) return ( <> {/* Critical Error Banners */} {criticalIssues.length > 0 && (
- + {criticalIssues.map((issue) => ( @@ -492,33 +584,66 @@ export const MonitoringTab = ({
- {/* Matview Status Section */} - {isMatView && matViewData && ( + {/* View Status Section (matview and live view) */} + {hasStatusSection && (
- + View Status - {matViewData.view_status === "valid" ? ( - <> - - Valid - - ) : matViewData.view_status === "refreshing" ? ( - Refreshing + {matViewUnavailable || liveViewUnavailable ? ( + + ) : matView ? ( + matView.view_status === "valid" ? ( + <> + + Valid + + ) : matView.view_status === "refreshing" ? ( + Refreshing + ) : ( + <> + + Invalid + + ) + ) : liveView ? ( + liveView.view_status === "active" ? ( + <> + + Active + + ) : isLiveViewFailureStatus(liveView.view_status) ? ( + <> + + + {LIVE_VIEW_FAILURE_STATUS_LABELS[liveView.view_status]} + + + ) : ( + + {liveView.view_status.charAt(0).toUpperCase() + + liveView.view_status.slice(1)} + + ) ) : ( - <> - - Invalid - + Loadingโ€ฆ )} @@ -527,7 +652,9 @@ export const MonitoringTab = ({ Base Table Status - {baseTableStatus === "Valid" && ( + {!baseTableName || baseTableStatus === null ? ( + Unknown + ) : baseTableStatus === "Valid" ? ( <> Valid - )} - {(baseTableStatus === "Suspended" || - baseTableStatus === "Dropped") && ( + ) : ( <> )} + {(liveView || liveViewUnavailable) && ( + <> +
+ + + Freshness + + + + ) : ( + formatTxnCount(liveView?.lag_seqtxn ?? null) + ) + } + boxedValue + fullWidth + /> + + ) : liveView?.lag_micros == null ? ( + "Never" + ) : ( + formatMicrosDuration(liveView.lag_micros) + ) + } + /> + + ) : liveView?.writer_stall_micros == null ? ( + "Unknown" + ) : ( + formatMicrosDuration(liveView.writer_stall_micros) + ) + } + issue={healthStatus?.fieldIssues.get("writerStall")} + /> + +
+ +
+ + + In-Memory Tier + + + + ) : ( + formatRowCount(liveView?.in_mem_rows ?? null) + ) + } + /> + + ) : ( + formatBytes(liveView?.in_mem_bytes ?? null) + ) + } + /> + {(hasLiveViewDroppedRows || liveViewDiagnosticsUnavailable) && ( + + ) : ( + `${formatRowCount(liveView?.below_lower_bound_count ?? null)} in-order ยท ${formatRowCount(liveView?.o3_rejected_count ?? null)} out-of-order` + ) + } + fullWidth + /> + )} + +
+ + )} +
{tableData.walEnabled && ( <> @@ -578,7 +802,10 @@ export const MonitoringTab = ({ {walExpanded && ( - + BIGINT_ZERO ? rawLag : BIGINT_ZERO + return formatTxnCount(lag) })()} issue={healthStatus?.fieldIssues.get("transactionLag")} showTrend @@ -662,7 +889,7 @@ export const MonitoringTab = ({ } issue={healthStatus?.fieldIssues.get("mergeRate")} /> - + )} diff --git a/src/scenes/Schema/TableDetailsDrawer/SchemaAIButton.tsx b/src/scenes/Schema/TableDetailsDrawer/SchemaAIButton.tsx index 39e398453..737f4c653 100644 --- a/src/scenes/Schema/TableDetailsDrawer/SchemaAIButton.tsx +++ b/src/scenes/Schema/TableDetailsDrawer/SchemaAIButton.tsx @@ -12,26 +12,37 @@ const AIButtonStyled = styled(Button).attrs({ prefixIcon: , })`` +type SchemaAIButtonProps = ButtonProps & + ( + | { disabled: boolean; disabledTooltip: string } + | { disabled?: undefined; disabledTooltip?: undefined } + ) + export const SchemaAIButton = ({ onClick, children, + disabled, + disabledTooltip, ...props -}: ButtonProps) => { +}: SchemaAIButtonProps) => { const { hasSchemaAccess, canUse, status } = useAIStatus() const isOperationInProgress = isBlockingAIStatus(status) + const aiDisabled = !canUse || !hasSchemaAccess || isOperationInProgress + const aiDisabledTooltip = !canUse + ? "AI Assistant is not configured" + : !hasSchemaAccess + ? "Schema access is not granted to this model" + : isOperationInProgress + ? "An operation is in progress" + : undefined + return ( {children} diff --git a/src/scenes/Schema/TableDetailsDrawer/healthCheck.test.ts b/src/scenes/Schema/TableDetailsDrawer/healthCheck.test.ts index 6c1b0a5fa..9c9ee8142 100644 --- a/src/scenes/Schema/TableDetailsDrawer/healthCheck.test.ts +++ b/src/scenes/Schema/TableDetailsDrawer/healthCheck.test.ts @@ -1,10 +1,52 @@ import { describe, it, expect } from "vitest" import { + calculateHealthStatus, calculateTrendRate, getTrendDirection, detectIngestionActive, + isLiveViewLoadFailure, type TimestampedSample, + type TrendData, } from "./healthCheck" +import type { + LiveView, + MaterializedView, + Table, +} from "../../../utils/questdb/types" +import type { TableKindData } from "./types" + +const ready = (data: T) => ({ status: "ready" as const, data }) +const loading = { status: "loading" } as const + +describe("isLiveViewLoadFailure", () => { + it.each(["version_unsupported", "state_unreadable"] as const)( + "should detect the %s load-failure state", + (viewStatus) => { + // Given + const liveView = { view_status: viewStatus } as LiveView + + // When + const result = isLiveViewLoadFailure(liveView) + + // Then + expect(result).toBe(true) + }, + ) + + it.each(["active", "invalid"] as const)( + "should keep diagnostics available for the %s state", + (viewStatus) => { + // Given + const liveView = { view_status: viewStatus } as LiveView + + // When + const result = isLiveViewLoadFailure(liveView) + + // Then + expect(result).toBe(false) + }, + ) +}) const makeSamples = ( values: number[], @@ -12,7 +54,7 @@ const makeSamples = ( startTime: number = 0, ): TimestampedSample[] => { return values.map((value, i) => ({ - value, + value: BigInt(value), timestamp: startTime + i * intervalMs, })) } @@ -20,7 +62,9 @@ const makeSamples = ( describe("calculateTrendRate", () => { it("should return 0 for less than 2 samples", () => { expect(calculateTrendRate([], 0)).toBe(0) - expect(calculateTrendRate([{ value: 100, timestamp: 0 }], 0)).toBe(0) + expect(calculateTrendRate([{ value: BigInt(100), timestamp: 0 }], 0)).toBe( + 0, + ) }) it("should calculate positive slope for increasing values", () => { @@ -40,14 +84,24 @@ describe("calculateTrendRate", () => { expect(calculateTrendRate(samples, 3000)).toBe(0) }) + it("should preserve deltas between unsafe LONG values", () => { + const base = BigInt("9007199254740992") + const samples: TimestampedSample[] = [0, 1, 2, 3, 4].map((i) => ({ + value: base + BigInt(i), + timestamp: i * 1000, + })) + + expect(calculateTrendRate(samples, 4000)).toBeCloseTo(1) + }) + it("should handle noisy data and find overall trend", () => { // [100, 150, 120, 180, 150] over 8 seconds - overall increasing const samples = [ - { value: 100, timestamp: 0 }, - { value: 150, timestamp: 2000 }, - { value: 120, timestamp: 4000 }, - { value: 180, timestamp: 6000 }, - { value: 150, timestamp: 8000 }, + { value: BigInt(100), timestamp: 0 }, + { value: BigInt(150), timestamp: 2000 }, + { value: BigInt(120), timestamp: 4000 }, + { value: BigInt(180), timestamp: 6000 }, + { value: BigInt(150), timestamp: 8000 }, ] const rate = calculateTrendRate(samples, 8000) // Should detect overall positive trend @@ -57,11 +111,11 @@ describe("calculateTrendRate", () => { it("should detect recovery after spike", () => { // [0, 100, 200, 100, 5] - spike then recovery const samples = [ - { value: 0, timestamp: 0 }, - { value: 100, timestamp: 2000 }, - { value: 200, timestamp: 4000 }, - { value: 100, timestamp: 6000 }, - { value: 5, timestamp: 8000 }, + { value: BigInt(0), timestamp: 0 }, + { value: BigInt(100), timestamp: 2000 }, + { value: BigInt(200), timestamp: 4000 }, + { value: BigInt(100), timestamp: 6000 }, + { value: BigInt(5), timestamp: 8000 }, ] const rate = calculateTrendRate(samples, 8000) // Linear regression gives small positive slope (~0.625) due to math @@ -72,11 +126,11 @@ describe("calculateTrendRate", () => { it("should only consider samples within 30-second window", () => { const now = 60000 // 60 seconds const samples = [ - { value: 1000, timestamp: 0 }, // 60s ago - should be excluded - { value: 900, timestamp: 10000 }, // 50s ago - should be excluded - { value: 100, timestamp: 35000 }, // 25s ago - included - { value: 200, timestamp: 45000 }, // 15s ago - included - { value: 300, timestamp: 55000 }, // 5s ago - included + { value: BigInt(1000), timestamp: 0 }, // 60s ago - should be excluded + { value: BigInt(900), timestamp: 10000 }, // 50s ago - should be excluded + { value: BigInt(100), timestamp: 35000 }, // 25s ago - included + { value: BigInt(200), timestamp: 45000 }, // 15s ago - included + { value: BigInt(300), timestamp: 55000 }, // 5s ago - included ] const rate = calculateTrendRate(samples, now) // Only last 3 samples within 30s window: 100 -> 200 -> 300 over 20s = 10/s @@ -86,8 +140,8 @@ describe("calculateTrendRate", () => { it("should handle single sample within window", () => { const now = 60000 const samples = [ - { value: 1000, timestamp: 0 }, // 60s ago - excluded - { value: 500, timestamp: 50000 }, // 10s ago - only sample in window + { value: BigInt(1000), timestamp: 0 }, // 60s ago - excluded + { value: BigInt(500), timestamp: 50000 }, // 10s ago - only sample in window ] // Only one sample in window, need 2+ for regression expect(calculateTrendRate(samples, now)).toBe(0) @@ -119,26 +173,59 @@ describe("getTrendDirection", () => { }) }) +describe("health issue prompt values", () => { + it("keeps WAL counter values locale-independent for AI prompts", () => { + const now = Date.now() + const table = { + walEnabled: true, + table_suspended: false, + table_memory_pressure_level: 0, + } as Table + const trendData: TrendData = { + transactionLag: [ + { value: BigInt(1_000), timestamp: now - 1_000 }, + { value: BigInt(1_500), timestamp: now }, + ], + walPendingRowCount: [ + { value: BigInt(2_000), timestamp: now - 1_000 }, + { value: BigInt(2_500), timestamp: now }, + ], + ingestionMetric: [], + } + + const status = calculateHealthStatus(table, { kind: "table" }, trendData) + + expect(status.issues.find((issue) => issue.id === "Y1")?.promptValue).toBe( + "1500 txns", + ) + expect(status.issues.find((issue) => issue.id === "Y2")?.promptValue).toBe( + "2500 rows", + ) + }) +}) + describe("detectIngestionActive", () => { it("should return false when less than 2 samples", () => { expect(detectIngestionActive([])).toBe(false) - expect(detectIngestionActive([{ value: 100, timestamp: 0 }])).toBe(false) + expect(detectIngestionActive([{ value: BigInt(100), timestamp: 0 }])).toBe( + false, + ) }) it("should return true when any increase detected in last 5 samples", () => { expect( detectIngestionActive([ - { value: 100, timestamp: 0 }, - { value: 100, timestamp: 1000 }, - { value: 101, timestamp: 2000 }, + { value: BigInt(100), timestamp: 0 }, + { value: BigInt(100), timestamp: 1000 }, + { value: BigInt(101), timestamp: 2000 }, ]), ).toBe(true) expect( detectIngestionActive([ - { value: 100, timestamp: 0 }, - { value: 101, timestamp: 1000 }, - { value: 100, timestamp: 2000 }, + { value: BigInt(100), timestamp: 0 }, + { value: BigInt(101), timestamp: 1000 }, + { value: BigInt(100), timestamp: 2000 }, ]), ).toBe(true) }) @@ -146,17 +233,17 @@ describe("detectIngestionActive", () => { it("should return false when no increase in last 5 samples", () => { expect( detectIngestionActive([ - { value: 100, timestamp: 0 }, - { value: 100, timestamp: 1000 }, - { value: 100, timestamp: 2000 }, + { value: BigInt(100), timestamp: 0 }, + { value: BigInt(100), timestamp: 1000 }, + { value: BigInt(100), timestamp: 2000 }, ]), ).toBe(false) expect( detectIngestionActive([ - { value: 100, timestamp: 0 }, - { value: 99, timestamp: 1000 }, - { value: 98, timestamp: 2000 }, + { value: BigInt(100), timestamp: 0 }, + { value: BigInt(99), timestamp: 1000 }, + { value: BigInt(98), timestamp: 2000 }, ]), ).toBe(false) }) @@ -164,14 +251,14 @@ describe("detectIngestionActive", () => { it("should only use last 5 samples even if more are available", () => { // First 3 samples have increases, but last 5 don't const samples = [ - { value: 100, timestamp: 0 }, - { value: 101, timestamp: 1000 }, - { value: 102, timestamp: 2000 }, - { value: 100, timestamp: 3000 }, - { value: 100, timestamp: 4000 }, - { value: 100, timestamp: 5000 }, - { value: 100, timestamp: 6000 }, - { value: 100, timestamp: 7000 }, + { value: BigInt(100), timestamp: 0 }, + { value: BigInt(101), timestamp: 1000 }, + { value: BigInt(102), timestamp: 2000 }, + { value: BigInt(100), timestamp: 3000 }, + { value: BigInt(100), timestamp: 4000 }, + { value: BigInt(100), timestamp: 5000 }, + { value: BigInt(100), timestamp: 6000 }, + { value: BigInt(100), timestamp: 7000 }, ] // Last 5: [100, 100, 100, 100, 100] - no increase expect(detectIngestionActive(samples)).toBe(false) @@ -179,15 +266,312 @@ describe("detectIngestionActive", () => { it("should detect increase in last 5 samples of longer array", () => { const samples = [ - { value: 100, timestamp: 0 }, - { value: 100, timestamp: 1000 }, - { value: 100, timestamp: 2000 }, - { value: 100, timestamp: 3000 }, - { value: 100, timestamp: 4000 }, - { value: 100, timestamp: 5000 }, - { value: 100, timestamp: 6000 }, - { value: 101, timestamp: 7000 }, // increase in last 5 + { value: BigInt(100), timestamp: 0 }, + { value: BigInt(100), timestamp: 1000 }, + { value: BigInt(100), timestamp: 2000 }, + { value: BigInt(100), timestamp: 3000 }, + { value: BigInt(100), timestamp: 4000 }, + { value: BigInt(100), timestamp: 5000 }, + { value: BigInt(100), timestamp: 6000 }, + { value: BigInt(101), timestamp: 7000 }, // increase in last 5 ] expect(detectIngestionActive(samples)).toBe(true) }) }) + +describe("write amplification health threshold", () => { + const emptyTrend: TrendData = { + walPendingRowCount: [], + transactionLag: [], + ingestionMetric: [], + } + const objectKinds: Array<{ name: string; kindData: TableKindData }> = [ + { name: "table", kindData: { kind: "table" } }, + { + name: "materialized view", + kindData: { kind: "matview", matView: loading }, + }, + { name: "live view", kindData: { kind: "liveview", liveView: loading } }, + ] + + for (const { name, kindData } of objectKinds) { + it(`should warn for a ${name} starting at 3x`, () => { + const table = { + walEnabled: true, + table_suspended: false, + table_memory_pressure_level: 0, + table_write_amp_p50: 3, + } as Table + + const status = calculateHealthStatus(table, kindData, emptyTrend) + + expect(status.issues.find((issue) => issue.id === "Y4")).toMatchObject({ + severity: "warning", + currentValue: "3.00x", + }) + }) + + it(`should not warn for a ${name} below 3x`, () => { + const table = { + walEnabled: true, + table_suspended: false, + table_memory_pressure_level: 0, + table_write_amp_p50: 2.99, + } as Table + + const status = calculateHealthStatus(table, kindData, emptyTrend) + + expect(status.issues.find((issue) => issue.id === "Y4")).toBeUndefined() + }) + } +}) + +describe("calculateHealthStatus for live views", () => { + const makeTable = (): Table => + ({ + table_name: "trades_ma", + walEnabled: true, + table_suspended: false, + table_memory_pressure_level: 0, + }) as Table + + const makeLiveView = (overrides: Partial): LiveView => + ({ + view_name: "trades_ma", + view_status: "active", + invalidation_reason: null, + lag_seqtxn: BigInt(0), + writer_stall_micros: BigInt(0), + flush_every_interval: BigInt(30), + flush_every_interval_unit: "SECOND", + ...overrides, + }) as LiveView + + const emptyTrend: TrendData = { + walPendingRowCount: [], + transactionLag: [], + ingestionMetric: [], + } + + it("should report no live view issues for a healthy active view", () => { + // Given + const liveView = makeLiveView({}) + + // When + const status = calculateHealthStatus( + makeTable(), + { kind: "liveview", liveView: ready(liveView) }, + emptyTrend, + ) + + // Then + expect(status.overallSeverity).toBe("healthy") + expect(status.issues).toEqual([]) + }) + + it("should not raise an issue or a trend for a large live view lag", () => { + // Given a live view far behind its base table + const liveView = makeLiveView({ lag_seqtxn: BigInt(10_000) }) + + // When its health is calculated + const status = calculateHealthStatus( + makeTable(), + { kind: "liveview", liveView: ready(liveView) }, + emptyTrend, + ) + + // Then lag contributes nothing: it is a flush-cadence sawtooth, not a signal + expect([...status.fieldIssues.keys()]).toEqual([]) + expect([...status.trendIndicators.keys()]).toEqual([]) + expect(status.issues).toEqual([]) + }) + + it("should report a critical issue when the live view is invalid", () => { + // Given + const liveView = makeLiveView({ + view_status: "invalid", + invalidation_reason: "base table column dropped", + }) + + // When + const status = calculateHealthStatus( + makeTable(), + { kind: "liveview", liveView: ready(liveView) }, + emptyTrend, + ) + + // Then + expect(status.overallSeverity).toBe("critical") + expect(status.fieldIssues.get("viewStatus")).toMatchObject({ + id: "R5", + severity: "critical", + message: "Live view is invalid: base table column dropped", + }) + }) + + it("should omit the reason when the live view is invalid without one", () => { + // Given + const liveView = makeLiveView({ + view_status: "invalid", + invalidation_reason: null, + }) + + // When + const status = calculateHealthStatus( + makeTable(), + { kind: "liveview", liveView: ready(liveView) }, + emptyTrend, + ) + + // Then + expect(status.fieldIssues.get("viewStatus")?.message).toBe( + "Live view is invalid", + ) + }) + + it("should warn about a stalled writer regardless of flush interval", () => { + // Given a fast flush interval and a stalled writer + const liveView = makeLiveView({ + writer_stall_micros: BigInt(6_000_000), + flush_every_interval: BigInt(1), + flush_every_interval_unit: "SECOND", + }) + + // When + const status = calculateHealthStatus( + makeTable(), + { kind: "liveview", liveView: ready(liveView) }, + emptyTrend, + ) + + // Then writer-stall detection remains independent of lag + expect(status.fieldIssues.get("writerStall")).toMatchObject({ id: "Y7" }) + }) + + it("should warn when the flush writer stalls beyond the threshold", () => { + // Given + const liveView = makeLiveView({ writer_stall_micros: BigInt(6_000_000) }) + + // When + const status = calculateHealthStatus( + makeTable(), + { kind: "liveview", liveView: ready(liveView) }, + emptyTrend, + ) + + // Then + expect(status.fieldIssues.get("writerStall")).toMatchObject({ + id: "Y7", + severity: "warning", + currentValue: "6.0 s", + }) + }) + + it("should report a critical issue when the live view format version is unsupported", () => { + // Given + const liveView = makeLiveView({ view_status: "version_unsupported" }) + + // When + const status = calculateHealthStatus( + makeTable(), + { kind: "liveview", liveView: ready(liveView) }, + emptyTrend, + ) + + // Then + expect(status.overallSeverity).toBe("critical") + expect(status.fieldIssues.get("viewStatus")).toMatchObject({ + id: "R6", + severity: "critical", + message: "Live view format is not readable by this server build", + }) + }) + + it("should report a critical issue when the live view state is unreadable", () => { + // Given + const liveView = makeLiveView({ view_status: "state_unreadable" }) + + // When + const status = calculateHealthStatus( + makeTable(), + { kind: "liveview", liveView: ready(liveView) }, + emptyTrend, + ) + + // Then + expect(status.overallSeverity).toBe("critical") + expect(status.fieldIssues.get("viewStatus")).toMatchObject({ + id: "R7", + severity: "critical", + message: "Live view state files are unreadable", + }) + }) + + it("should report no live view issues while the metadata is still loading", () => { + // Given a live view target whose metadata has not arrived yet + + // When + const status = calculateHealthStatus( + makeTable(), + { kind: "liveview", liveView: loading }, + emptyTrend, + ) + + // Then + expect(status.issues).toEqual([]) + }) + + it("should report unknown when live view metadata is unavailable", () => { + // Given + const table = makeTable() + + // When + const status = calculateHealthStatus( + table, + { kind: "liveview", liveView: { status: "unavailable" } }, + emptyTrend, + ) + + // Then + expect(status.overallSeverity).toBe("unknown") + expect(status.hasUnavailableSource).toBe(true) + expect(status.issues).toEqual([]) + }) + + it("should keep a known critical issue above unavailable metadata", () => { + // Given + const table = { ...makeTable(), table_suspended: true } + + // When + const status = calculateHealthStatus( + table, + { kind: "liveview", liveView: { status: "unavailable" } }, + emptyTrend, + ) + + // Then + expect(status.overallSeverity).toBe("critical") + expect(status.fieldIssues.get("walStatus")?.id).toBe("R1") + }) + + it("should omit the reason when the matview is invalid without one", () => { + // Given + const matView = { + view_name: "trades_ma", + view_status: "invalid", + invalidation_reason: null, + } as MaterializedView + + // When + const status = calculateHealthStatus( + makeTable(), + { kind: "matview", matView: ready(matView) }, + emptyTrend, + ) + + // Then + expect(status.fieldIssues.get("viewStatus")?.message).toBe( + "Materialized view is invalid", + ) + }) +}) diff --git a/src/scenes/Schema/TableDetailsDrawer/healthCheck.ts b/src/scenes/Schema/TableDetailsDrawer/healthCheck.ts index a79a21626..8f499e5cc 100644 --- a/src/scenes/Schema/TableDetailsDrawer/healthCheck.ts +++ b/src/scenes/Schema/TableDetailsDrawer/healthCheck.ts @@ -1,21 +1,42 @@ -import type { Table, MaterializedView } from "../../../utils/questdb/types" +import { type Table, type LiveView } from "../../../utils/questdb/types" +import type { TableKindData } from "./types" +import { formatMicrosDuration } from "./utils" const DOCS_BASE_URL = "https://questdb.com/docs" const MONITORING_DOCS_URL = `${DOCS_BASE_URL}/operations/monitoring-alerting` +const LIVE_VIEWS_MONITORING_DOCS_URL = `${DOCS_BASE_URL}/concepts/live-views/#monitoring` export const ISSUE_DOCS_URLS: Record = { R1: `${MONITORING_DOCS_URL}/#detect-suspended-tables`, // WAL suspended R2: `${MONITORING_DOCS_URL}/#detect-invalid-materialized-views`, // MatView invalid R3: `${MONITORING_DOCS_URL}/#detect-memory-pressure`, // Memory backoff (level 2) R4: `${DOCS_BASE_URL}/concepts/views/#view-invalidation`, // View invalid + R5: LIVE_VIEWS_MONITORING_DOCS_URL, // Live view invalid + R6: LIVE_VIEWS_MONITORING_DOCS_URL, // Live view format version unsupported + R7: LIVE_VIEWS_MONITORING_DOCS_URL, // Live view state unreadable Y1: `${MONITORING_DOCS_URL}/#detect-transaction-lag-and-pending-rows`, // Transaction lag increasing Y2: `${MONITORING_DOCS_URL}/#detect-transaction-lag-and-pending-rows`, // Pending rows increasing Y3: `${MONITORING_DOCS_URL}/#detect-small-transactions`, // Small transactions Y4: `${MONITORING_DOCS_URL}/#detect-high-write-amplification`, // High write amplification Y5: `${MONITORING_DOCS_URL}/#detect-memory-pressure`, // Reduced parallelism (level 1) + Y7: LIVE_VIEWS_MONITORING_DOCS_URL, // Live view flush writer stalled } -export type HealthSeverity = "critical" | "warning" | "healthy" | "recovering" +const LIVE_VIEW_ISSUE_GUIDANCE: Record = { + R5: "Invalidation is permanent. Save the definition with SHOW CREATE LIVE VIEW, then drop and recreate the view. RESUME WAL does not recover an invalid live view.", + R6: "The on-disk format is not readable by this server build, usually after a binary downgrade. Restore the newer binary and restart. If the binary is correct, the file header is damaged: drop and recreate the view. The view does not refresh until this is resolved.", + R7: "The view state files are corrupt or missing, and automatic recovery failed. A restart does not fix this. Save the definition with SHOW CREATE LIVE VIEW, then drop and recreate the view. Existing rows stay queryable but frozen.", +} + +export const getLiveViewIssueGuidance = (issueId: string): string | undefined => + (LIVE_VIEW_ISSUE_GUIDANCE as Partial>)[issueId] + +export type HealthSeverity = + | "critical" + | "warning" + | "unknown" + | "healthy" + | "recovering" export type TrendDirection = "increasing" | "decreasing" | "stable" @@ -25,6 +46,7 @@ export type HealthIssue = { field: string message: string currentValue?: string + promptValue?: string } export type TrendIndicator = { @@ -36,13 +58,14 @@ export type TrendIndicator = { export type HealthStatus = { overallSeverity: HealthSeverity + hasUnavailableSource: boolean issues: HealthIssue[] fieldIssues: Map trendIndicators: Map } export type TimestampedSample = { - value: number + value: bigint timestamp: number } @@ -52,13 +75,19 @@ export type TrendData = { ingestionMetric: TimestampedSample[] } +// The drawer polls live_views() on this period for point-in-time status and +// metrics. lag_seqtxn is deliberately not trended: it is a flush-cadence +// sawtooth, and a drawer session is usually too short to observe enough flush +// cycles across QuestDB's full supported interval range. +export const LIVE_VIEW_POLL_MS = 1_000 + const TREND_WINDOW_MS = 30_000 export const MAX_TREND_SAMPLES = 150 const RATE_THRESHOLD = 0.5 function getRecentSamples( samples: TimestampedSample[], - now: number = Date.now(), + now: number, ): TimestampedSample[] { const cutoff = now - TREND_WINDOW_MS return samples.filter((s) => s.timestamp >= cutoff) @@ -72,9 +101,12 @@ export function calculateTrendRate( if (recent.length < 2) return 0 const first = recent[0].timestamp + const firstValue = recent[0].value const points = recent.map((s) => ({ t: (s.timestamp - first) / 1000, - v: s.value, + // Trend rates are intentionally floating point. Source counters and their + // subtraction stay exact until this derived value is calculated. + v: Number(s.value - firstValue), })) const n = points.length @@ -107,12 +139,64 @@ export function detectIngestionActive(samples: TimestampedSample[]): boolean { return false } +const BIGINT_ZERO = BigInt(0) +const BIGINT_ONE_HUNDRED = BigInt(100) +const LIVE_VIEW_WRITER_STALL_WARNING_MICROS = BigInt(5_000_000) +const HIGH_WRITE_AMPLIFICATION_THRESHOLD = 3 + +export type LiveViewFailure = { + issueId: "R5" | "R6" | "R7" + message: string +} + +export const getLiveViewFailure = ( + liveView: LiveView, +): LiveViewFailure | null => { + switch (liveView.view_status) { + case "invalid": + return { + issueId: "R5", + message: liveView.invalidation_reason + ? `Live view is invalid: ${liveView.invalidation_reason}` + : "Live view is invalid", + } + case "version_unsupported": + return { + issueId: "R6", + message: "Live view format is not readable by this server build", + } + case "state_unreadable": + return { + issueId: "R7", + message: "Live view state files are unreadable", + } + default: + return null + } +} + +export const isLiveViewLoadFailure = (liveView: LiveView | null): boolean => + liveView?.view_status === "version_unsupported" || + liveView?.view_status === "state_unreadable" + export function calculateHealthStatus( tableData: Table, - matViewData: MaterializedView | null, + kindData: TableKindData, trendData: TrendData, - isMatView: boolean, ): HealthStatus { + const matViewData = + kindData.kind === "matview" && kindData.matView.status === "ready" + ? kindData.matView.data + : null + const liveViewData = + kindData.kind === "liveview" && kindData.liveView.status === "ready" + ? kindData.liveView.data + : null + const kindSourceUnavailable = + (kindData.kind === "view" && kindData.view.status === "unavailable") || + (kindData.kind === "matview" && + kindData.matView.status === "unavailable") || + (kindData.kind === "liveview" && kindData.liveView.status === "unavailable") const issues: HealthIssue[] = [] // ============================================================ @@ -130,12 +214,14 @@ export function calculateHealthStatus( } // R2: MatView Invalid (affects header dot only, UI has dedicated section) - if (isMatView && matViewData?.view_status === "invalid") { + if (matViewData?.view_status === "invalid") { issues.push({ id: "R2", severity: "critical", field: "viewStatus", - message: `Materialized view is invalid: ${matViewData.invalidation_reason}`, + message: matViewData.invalidation_reason + ? `Materialized view is invalid: ${matViewData.invalidation_reason}` + : "Materialized view is invalid", }) } @@ -149,6 +235,18 @@ export function calculateHealthStatus( }) } + // R5: invalid; R6/R7: failed to load at boot (stub states that never + // refresh again). Shared with the schema tree via getLiveViewFailure. + const liveViewFailure = liveViewData ? getLiveViewFailure(liveViewData) : null + if (liveViewFailure) { + issues.push({ + id: liveViewFailure.issueId, + severity: "critical", + field: "viewStatus", + message: liveViewFailure.message, + }) + } + // ============================================================ // YELLOW (Warning) - Needs attention // ============================================================ @@ -164,12 +262,13 @@ export function calculateHealthStatus( const pendingDirection = getTrendDirection(pendingRate) const currentLag = - trendData.transactionLag[trendData.transactionLag.length - 1]?.value ?? 0 + trendData.transactionLag[trendData.transactionLag.length - 1]?.value ?? + BIGINT_ZERO const currentPending = trendData.walPendingRowCount[trendData.walPendingRowCount.length - 1] - ?.value ?? 0 + ?.value ?? BIGINT_ZERO - if (currentLag > 0 && txLagDirection !== "stable") { + if (currentLag > BIGINT_ZERO && txLagDirection !== "stable") { trendIndicators.set("transactionLag", { field: "transactionLag", direction: txLagDirection, @@ -187,12 +286,13 @@ export function calculateHealthStatus( severity: "warning", field: "transactionLag", message: "Transaction lag increasing", - currentValue: `${currentLag} txns`, + currentValue: `${currentLag.toLocaleString()} txns`, + promptValue: `${currentLag.toString()} txns`, }) } } - if (currentPending > 0 && pendingDirection !== "stable") { + if (currentPending > BIGINT_ZERO && pendingDirection !== "stable") { trendIndicators.set("pendingRows", { field: "pendingRows", direction: pendingDirection, @@ -211,6 +311,7 @@ export function calculateHealthStatus( field: "pendingRows", message: "Pending rows accumulating", currentValue: `${currentPending.toLocaleString()} rows`, + promptValue: `${currentPending.toString()} rows`, }) } } @@ -218,22 +319,22 @@ export function calculateHealthStatus( // Y3: Small Transactions (p90 < 100 rows, but > 0 to exclude empty tables) if ( tableData.wal_tx_size_p90 != null && - tableData.wal_tx_size_p90 > 0 && - tableData.wal_tx_size_p90 < 100 + tableData.wal_tx_size_p90 > BIGINT_ZERO && + tableData.wal_tx_size_p90 < BIGINT_ONE_HUNDRED ) { issues.push({ id: "Y3", severity: "warning", field: "txSizeP90", message: "Small transactions - consider batching", - currentValue: `${tableData.wal_tx_size_p90} rows`, + currentValue: `${tableData.wal_tx_size_p90.toLocaleString()} rows`, }) } - // Y4: High Write Amplification (p50 > 2.0 means significant O3 merge overhead) + // Y4: High Write Amplification (p50 >= 3.0 means significant O3 merge overhead) if ( tableData.table_write_amp_p50 != null && - tableData.table_write_amp_p50 > 2.0 + tableData.table_write_amp_p50 >= HIGH_WRITE_AMPLIFICATION_THRESHOLD ) { issues.push({ id: "Y4", @@ -256,13 +357,30 @@ export function calculateHealthStatus( } } + if (liveViewData) { + // Y7: Flush writer stalled + if ( + liveViewData.writer_stall_micros != null && + liveViewData.writer_stall_micros > LIVE_VIEW_WRITER_STALL_WARNING_MICROS + ) { + issues.push({ + id: "Y7", + severity: "warning", + field: "writerStall", + message: "Live view flush writer stalled", + currentValue: formatMicrosDuration(liveViewData.writer_stall_micros), + }) + } + } + // Build field -> issue map (highest severity wins per field) const fieldIssues = new Map() const severityOrder: Record = { critical: 0, warning: 1, - recovering: 2, - healthy: 3, + unknown: 2, + recovering: 3, + healthy: 4, } for (const issue of issues) { @@ -280,9 +398,17 @@ export function calculateHealthStatus( overallSeverity = "critical" } else if (issues.some((i) => i.severity === "warning")) { overallSeverity = "warning" + } else if (kindSourceUnavailable) { + overallSeverity = "unknown" } else if (issues.some((i) => i.severity === "recovering")) { overallSeverity = "recovering" } - return { overallSeverity, issues, fieldIssues, trendIndicators } + return { + overallSeverity, + hasUnavailableSource: kindSourceUnavailable, + issues, + fieldIssues, + trendIndicators, + } } diff --git a/src/scenes/Schema/TableDetailsDrawer/index.tsx b/src/scenes/Schema/TableDetailsDrawer/index.tsx index 6a24821bf..d53a77f37 100644 --- a/src/scenes/Schema/TableDetailsDrawer/index.tsx +++ b/src/scenes/Schema/TableDetailsDrawer/index.tsx @@ -4,6 +4,7 @@ import React, { useState, useCallback, useMemo, + useRef, } from "react" import { useSelector, useDispatch } from "react-redux" import styled from "styled-components" @@ -29,24 +30,35 @@ import { QuestContext, useSettings } from "../../../providers" import * as QuestDB from "../../../utils/questdb" import { getTableKind, + getTableKindLabel, type Table, + type TableKind, type Column, + type LiveView, type MaterializedView, + type StoragePolicy, type View, } from "../../../utils/questdb/types" import { calculateHealthStatus, detectIngestionActive, + getLiveViewIssueGuidance, + LIVE_VIEW_POLL_MS, MAX_TREND_SAMPLES, type TrendData, type HealthIssue, + type HealthSeverity, } from "./healthCheck" +import { getTrendSamplesForIssue } from "./utils" import { HealthStatusLabel } from "./HealthStatusLabel" import { useDebouncedWarnings } from "./useDebouncedWarnings" +import { useCatalogSource } from "./useCatalogSource" import { SuspensionDialog } from "../SuspensionDialog" import { useAdaptivePoll, useAIQuickActions } from "../../../hooks" import { MonitoringTab } from "./MonitoringTab" import { DetailsTab } from "./DetailsTab" +import { ErrorBanner } from "./ErrorBanner" +import type { TableKindData } from "./types" import { trackEvent } from "../../../modules/ConsoleEventTracker" import { ConsoleEvent } from "../../../modules/ConsoleEventTracker/events" @@ -54,6 +66,8 @@ const TypeBadge = styled(Badge).attrs({ variant: "neutral", size: "sm" })` flex-shrink: 0; ` +const BIGINT_ZERO = BigInt(0) + const LoadingContainer = styled(Box).attrs({ align: "center", justifyContent: "center", @@ -92,6 +106,10 @@ const EmptyStateSubheading = styled.p` line-height: 1.5; ` +const MetadataErrorBannerWrapper = styled.div` + padding: 1.5rem; +` + const TitleContainer = styled(Dialog.Title).attrs({})` display: flex; padding: 0; @@ -111,6 +129,75 @@ const CopyButtonSlot = styled.span` type TabType = "monitoring" | "details" +const TABLE_POLL_MIN_MS = 200 +const TABLE_POLL_MAX_MS = 5_000 +const DETAILS_TABLE_POLL_MS = 1_000 +const KIND_POLL_MS = 1_000 +// A storage policy only changes when an operator runs DDL against the table, +// so it is polled far less often than the columns and DDL beside it. +const STORAGE_POLICY_POLL_MS = 5_000 + +type TableSourceData = { type: "found"; data: Table } | { type: "missing" } + +const firstCatalogRow = >( + response: QuestDB.QueryRawResult, +): T | undefined => { + const result = QuestDB.Client.transformQueryRawResult(response, { + convertLongsToBigInt: true, + }) + return result.type === QuestDB.Type.DQL ? result.data[0] : undefined +} + +const firstRow = >( + response: QuestDB.QueryRawResult, +): T | undefined => { + const result = QuestDB.Client.transformQueryRawResult(response) + return result.type === QuestDB.Type.DQL ? result.data[0] : undefined +} + +const transformTableResponse = ( + response: QuestDB.QueryRawResult, +): TableSourceData | undefined => { + const result = QuestDB.Client.transformQueryRawResult(response, { + convertLongsToBigInt: true, + }) + if (result.type !== QuestDB.Type.DQL) return undefined + return result.data[0] + ? { type: "found", data: result.data[0] } + : { type: "missing" } +} + +const transformMatViewResponse = (response: QuestDB.QueryRawResult) => + firstCatalogRow(response) + +const transformViewResponse = (response: QuestDB.QueryRawResult) => + firstRow(response) + +const transformLiveViewResponse = (response: QuestDB.QueryRawResult) => + firstCatalogRow(response) + +const transformColumnsResponse = ( + response: QuestDB.QueryRawResult, +): Column[] | undefined => { + const result = QuestDB.Client.transformQueryRawResult(response) + return result.type === QuestDB.Type.DQL ? result.data : undefined +} + +const transformDDLResponse = ( + response: QuestDB.QueryRawResult, +): string | undefined => { + const row = firstRow<{ ddl: string }>(response) + return row?.ddl ? row.ddl.replace(/\n{2,}/g, "\n") : undefined +} + +const transformStoragePolicyResponse = ( + response: QuestDB.QueryRawResult, +): StoragePolicy | null | undefined => { + const result = QuestDB.Client.transformQueryRawResult(response) + if (result.type !== QuestDB.Type.DQL) return undefined + return result.data[0] ?? null +} + const TabsContainer = styled.div` display: flex; flex-direction: column; @@ -149,10 +236,14 @@ export const TableDetailsDrawer = () => { const dispatch = useDispatch() const activeSidebar = useSelector(selectors.console.getActiveSidebar) const target = useSelector(selectors.console.getTableDetailsTarget) + const targetRef = useRef(target) + const activeSidebarRef = useRef(activeSidebar) const tableName = target?.tableName ?? "" - const isMatView = target?.isMatView ?? false - const isView = target?.isView ?? false + const kind: TableKind = target?.kind ?? "table" + const isMatView = kind === "matview" + const isView = kind === "view" + const isLiveView = kind === "liveview" const hasTarget = target !== null const isOpen = activeSidebar?.type === "tableDetails" @@ -160,7 +251,113 @@ export const TableDetailsDrawer = () => { dispatch(actions.console.closeSidebar()) } + const isCurrentTarget = useCallback( + (candidateTableName: string, candidateKind: TableKind) => { + const currentTarget = targetRef.current + return ( + activeSidebarRef.current?.type === "tableDetails" && + currentTarget?.tableName === candidateTableName && + currentTarget.kind === candidateKind + ) + }, + [], + ) + + const clearIfCurrentTarget = useCallback( + (missingTableName: string, missingKind: TableKind) => { + if (isCurrentTarget(missingTableName, missingKind)) { + dispatch( + actions.console.replaceSidebarHistory({ + type: "tableDetails", + payload: null, + }), + ) + } + }, + [dispatch, isCurrentTarget], + ) + const tables = useSelector(selectors.query.getTables) + const { quest } = useContext(QuestContext) + const { settings } = useSettings() + const isEnterprise = settings["release.type"] === "EE" + const [activeTab, setActiveTab] = useState("monitoring") + + const escapedTableName = QuestDB.escapeSqlLiteral(tableName) + const sourcePrefix = `${kind}:${tableName}` + const tableSource = useCatalogSource({ + sourceKey: `${sourcePrefix}:tables`, + sourceName: "table metadata", + enabled: isOpen && hasTarget, + query: `tables() where table_name = '${escapedTableName}';`, + pollIntervalMs: null, + transformResponse: transformTableResponse, + }) + const matViewSource = useCatalogSource({ + sourceKey: `${sourcePrefix}:materialized-views`, + sourceName: "materialized view metadata", + enabled: isOpen && hasTarget && isMatView, + query: `materialized_views() WHERE view_name = '${escapedTableName}';`, + pollIntervalMs: KIND_POLL_MS, + transformResponse: transformMatViewResponse, + }) + const viewSource = useCatalogSource({ + sourceKey: `${sourcePrefix}:views`, + sourceName: "view metadata", + enabled: isOpen && hasTarget && isView, + query: `views() WHERE view_name = '${escapedTableName}';`, + pollIntervalMs: KIND_POLL_MS, + transformResponse: transformViewResponse, + }) + const liveViewSource = useCatalogSource({ + sourceKey: `${sourcePrefix}:live-views`, + sourceName: "live view metadata", + enabled: isOpen && hasTarget && isLiveView, + query: `live_views() WHERE view_name = '${escapedTableName}'`, + pollIntervalMs: LIVE_VIEW_POLL_MS, + transformResponse: transformLiveViewResponse, + }) + const columnsSource = useCatalogSource({ + sourceKey: `${sourcePrefix}:columns`, + sourceName: "columns", + enabled: isOpen && hasTarget, + query: `SHOW COLUMNS FROM '${escapedTableName}';`, + pollIntervalMs: + isView || activeTab === "details" ? DETAILS_TABLE_POLL_MS : null, + transformResponse: transformColumnsResponse, + }) + const ddlSource = useCatalogSource({ + sourceKey: `${sourcePrefix}:ddl`, + sourceName: "DDL", + enabled: isOpen && hasTarget, + query: QuestDB.buildDDLQuery(tableName, kind), + pollIntervalMs: + isView || activeTab === "details" ? DETAILS_TABLE_POLL_MS : null, + transformResponse: transformDDLResponse, + }) + const currentTableResult = + tableSource.state.status === "ready" + ? tableSource.state.data + : tableSource.lastReadyData + const tableData = + currentTableResult?.type === "found" ? currentTableResult.data : null + const storageDirectoryName = tableData?.directoryName ?? "" + const escapedStorageDirectoryName = + QuestDB.escapeSqlLiteral(storageDirectoryName) + const storagePolicySource = useCatalogSource({ + sourceKey: `${sourcePrefix}:storage-policy:${storageDirectoryName}`, + sourceName: "storage policy", + enabled: + isOpen && + hasTarget && + isEnterprise && + kind === "table" && + activeTab === "details" && + tableData !== null, + query: `storage_policies WHERE table_dir_name = '${escapedStorageDirectoryName}';`, + pollIntervalMs: STORAGE_POLICY_POLL_MS, + transformResponse: transformStoragePolicyResponse, + }) const tableOptions: TableOption[] = useMemo( () => @@ -180,31 +377,17 @@ export const TableDetailsDrawer = () => { dispatch( actions.console.pushSidebarHistory({ type: "tableDetails", - payload: { - tableName: option.label, - isMatView: option.kind === "matview", - isView: option.kind === "view", - }, + payload: { tableName: option.label, kind: option.kind ?? "table" }, }), ) }, [], ) - const { quest } = useContext(QuestContext) - const { settings } = useSettings() - const isEnterprise = settings["release.type"] === "EE" - const [tableData, setTableData] = useState
(null) - const [matViewData, setMatViewData] = useState(null) - const [viewData, setViewData] = useState(null) - const [columns, setColumns] = useState([]) - const [ddl, setDdl] = useState("") - const [loading, setLoading] = useState(true) const [columnsExpanded, setColumnsExpanded] = useState(false) const [walExpanded, setWalExpanded] = useState(true) const [hasAutoExpanded, setHasAutoExpanded] = useState(false) const [suspensionDialogOpen, setSuspensionDialogOpen] = useState(false) - const [activeTab, setActiveTab] = useState("monitoring") const [trendData, setTrendData] = useState({ walPendingRowCount: [], transactionLag: [], @@ -213,59 +396,102 @@ export const TableDetailsDrawer = () => { const [baseTableStatus, setBaseTableStatus] = useState< "Dropped" | "Suspended" | "Valid" | null >(null) + + const matViewData = + matViewSource.state.status === "ready" ? matViewSource.state.data : null + const viewData = + viewSource.state.status === "ready" ? viewSource.state.data : null + const liveViewData = + liveViewSource.state.status === "ready" ? liveViewSource.state.data : null + const columns = + columnsSource.state.status === "ready" ? columnsSource.state.data : [] + const ddl = ddlSource.state.status === "ready" ? ddlSource.state.data : "" + // The kind source carries the invalid/unreadable status, so rendering before + // it answers would show a healthy view that is not. + const kindSourceLoading = + (isView && viewSource.state.status === "loading") || + (isMatView && matViewSource.state.status === "loading") || + (isLiveView && liveViewSource.state.status === "loading") + const loading = + (tableSource.state.status === "loading" && tableData === null) || + kindSourceLoading + const tablesUnavailable = tableSource.state.status === "unavailable" + const kindSourceUnavailable = + (isView && viewSource.state.status === "unavailable") || + (isMatView && matViewSource.state.status === "unavailable") || + (isLiveView && liveViewSource.state.status === "unavailable") const baseTableExists = baseTableStatus === "Valid" || baseTableStatus === "Suspended" + const baseTableName = + matViewData?.base_table_name ?? liveViewData?.base_table_name ?? undefined + + const kindData: TableKindData = useMemo( + () => + kind === "view" + ? { kind, view: viewSource.state } + : kind === "matview" + ? { kind, matView: matViewSource.state } + : kind === "liveview" + ? { kind, liveView: liveViewSource.state } + : { kind: "table" }, + [kind, viewSource.state, matViewSource.state, liveViewSource.state], + ) + const handleNavigateToBaseTable = useCallback(() => { - if (!matViewData?.base_table_name || !baseTableExists) return - const baseTable = tables.find( - (t) => t.table_name === matViewData.base_table_name, - ) - const kind = baseTable ? getTableKind(baseTable) : "table" + if (!baseTableName || !baseTableExists) return + const baseTable = tables.find((t) => t.table_name === baseTableName) dispatch( actions.console.pushSidebarHistory({ type: "tableDetails", payload: { - tableName: matViewData.base_table_name, - isMatView: kind === "matview", - isView: kind === "view", + tableName: baseTableName, + kind: baseTable ? getTableKind(baseTable) : "table", }, }), ) - }, [dispatch, matViewData?.base_table_name, baseTableExists, tables]) + }, [dispatch, baseTableName, baseTableExists, tables]) const { handleExplainSchema, handleAskAIForHealthIssue } = useAIQuickActions() const handleExplainWithAI = useCallback(() => { void trackEvent(ConsoleEvent.TABLE_DETAILS_SCHEMA_EXPLAIN) if (tableData?.id == null) return - void handleExplainSchema( - tableData.id, - tableName, - isView ? "view" : isMatView ? "matview" : "table", - { - partitionBy: tableData.partitionBy, - walEnabled: tableData.walEnabled, - designatedTimestamp: tableData.designatedTimestamp, - }, - ) - }, [handleExplainSchema, tableData, tableName, isMatView, isView]) + void handleExplainSchema(tableData.id, tableName, kind, { + partitionBy: tableData.partitionBy, + walEnabled: tableData.walEnabled, + designatedTimestamp: tableData.designatedTimestamp, + }) + }, [handleExplainSchema, tableData, tableName, kind]) const handleAskAIForIssue = useCallback( (issue: HealthIssue) => { void trackEvent(ConsoleEvent.TABLE_DETAILS_ASK_AI) if (tableData?.id == null) return - let samples = undefined - if (issue.field === "transactionLag") { - samples = trendData.transactionLag - } else if (issue.field === "pendingRows") { - samples = trendData.walPendingRowCount - } - - void handleAskAIForHealthIssue(tableData.id, tableName, issue, samples) + const diagnosticContext = + kindData.kind === "matview" && kindData.matView.status === "ready" + ? { + source: "materialized_views()" as const, + data: kindData.matView.data, + } + : kindData.kind === "liveview" && kindData.liveView.status === "ready" + ? { + source: "live_views()" as const, + data: kindData.liveView.data, + guidance: getLiveViewIssueGuidance(issue.id), + } + : undefined + + void handleAskAIForHealthIssue( + tableData.id, + tableName, + issue, + getTrendSamplesForIssue(issue.field, trendData), + diagnosticContext, + ) }, - [handleAskAIForHealthIssue, tableData, tableName, trendData], + [handleAskAIForHealthIssue, tableData, tableName, trendData, kindData], ) const handleAskAIForViewIssue = useCallback(() => { @@ -284,263 +510,134 @@ export const TableDetailsDrawer = () => { viewData?.invalidation_reason, ]) - const fetchTableData = useCallback(async () => { - try { - const escapedName = tableName.replace(/'/g, "''") - const response = await quest.query
( - `tables() WHERE table_name = '${escapedName}'`, - ) - if (response.type === QuestDB.Type.DQL && response.data.length > 0) { - setTableData(response.data[0]) - } else if ( - response.type === QuestDB.Type.DQL && - response.data.length === 0 - ) { - dispatch( - actions.console.replaceSidebarHistory({ - type: "tableDetails", - payload: null, - }), - ) - } - } catch (error) { - console.error("Failed to fetch table data:", error) - } - }, [quest, tableName]) - - const fetchMatViewData = useCallback(async () => { - if (!isMatView) return - try { - const escapedName = tableName.replace(/'/g, "''") - const response = await quest.query( - `materialized_views() WHERE view_name = '${escapedName}'`, - ) - if (response.type === QuestDB.Type.DQL && response.data.length > 0) { - setMatViewData(response.data[0]) - } - } catch (error) { - console.error("Failed to fetch materialized view data:", error) - } - }, [quest, tableName, isMatView]) - - const fetchViewData = useCallback(async () => { - if (!isView) return - try { - const escapedName = tableName.replace(/'/g, "''") - const response = await quest.query( - `views() WHERE view_name = '${escapedName}'`, - ) - if (response.type === QuestDB.Type.DQL && response.data.length > 0) { - setViewData(response.data[0]) - } else if ( - response.type === QuestDB.Type.DQL && - response.data.length === 0 - ) { - dispatch( - actions.console.replaceSidebarHistory({ - type: "tableDetails", - payload: null, - }), - ) - } - } catch (error) { - console.error("Failed to fetch view data:", error) - } - }, [quest, tableName, isView, dispatch]) + useEffect(() => { + targetRef.current = target + activeSidebarRef.current = activeSidebar + }, [target, activeSidebar]) - const fetchColumns = useCallback(async () => { - try { - const response = await quest.showColumns(tableName) - if (response.type === QuestDB.Type.DQL) { - setColumns(response.data) - } - } catch (error) { - console.error("Failed to fetch columns:", error) - } - }, [quest, tableName]) - - const fetchDDL = useCallback(async () => { - try { - const response = isView - ? await quest.showViewDDL(tableName) - : isMatView - ? await quest.showMatViewDDL(tableName) - : await quest.showTableDDL(tableName) - if (response.type === QuestDB.Type.DQL && response.data[0]?.ddl) { - setDdl(response.data[0].ddl) - } - } catch (error) { - console.error("Failed to fetch DDL:", error) + useEffect(() => { + setColumnsExpanded(isOpen && hasTarget ? isView : false) + setWalExpanded(true) + setHasAutoExpanded(false) + setTrendData({ + walPendingRowCount: [], + transactionLag: [], + ingestionMetric: [], + }) + setBaseTableStatus(null) + }, [isOpen, hasTarget, isView, sourcePrefix]) + + useEffect(() => { + if ( + tableSource.state.status === "ready" && + tableSource.state.data.type === "missing" + ) { + clearIfCurrentTarget(tableName, kind) } - }, [quest, tableName, isMatView, isView]) + }, [clearIfCurrentTarget, kind, tableName, tableSource.state]) - const checkBaseTableStatus = useCallback(async () => { - if (!isMatView || !matViewData?.base_table_name) { + useEffect(() => { + if (!baseTableName || kindSourceUnavailable) { setBaseTableStatus(null) return } - try { - const escapedName = matViewData.base_table_name.replace(/'/g, "''") - const response = await quest.query
( - `tables() WHERE table_name = '${escapedName}'`, - ) - const baseTableExists = - response.type === QuestDB.Type.DQL && response.data.length > 0 - const suspended = baseTableExists - ? response.data[0]?.table_suspended - : false - const status = baseTableExists - ? suspended - ? "Suspended" - : "Valid" - : "Dropped" - setBaseTableStatus(status) - } catch (error) { - console.error("Failed to check base table existence:", error) - setBaseTableStatus(null) - } - }, [quest, isMatView, matViewData?.base_table_name]) - - const fetchAllData = useCallback(async () => { - setLoading(true) - await Promise.all([ - fetchTableData(), - fetchMatViewData(), - fetchViewData(), - fetchColumns(), - fetchDDL(), - ]) - setLoading(false) - }, [fetchTableData, fetchMatViewData, fetchViewData, fetchColumns, fetchDDL]) - useEffect(() => { - if (isOpen && hasTarget) { - setTableData(null) - setMatViewData(null) - setViewData(null) - setColumns([]) - setDdl("") - setColumnsExpanded(isView) - setWalExpanded(true) - setHasAutoExpanded(false) - setTrendData({ - walPendingRowCount: [], - transactionLag: [], - ingestionMetric: [], - }) - setBaseTableStatus(null) - void fetchAllData() - } else if (!isOpen || !hasTarget) { - setTableData(null) - setMatViewData(null) - setViewData(null) - setColumns([]) - setDdl("") - setColumnsExpanded(false) - setWalExpanded(true) - setHasAutoExpanded(false) - setTrendData({ - walPendingRowCount: [], - transactionLag: [], - ingestionMetric: [], - }) - setBaseTableStatus(null) + let active = true + + const checkBaseTableStatus = async () => { + try { + const response = await quest.getTableDetails(baseTableName) + if (!active) return + + const baseTableExists = + response.type === QuestDB.Type.DQL && response.data.length > 0 + const suspended = baseTableExists + ? response.data[0]?.table_suspended + : false + const status = baseTableExists + ? suspended + ? "Suspended" + : "Valid" + : "Dropped" + setBaseTableStatus(status) + } catch (error) { + if (!active) return + + console.error("Failed to check base table existence:", error) + setBaseTableStatus(null) + } } - }, [isOpen, hasTarget, tableName, fetchAllData]) - useEffect(() => { - if (matViewData?.base_table_name) { - void checkBaseTableStatus() + void checkBaseTableStatus() + + return () => { + active = false } - }, [matViewData?.base_table_name, checkBaseTableStatus]) + }, [baseTableName, kindSourceUnavailable, quest, sourcePrefix]) + + const usesDetailsPolling = isView || activeTab === "details" useAdaptivePoll({ - fetchFn: fetchTableData, - enabled: isOpen && hasTarget && !loading && !isView, - key: `${tableName}-${activeTab}`, - minIntervalMs: activeTab === "monitoring" ? 200 : 1000, - maxIntervalMs: activeTab === "monitoring" ? 5000 : 1000, + fetchFn: tableSource.fetchNow, + enabled: isOpen && hasTarget, + key: `${sourcePrefix}-${activeTab}`, + minIntervalMs: usesDetailsPolling + ? DETAILS_TABLE_POLL_MS + : TABLE_POLL_MIN_MS, + maxIntervalMs: usesDetailsPolling + ? DETAILS_TABLE_POLL_MS + : TABLE_POLL_MAX_MS, multiplier: 1.5, }) useEffect(() => { - if (tableData && !loading) { + if ( + tableSource.state.status === "ready" && + tableSource.state.data.type === "found" + ) { + const currentTableData = tableSource.state.data.data const now = Date.now() setTrendData((prev) => { - // For ingestion detection: use wal_txn for WAL tables, table_row_count for non-WAL - const ingestionValue = tableData.walEnabled - ? (tableData.wal_txn ?? 0) - : (tableData.table_row_count ?? 0) + const ingestionValue = currentTableData.walEnabled + ? (currentTableData.wal_txn ?? BIGINT_ZERO) + : (currentTableData.table_row_count ?? BIGINT_ZERO) + const transactionLag = + (currentTableData.wal_txn ?? BIGINT_ZERO) - + (currentTableData.table_txn ?? BIGINT_ZERO) return { - walPendingRowCount: tableData.walEnabled + walPendingRowCount: currentTableData.walEnabled ? [ ...prev.walPendingRowCount.slice(-(MAX_TREND_SAMPLES - 1)), { - value: Number(tableData.wal_pending_row_count) || 0, + value: currentTableData.wal_pending_row_count ?? BIGINT_ZERO, timestamp: now, }, ] : prev.walPendingRowCount, - transactionLag: tableData.walEnabled + transactionLag: currentTableData.walEnabled ? [ ...prev.transactionLag.slice(-(MAX_TREND_SAMPLES - 1)), { - value: Math.max( - 0, - (Number(tableData.wal_txn) || 0) - - (Number(tableData.table_txn) || 0), - ), + value: + transactionLag > BIGINT_ZERO ? transactionLag : BIGINT_ZERO, timestamp: now, }, ] : prev.transactionLag, ingestionMetric: [ ...prev.ingestionMetric.slice(-(MAX_TREND_SAMPLES - 1)), - { value: Number(ingestionValue) || 0, timestamp: now }, + { value: ingestionValue, timestamp: now }, ], } }) } - }, [tableData, loading]) - - useEffect(() => { - if (!isOpen || !hasTarget || !isMatView) return - - const interval = setInterval(() => { - void fetchMatViewData() - }, 1000) - - return () => clearInterval(interval) - }, [isOpen, hasTarget, isMatView, fetchMatViewData]) - - useEffect(() => { - if (!isOpen || !hasTarget || !isView) return - - const interval = setInterval(() => { - void fetchViewData() - }, 1000) - - return () => clearInterval(interval) - }, [isOpen, hasTarget, isView, fetchViewData]) - - useEffect(() => { - if (!isOpen || !hasTarget) return - // Not needed for monitoring - if (!isView && activeTab !== "details") return - - const interval = setInterval(() => { - void fetchColumns() - void fetchDDL() - }, 1000) - - return () => clearInterval(interval) - }, [isOpen, hasTarget, isView, activeTab, fetchColumns, fetchDDL]) + }, [tableSource.state]) const rawHealthStatus = useMemo(() => { if (!tableData) return null - return calculateHealthStatus(tableData, matViewData, trendData, isMatView) - }, [tableData, matViewData, trendData, isMatView]) + return calculateHealthStatus(tableData, kindData, trendData) + }, [tableData, kindData, trendData]) const healthStatus = useDebouncedWarnings(rawHealthStatus) @@ -570,13 +667,10 @@ export const TableDetailsDrawer = () => { }, [healthStatus]) const monitoringIssuesCounts = useMemo(() => { - if (!healthStatus) return { warnings: 0, errors: 0 } - const errors = healthStatus.issues.filter( - (i) => i.severity === "critical", - ).length - const warnings = healthStatus.issues.filter( - (i) => i.severity === "warning", - ).length + const errors = + healthStatus?.issues.filter((i) => i.severity === "critical").length ?? 0 + const warnings = + healthStatus?.issues.filter((i) => i.severity === "warning").length ?? 0 return { warnings, errors } }, [healthStatus]) @@ -591,11 +685,12 @@ export const TableDetailsDrawer = () => { }, [healthStatus]) const isIngestionDisabled = useMemo(() => { - // Disable ingestion section when WAL is suspended or matview is invalid const walSuspended = tableData?.walEnabled && tableData?.table_suspended const matViewInvalid = isMatView && matViewData?.view_status === "invalid" - return walSuspended || matViewInvalid - }, [tableData, isMatView, matViewData]) + const liveViewInvalid = + isLiveView && liveViewData?.view_status === "invalid" + return walSuspended || matViewInvalid || liveViewInvalid + }, [tableData, isMatView, matViewData, isLiveView, liveViewData]) useEffect(() => { if (hasIngestionWarning && !hasAutoExpanded && !walExpanded) { @@ -604,20 +699,41 @@ export const TableDetailsDrawer = () => { } }, [hasIngestionWarning, hasAutoExpanded, walExpanded]) + const healthSeverity = useMemo(() => { + const calculatedSeverity = isView + ? viewData?.view_status === "invalid" + ? "critical" + : kindSourceUnavailable + ? "unknown" + : "healthy" + : (healthStatus?.overallSeverity ?? "healthy") + + if ( + tablesUnavailable && + calculatedSeverity !== "critical" && + calculatedSeverity !== "warning" + ) { + return "unknown" + } + return calculatedSeverity + }, [ + healthStatus?.overallSeverity, + isView, + kindSourceUnavailable, + tablesUnavailable, + viewData?.view_status, + ]) + + const kindSourceTitle = isLiveView + ? "Unable to load live view metadata" + : isMatView + ? "Unable to load materialized view metadata" + : "Unable to load view metadata" + const drawerTitle = useMemo( () => ( - {hasTarget && ( - - )} + {hasTarget && } { )} ), - [ - hasTarget, - isView, - viewData?.view_status, - healthStatus?.overallSeverity, - tableOptions, - tableName, - ], + [hasTarget, healthSeverity, tableOptions, tableName], ) return ( @@ -661,7 +770,7 @@ export const TableDetailsDrawer = () => { afterTitle={ hasTarget ? ( - {isView ? "View" : isMatView ? "Materialized View" : "Table"} + {getTableKindLabel(kind)} ) : undefined } @@ -676,8 +785,37 @@ export const TableDetailsDrawer = () => { Loading table details... + ) : hasTarget && tablesUnavailable && tableData === null ? ( + + + ) : tableData ? ( <> + {tablesUnavailable && ( + + + + )} + {kindSourceUnavailable && ( + + + + )} {!isView && ( @@ -738,13 +876,13 @@ export const TableDetailsDrawer = () => { {!isView && activeTab === "monitoring" && ( { {(isView || activeTab === "details") && ( { {!isView && ( diff --git a/src/scenes/Schema/TableDetailsDrawer/shared-styles.tsx b/src/scenes/Schema/TableDetailsDrawer/shared-styles.tsx index 70423fccf..740fd1c89 100644 --- a/src/scenes/Schema/TableDetailsDrawer/shared-styles.tsx +++ b/src/scenes/Schema/TableDetailsDrawer/shared-styles.tsx @@ -61,3 +61,10 @@ export const CaretIcon = styled(CaretRightIcon)<{ $expanded?: boolean }>` transition: transform 150ms ease; transform: rotate(${({ $expanded }) => ($expanded ? "90deg" : "0deg")}); ` + +export const UnavailableValue = styled.span.attrs({ + children: "Unavailable", +})` + color: ${({ theme }) => theme.color.contentSecondary}; + font-size: ${({ theme }) => theme.fontSize.md}; +` diff --git a/src/scenes/Schema/TableDetailsDrawer/sourceState.test.ts b/src/scenes/Schema/TableDetailsDrawer/sourceState.test.ts new file mode 100644 index 000000000..094ea206c --- /dev/null +++ b/src/scenes/Schema/TableDetailsDrawer/sourceState.test.ts @@ -0,0 +1,255 @@ +import { describe, expect, it } from "vitest" +import { + createSourceMachineState, + nextSourceState, + SOURCE_FAILURE_GRACE_MS, +} from "./sourceState" + +describe("source state", () => { + it("keeps the source ready during two transient failures", () => { + // Given + const ready = nextSourceState(createSourceMachineState("trades"), { + type: "success", + key: "trades", + data: "metadata", + }) + + // When + const firstFailure = nextSourceState(ready, { + type: "failure", + key: "trades", + at: 0, + }) + const secondFailure = nextSourceState(firstFailure, { + type: "failure", + key: "trades", + at: SOURCE_FAILURE_GRACE_MS, + }) + + // Then + expect(secondFailure.source).toEqual({ + status: "ready", + data: "metadata", + }) + }) + + it("waits for the grace deadline after three fast failures", () => { + // Given + const initial = createSourceMachineState("trades") + + // When + const first = nextSourceState(initial, { + type: "failure", + key: "trades", + at: 0, + }) + const second = nextSourceState(first, { + type: "failure", + key: "trades", + at: 100, + }) + const third = nextSourceState(second, { + type: "failure", + key: "trades", + at: 200, + }) + const admitted = nextSourceState(third, { + type: "failure-deadline", + key: "trades", + at: SOURCE_FAILURE_GRACE_MS, + }) + + // Then + expect(third.source.status).toBe("loading") + expect(admitted.source.status).toBe("unavailable") + }) + + it("admits the third failure after the grace period", () => { + // Given + const initial = createSourceMachineState("trades") + + // When + const first = nextSourceState(initial, { + type: "failure", + key: "trades", + at: 0, + }) + const second = nextSourceState(first, { + type: "failure", + key: "trades", + at: 1_000, + }) + const third = nextSourceState(second, { + type: "failure", + key: "trades", + at: SOURCE_FAILURE_GRACE_MS, + }) + + // Then + expect(third.source.status).toBe("unavailable") + }) + + it("admits one timed out request", () => { + // Given + const initial = createSourceMachineState("trades") + + // When + const result = nextSourceState(initial, { + type: "timeout", + key: "trades", + }) + + // Then + expect(result.source.status).toBe("unavailable") + }) + + it("requires two consecutive successes to recover", () => { + // Given + const unavailable = nextSourceState(createSourceMachineState("trades"), { + type: "timeout", + key: "trades", + }) + + // When + const first = nextSourceState(unavailable, { + type: "success", + key: "trades", + data: "first", + }) + const second = nextSourceState(first, { + type: "success", + key: "trades", + data: "second", + }) + + // Then + expect(first.source.status).toBe("unavailable") + expect(second.source).toEqual({ status: "ready", data: "second" }) + }) + + it("resets the failure sequence after a success", () => { + // Given + const initial = createSourceMachineState("trades") + const first = nextSourceState(initial, { + type: "failure", + key: "trades", + at: 0, + }) + const second = nextSourceState(first, { + type: "failure", + key: "trades", + at: 1_000, + }) + + // When + const successful = nextSourceState(second, { + type: "success", + key: "trades", + data: "metadata", + }) + const nextFailure = nextSourceState(successful, { + type: "failure", + key: "trades", + at: 3_000, + }) + + // Then + expect(nextFailure.source.status).toBe("ready") + expect(nextFailure.consecutiveFailures).toBe(1) + expect(nextFailure.firstFailureAt).toBe(3_000) + }) + + it("resets recovery progress after a failure", () => { + // Given + const unavailable = nextSourceState(createSourceMachineState("trades"), { + type: "timeout", + key: "trades", + }) + const recovering = nextSourceState(unavailable, { + type: "success", + key: "trades", + data: "first", + }) + + // When + const failed = nextSourceState(recovering, { + type: "failure", + key: "trades", + at: 0, + }) + const nextSuccess = nextSourceState(failed, { + type: "success", + key: "trades", + data: "second", + }) + + // Then + expect(nextSuccess.source.status).toBe("unavailable") + }) + + it("resets all internal state for a new target", () => { + // Given + const oldTarget = nextSourceState(createSourceMachineState("old"), { + type: "failure", + key: "old", + at: 100, + }) + + // When + const newTarget = createSourceMachineState("new") + const staleOutcome = nextSourceState(newTarget, { + type: "failure", + key: oldTarget.key, + at: 200, + }) + + // Then + expect(newTarget).toEqual({ + key: "new", + source: { status: "loading" }, + lastReadyData: null, + consecutiveFailures: 0, + firstFailureAt: null, + consecutiveRecoveries: 0, + }) + expect(staleOutcome).toBe(newTarget) + }) + + it("ignores cancellations and stale target outcomes", () => { + // Given + const initial = createSourceMachineState("new-target") + + // When + const cancelled = nextSourceState(initial, { + type: "cancelled", + key: "new-target", + }) + const stale = nextSourceState(cancelled, { + type: "success", + key: "old-target", + data: "stale", + }) + + // Then + expect(stale).toBe(initial) + }) + + it("retains the last ready data after admitted failures", () => { + // Given + const ready = nextSourceState(createSourceMachineState("trades"), { + type: "success", + key: "trades", + data: "metadata", + }) + + // When + const unavailable = nextSourceState(ready, { + type: "timeout", + key: "trades", + }) + + // Then + expect(unavailable.source.status).toBe("unavailable") + expect(unavailable.lastReadyData).toBe("metadata") + }) +}) diff --git a/src/scenes/Schema/TableDetailsDrawer/sourceState.ts b/src/scenes/Schema/TableDetailsDrawer/sourceState.ts new file mode 100644 index 000000000..7c6108311 --- /dev/null +++ b/src/scenes/Schema/TableDetailsDrawer/sourceState.ts @@ -0,0 +1,103 @@ +import type { SourceState } from "./types" + +export const SOURCE_FAILURE_THRESHOLD = 3 +export const SOURCE_FAILURE_GRACE_MS = 2_000 +const SOURCE_RECOVERY_THRESHOLD = 2 +export const SOURCE_TIMEOUT_MS = 10_000 + +export type SourceMachineState = { + key: string + source: SourceState + lastReadyData: T | null + consecutiveFailures: number + firstFailureAt: number | null + consecutiveRecoveries: number +} + +export type SourceOutcome = + | { type: "success"; key: string; data: T } + | { type: "failure"; key: string; at: number } + | { type: "failure-deadline"; key: string; at: number } + | { type: "timeout"; key: string } + | { type: "cancelled"; key: string } + +export const createSourceMachineState = ( + key: string, +): SourceMachineState => ({ + key, + source: { status: "loading" }, + lastReadyData: null, + consecutiveFailures: 0, + firstFailureAt: null, + consecutiveRecoveries: 0, +}) + +const unavailableState = ( + state: SourceMachineState, +): SourceMachineState => ({ + ...state, + source: { status: "unavailable" }, + consecutiveRecoveries: 0, +}) + +export const nextSourceState = ( + state: SourceMachineState, + outcome: SourceOutcome, +): SourceMachineState => { + if (outcome.key !== state.key || outcome.type === "cancelled") { + return state + } + + if (outcome.type === "timeout") { + return unavailableState(state) + } + + if (outcome.type === "success") { + if (state.source.status === "unavailable") { + const consecutiveRecoveries = state.consecutiveRecoveries + 1 + if (consecutiveRecoveries < SOURCE_RECOVERY_THRESHOLD) { + return { + ...state, + consecutiveFailures: 0, + firstFailureAt: null, + consecutiveRecoveries, + } + } + } + + return { + ...state, + source: { status: "ready", data: outcome.data }, + lastReadyData: outcome.data, + consecutiveFailures: 0, + firstFailureAt: null, + consecutiveRecoveries: 0, + } + } + + if (outcome.type === "failure-deadline") { + const failureWindowElapsed = + state.firstFailureAt !== null && + outcome.at - state.firstFailureAt >= SOURCE_FAILURE_GRACE_MS + return state.consecutiveFailures >= SOURCE_FAILURE_THRESHOLD && + failureWindowElapsed + ? unavailableState(state) + : state + } + + const firstFailureAt = state.firstFailureAt ?? outcome.at + const consecutiveFailures = state.consecutiveFailures + 1 + const failureWindowElapsed = + outcome.at - firstFailureAt >= SOURCE_FAILURE_GRACE_MS + + const failedState: SourceMachineState = { + ...state, + consecutiveFailures, + firstFailureAt, + consecutiveRecoveries: 0, + } + + return consecutiveFailures >= SOURCE_FAILURE_THRESHOLD && failureWindowElapsed + ? unavailableState(failedState) + : failedState +} diff --git a/src/scenes/Schema/TableDetailsDrawer/types.ts b/src/scenes/Schema/TableDetailsDrawer/types.ts new file mode 100644 index 000000000..4a713cbed --- /dev/null +++ b/src/scenes/Schema/TableDetailsDrawer/types.ts @@ -0,0 +1,16 @@ +import type { + LiveView, + MaterializedView, + View, +} from "../../../utils/questdb/types" + +export type SourceState = + | { status: "loading" } + | { status: "ready"; data: T } + | { status: "unavailable" } + +export type TableKindData = + | { kind: "table" } + | { kind: "view"; view: SourceState } + | { kind: "matview"; matView: SourceState } + | { kind: "liveview"; liveView: SourceState } diff --git a/src/scenes/Schema/TableDetailsDrawer/useCatalogSource.ts b/src/scenes/Schema/TableDetailsDrawer/useCatalogSource.ts new file mode 100644 index 000000000..82a25f9a5 --- /dev/null +++ b/src/scenes/Schema/TableDetailsDrawer/useCatalogSource.ts @@ -0,0 +1,204 @@ +import { useCallback, useContext, useEffect, useRef, useState } from "react" +import { QuestContext } from "../../../providers" +import * as QuestDB from "../../../utils/questdb" +import { + createSourceMachineState, + nextSourceState, + SOURCE_FAILURE_GRACE_MS, + SOURCE_FAILURE_THRESHOLD, + SOURCE_TIMEOUT_MS, + type SourceMachineState, +} from "./sourceState" +import type { SourceState } from "./types" + +type Params = { + sourceKey: string + sourceName: string + enabled: boolean + query: string + pollIntervalMs: number | null + transformResponse: (response: QuestDB.QueryRawResult) => T | undefined +} + +type CatalogSource = { + state: SourceState + lastReadyData: T | null + fetchNow: () => Promise +} + +const isCancelledRequest = (error: unknown): boolean => + typeof error === "object" && + error !== null && + "error" in error && + error.error === "Cancelled by user" + +export const useCatalogSource = ({ + sourceKey, + sourceName, + enabled, + query, + pollIntervalMs, + transformResponse, +}: Params): CatalogSource => { + const { quest } = useContext(QuestContext) + const [machine, setMachine] = useState>(() => + createSourceMachineState(sourceKey), + ) + const activeQueryIdRef = useRef(null) + const currentKeyRef = useRef(sourceKey) + + const fetchNow = useCallback(async () => { + if (!enabled || activeQueryIdRef.current !== null) return + + const requestKey = sourceKey + let queryId: QuestDB.QueryId | null = null + let timeoutId: number | null = null + let timedOut = false + + try { + const request = quest.queryRaw(query, { cancellable: true }) + queryId = request.queryId + activeQueryIdRef.current = request.queryId + + const timeout = new Promise((_, reject) => { + timeoutId = window.setTimeout(() => { + timedOut = true + if (activeQueryIdRef.current === request.queryId) { + quest.abort(request.queryId) + } + reject(new Error(`${sourceName} request timed out`)) + }, SOURCE_TIMEOUT_MS) + }) + + const response = await Promise.race([request.promise, timeout]) + if ( + currentKeyRef.current !== requestKey || + activeQueryIdRef.current !== queryId + ) { + return + } + + const data = transformResponse(response) + setMachine((previous) => + nextSourceState( + previous, + data === undefined + ? { type: "failure", key: requestKey, at: Date.now() } + : { type: "success", key: requestKey, data }, + ), + ) + } catch (error) { + if ( + currentKeyRef.current !== requestKey || + activeQueryIdRef.current !== queryId + ) { + return + } + if (isCancelledRequest(error) && !timedOut) return + + setMachine((previous) => + nextSourceState( + previous, + timedOut + ? { type: "timeout", key: requestKey } + : { type: "failure", key: requestKey, at: Date.now() }, + ), + ) + console.error(`Failed to fetch ${sourceName}:`, error) + } finally { + if (timeoutId !== null) { + window.clearTimeout(timeoutId) + } + if (activeQueryIdRef.current === queryId) { + activeQueryIdRef.current = null + } + } + }, [enabled, quest, query, sourceKey, sourceName, transformResponse]) + + useEffect(() => { + currentKeyRef.current = sourceKey + }, [sourceKey]) + + // Keyed to the source identity alone: a source that is merely disabled keeps + // its last answer, so re-enabling it renders that instead of a loading state. + useEffect(() => { + setMachine(createSourceMachineState(sourceKey)) + }, [sourceKey]) + + useEffect(() => { + const activeQueryId = activeQueryIdRef.current + if (activeQueryId !== null) { + quest.abort(activeQueryId) + activeQueryIdRef.current = null + } + + if (!enabled) return + + void fetchNow() + + return () => { + const currentQueryId = activeQueryIdRef.current + if (currentQueryId !== null) { + quest.abort(currentQueryId) + activeQueryIdRef.current = null + } + } + }, [enabled, fetchNow, quest, sourceKey]) + + useEffect(() => { + if (!enabled || pollIntervalMs === null) return + + const intervalId = window.setInterval(() => { + void fetchNow() + }, pollIntervalMs) + + return () => window.clearInterval(intervalId) + }, [enabled, fetchNow, pollIntervalMs]) + + useEffect(() => { + if ( + machine.key !== sourceKey || + machine.source.status === "unavailable" || + machine.consecutiveFailures < SOURCE_FAILURE_THRESHOLD || + machine.firstFailureAt === null + ) { + return + } + + const remaining = Math.max( + 0, + SOURCE_FAILURE_GRACE_MS - (Date.now() - machine.firstFailureAt), + ) + const deadlineId = window.setTimeout(() => { + setMachine((previous) => + nextSourceState(previous, { + type: "failure-deadline", + key: sourceKey, + at: Date.now(), + }), + ) + }, remaining) + + return () => window.clearTimeout(deadlineId) + }, [ + machine.consecutiveFailures, + machine.firstFailureAt, + machine.key, + machine.source.status, + sourceKey, + ]) + + if (machine.key !== sourceKey) { + return { + state: { status: "loading" }, + lastReadyData: null, + fetchNow, + } + } + + return { + state: machine.source, + lastReadyData: machine.lastReadyData, + fetchNow, + } +} diff --git a/src/scenes/Schema/TableDetailsDrawer/useDebouncedWarnings.test.ts b/src/scenes/Schema/TableDetailsDrawer/useDebouncedWarnings.test.ts index 6f4772ba2..9f4092083 100644 --- a/src/scenes/Schema/TableDetailsDrawer/useDebouncedWarnings.test.ts +++ b/src/scenes/Schema/TableDetailsDrawer/useDebouncedWarnings.test.ts @@ -2,6 +2,11 @@ import { describe, it, expect, beforeEach } from "vitest" import { applyDebounceFilter, updateFirstSeen } from "./useDebouncedWarnings" import type { HealthStatus, HealthIssue, TrendIndicator } from "./healthCheck" +const DEBOUNCED_ISSUE_FIELDS: Record = { + Y1: "transactionLag", + Y2: "pendingRows", +} + const makeHealthStatus = ( issueIds: string[], options?: { trendDirection?: "increasing" | "decreasing" }, @@ -9,14 +14,13 @@ const makeHealthStatus = ( const issues: HealthIssue[] = issueIds.map((id) => ({ id, severity: "warning" as const, - field: id === "Y1" ? "transactionLag" : id === "Y2" ? "pendingRows" : id, + field: DEBOUNCED_ISSUE_FIELDS[id] ?? id, message: `Issue ${id}`, })) const trendIndicators = new Map() for (const id of issueIds) { - const trendKey = - id === "Y1" ? "transactionLag" : id === "Y2" ? "pendingRows" : null + const trendKey = DEBOUNCED_ISSUE_FIELDS[id] ?? null if (trendKey) { trendIndicators.set(trendKey, { field: trendKey, @@ -34,6 +38,7 @@ const makeHealthStatus = ( return { overallSeverity: issues.length > 0 ? "warning" : "healthy", + hasUnavailableSource: false, issues, fieldIssues, trendIndicators, @@ -80,6 +85,20 @@ describe("useDebouncedWarnings", () => { expect(result.trendIndicators.has("transactionLag")).toBe(false) }) + it("should preserve unknown when an unavailable source has an unconfirmed warning", () => { + // Given + const raw = { + ...makeHealthStatus(["Y1"]), + hasUnavailableSource: true, + } + + // When + const result = sim.process(raw, 0) + + // Then + expect(result?.overallSeverity).toBe("unknown") + }) + it("should confirm Y1 after 5 seconds of continuous presence", () => { const raw = makeHealthStatus(["Y1"]) @@ -213,6 +232,7 @@ describe("useDebouncedWarnings", () => { for (const i of issues) fieldIssues.set(i.field, i) const raw: HealthStatus = { overallSeverity: "warning", + hasUnavailableSource: false, issues, fieldIssues, trendIndicators: new Map(), diff --git a/src/scenes/Schema/TableDetailsDrawer/useDebouncedWarnings.ts b/src/scenes/Schema/TableDetailsDrawer/useDebouncedWarnings.ts index e4db3d710..a1bebc375 100644 --- a/src/scenes/Schema/TableDetailsDrawer/useDebouncedWarnings.ts +++ b/src/scenes/Schema/TableDetailsDrawer/useDebouncedWarnings.ts @@ -50,12 +50,15 @@ export function applyDebounceFilter( overallSeverity = "critical" } else if (filteredIssues.some((i) => i.severity === "warning")) { overallSeverity = "warning" + } else if (rawHealthStatus.hasUnavailableSource) { + overallSeverity = "unknown" } else if (filteredIssues.some((i) => i.severity === "recovering")) { overallSeverity = "recovering" } return { overallSeverity, + hasUnavailableSource: rawHealthStatus.hasUnavailableSource, issues: filteredIssues, fieldIssues, trendIndicators: adjustedTrends, diff --git a/src/scenes/Schema/TableDetailsDrawer/utils.test.ts b/src/scenes/Schema/TableDetailsDrawer/utils.test.ts index 9bfc0da61..8b6811ca2 100644 --- a/src/scenes/Schema/TableDetailsDrawer/utils.test.ts +++ b/src/scenes/Schema/TableDetailsDrawer/utils.test.ts @@ -1,5 +1,19 @@ import { describe, it, expect } from "vitest" -import { extractStoragePolicyClauses, formatTTL } from "./utils" +import { + formatStoragePolicyClauses, + formatBytes, + formatInterval, + formatMicrosDuration, + formatRowCount, + formatTTL, + formatTxnCount, + formatUtcTimestamp, + getTrendSamplesForIssue, +} from "./utils" +import type { TrendData } from "./healthCheck" +import type { StoragePolicy } from "../../../utils/questdb/types" + +const digitsOf = (formatted: string) => formatted.replace(/\D/g, "") describe("formatTTL", () => { it("returns None for a value of 0", () => { @@ -28,73 +42,190 @@ describe("formatTTL", () => { }) }) -describe("extractStoragePolicyClauses", () => { - it("returns an empty array when the DDL has no storage policy", () => { - const ddl = `CREATE TABLE 'trades' ( - symbol SYMBOL, price DOUBLE, ts TIMESTAMP - ) timestamp(ts) PARTITION BY DAY;` - expect(extractStoragePolicyClauses(ddl)).toEqual([]) +describe("formatInterval", () => { + it("returns None for a zero value or a missing unit", () => { + expect(formatInterval(BigInt(0), "SECONDS")).toBe("None") + expect(formatInterval(BigInt(5), "")).toBe("None") }) - it("returns an empty array for unparseable DDL", () => { - expect(extractStoragePolicyClauses("not a valid sql statement")).toEqual([]) + it("uses a singular Title Case unit for a value of 1", () => { + expect(formatInterval(BigInt(1), "SECONDS")).toBe("1 Second") + expect(formatInterval(BigInt(1), "SECOND")).toBe("1 Second") }) - it("returns an empty array for an empty string", () => { - expect(extractStoragePolicyClauses("")).toEqual([]) + it("uses a plural Title Case unit for values other than 1", () => { + expect(formatInterval(BigInt(5), "SECOND")).toBe("5 Seconds") + expect(formatInterval(BigInt(2), "MINUTES")).toBe("2 Minutes") }) +}) - it("extracts a single clause with a plural unit", () => { - const ddl = `CREATE TABLE 'trades' (price DOUBLE, ts TIMESTAMP) - timestamp(ts) PARTITION BY DAY - STORAGE POLICY(TO PARQUET 3 DAYS);` - expect(extractStoragePolicyClauses(ddl)).toEqual([ - { action: "To Parquet", duration: "3 Days" }, - ]) +describe("formatUtcTimestamp", () => { + it("drops an all-zero fraction for a compact UTC string", () => { + expect(formatUtcTimestamp("2026-08-24T10:31:00.000000Z")).toBe( + "2026-08-24 10:31:00 UTC", + ) + }) + + it("keeps the microsecond precision the server sends", () => { + // Given a START FROM NOW boundary resolved mid-second on the server + expect(formatUtcTimestamp("2026-08-25T13:00:00.472913Z")).toBe( + "2026-08-25 13:00:00.472913 UTC", + ) + }) + + it("trims trailing zeros from the fraction", () => { + expect(formatUtcTimestamp("2026-08-25T13:00:00.472000Z")).toBe( + "2026-08-25 13:00:00.472 UTC", + ) + }) + + it("returns the raw value when the timestamp does not parse", () => { + expect(formatUtcTimestamp("not a timestamp")).toBe("not a timestamp") + }) +}) + +describe("formatTxnCount", () => { + it("should render Unknown for a null count", () => { + expect(formatTxnCount(null)).toBe("Unknown") + }) + + it("should use the singular unit for a count of 1", () => { + expect(formatTxnCount(BigInt(1))).toBe("1 txn") + }) + + it("should use the plural unit and the viewer's locale grouping", () => { + expect(formatTxnCount(BigInt(0))).toBe("0 txns") + expect(formatTxnCount(BigInt(1500))).toBe(`${(1500).toLocaleString()} txns`) + }) + + it("should preserve unsafe LONG values", () => { + // Given a LONG above Number.MAX_SAFE_INTEGER, whatever the viewer's locale + const unsafeLong = BigInt("9007199254740993") + + // Then every digit survives the formatting + expect(digitsOf(formatTxnCount(unsafeLong))).toBe("9007199254740993") + expect(digitsOf(formatRowCount(unsafeLong))).toBe("9007199254740993") + }) +}) + +describe("formatMicrosDuration", () => { + it("renders sub-second values as rounded milliseconds", () => { + expect(formatMicrosDuration(BigInt(0))).toBe("0 ms") + expect(formatMicrosDuration(BigInt(2_500))).toBe("3 ms") + expect(formatMicrosDuration(BigInt(999_999))).toBe("1000 ms") + }) + + it("renders sub-minute values as seconds with one decimal", () => { + expect(formatMicrosDuration(BigInt(1_000_000))).toBe("1.0 s") + expect(formatMicrosDuration(BigInt(2_500_000))).toBe("2.5 s") }) - it("normalizes a value of 1 to a singular unit", () => { - const ddl = `CREATE TABLE 'trades' (price DOUBLE, ts TIMESTAMP) - timestamp(ts) PARTITION BY DAY - STORAGE POLICY(TO PARQUET 1 DAYS);` - expect(extractStoragePolicyClauses(ddl)).toEqual([ + it("renders sub-hour values as minutes with one decimal", () => { + expect(formatMicrosDuration(BigInt(60_000_000))).toBe("1.0 min") + expect(formatMicrosDuration(BigInt(90_000_000))).toBe("1.5 min") + }) + + it("renders values of an hour and above as hours with one decimal", () => { + expect(formatMicrosDuration(BigInt(3_600_000_000))).toBe("1.0 h") + expect(formatMicrosDuration(BigInt(5_400_000_000))).toBe("1.5 h") + }) +}) + +describe("formatBytes", () => { + it("renders values under one KiB as bytes", () => { + expect(formatBytes(BigInt(0))).toBe("0 B") + expect(formatBytes(BigInt(1023))).toBe("1023 B") + }) + + it("renders values under one MiB as KiB with one decimal", () => { + expect(formatBytes(BigInt(1024))).toBe("1.0 KiB") + expect(formatBytes(BigInt(1536))).toBe("1.5 KiB") + }) + + it("renders values under one GiB as MiB with one decimal", () => { + expect(formatBytes(BigInt(1024 ** 2))).toBe("1.0 MiB") + expect(formatBytes(BigInt(8_388_608))).toBe("8.0 MiB") + }) + + it("renders values of one GiB and above as GiB with one decimal", () => { + expect(formatBytes(BigInt(1024 ** 3))).toBe("1.0 GiB") + expect(formatBytes(BigInt(1024 ** 3 * 1.5))).toBe("1.5 GiB") + }) +}) + +describe("formatStoragePolicyClauses", () => { + const policy: StoragePolicy = { + table_dir_name: "trades~1", + to_parquet: "24h", + to_remote: "168h", + drop_local: "2160h", + drop_remote: "1m", + status: "A", + last_updated: "2026-09-01T00:00:00.000000Z", + } + + it("formats the catalog durations in pipeline order", () => { + // Given / When / Then + expect(formatStoragePolicyClauses(policy)).toEqual([ { action: "To Parquet", duration: "1 Day" }, + { action: "To Remote", duration: "1 Week" }, + { action: "Drop Local", duration: "90 Days" }, + { action: "Drop Remote", duration: "1 Month" }, ]) }) - it("extracts all clauses in pipeline order", () => { - const ddl = `CREATE TABLE 'trades' (price DOUBLE, ts TIMESTAMP) - timestamp(ts) PARTITION BY DAY - STORAGE POLICY(TO PARQUET 1 DAYS, TO REMOTE 10 DAYS, DROP LOCAL 1 MONTHS, DROP REMOTE 2 YEARS);` - expect(extractStoragePolicyClauses(ddl)).toEqual([ + it("omits stages that the catalog reports as zero", () => { + // Given + const policyWithoutOptionalStages = { + ...policy, + to_remote: "0h", + drop_local: "0h", + drop_remote: "0h", + } + + // When / Then + expect(formatStoragePolicyClauses(policyWithoutOptionalStages)).toEqual([ { action: "To Parquet", duration: "1 Day" }, - { action: "To Remote", duration: "10 Days" }, - { action: "Drop Local", duration: "1 Month" }, - { action: "Drop Remote", duration: "2 Years" }, ]) }) - it("returns an empty array for the retired DROP NATIVE syntax", () => { - // Given a DDL from a pre-release server build that still emits the - // removed DROP NATIVE stage โ€” the parse failure deliberately degrades - // to "no clauses" rather than an error - const ddl = `CREATE TABLE 'trades' (price DOUBLE, ts TIMESTAMP) - timestamp(ts) PARTITION BY DAY - STORAGE POLICY(TO PARQUET 3 DAYS, DROP NATIVE 10 DAYS);` - expect(extractStoragePolicyClauses(ddl)).toEqual([]) - }) - - it("extracts TO PARQUET + TO REMOTE with a trailing OWNED BY (reported repro)", () => { - // Given the reported DDL whose TO REMOTE clause used to fail the parse - const ddl = `CREATE TABLE 'corporate_bonds' ( - ts TIMESTAMP, isin SYMBOL, price DOUBLE - ) timestamp(ts) PARTITION BY DAY - STORAGE POLICY(TO PARQUET 3 DAYS, TO REMOTE 30 DAYS) - OWNED BY 'admin';` - // Then both stages render instead of an empty "Not configured" section - expect(extractStoragePolicyClauses(ddl)).toEqual([ - { action: "To Parquet", duration: "3 Days" }, - { action: "To Remote", duration: "30 Days" }, - ]) + it("formats complete years that the catalog reports as months", () => { + // Given + const policyWithYear = { ...policy, drop_remote: "12m" } + + // When / Then + expect(formatStoragePolicyClauses(policyWithYear)).toContainEqual({ + action: "Drop Remote", + duration: "1 Year", + }) + }) + + it("returns no clauses when the table has no policy row", () => { + // Given / When / Then + expect(formatStoragePolicyClauses(null)).toEqual([]) + }) +}) + +describe("getTrendSamplesForIssue", () => { + const trendData: TrendData = { + walPendingRowCount: [{ value: BigInt(1), timestamp: 1 }], + transactionLag: [{ value: BigInt(2), timestamp: 2 }], + ingestionMetric: [{ value: BigInt(3), timestamp: 3 }], + } + + it("should return the matching series for every field that has a trend", () => { + // Given / When / Then + expect(getTrendSamplesForIssue("transactionLag", trendData)).toBe( + trendData.transactionLag, + ) + expect(getTrendSamplesForIssue("pendingRows", trendData)).toBe( + trendData.walPendingRowCount, + ) + }) + + it("should return undefined for a field with no trend series", () => { + // Given / When / Then + expect(getTrendSamplesForIssue("writerStall", trendData)).toBeUndefined() + expect(getTrendSamplesForIssue("viewStatus", trendData)).toBeUndefined() }) }) diff --git a/src/scenes/Schema/TableDetailsDrawer/utils.ts b/src/scenes/Schema/TableDetailsDrawer/utils.ts index b8352b9e8..61f2bc6ad 100644 --- a/src/scenes/Schema/TableDetailsDrawer/utils.ts +++ b/src/scenes/Schema/TableDetailsDrawer/utils.ts @@ -1,7 +1,11 @@ import { formatDistance } from "date-fns" -import { parseOne, type StoragePolicy } from "@questdb/sql-parser" +import type { TimestampedSample, TrendData } from "./healthCheck" +import type { StoragePolicy } from "../../../utils/questdb/types" import { fetchUserLocale, getLocaleFromLanguage } from "../../../utils" +const BIGINT_ZERO = BigInt(0) +const BIGINT_ONE = BigInt(1) + export function formatRelativeTimestamp(timestamp: string | null): string { if (!timestamp) return "Never" const date = new Date(timestamp) @@ -28,17 +32,16 @@ export function formatMemoryPressure(level: number | null): string { } } -export function formatRowCount(count: number | string | null): string { +export function formatRowCount(count: bigint | null): string { if (count == null) return "0" - return typeof count === "number" - ? count.toLocaleString() - : Number(count).toLocaleString() + return count.toLocaleString() } -function formatDurationUnit(value: number, unit: string): string { +function formatDurationUnit(value: number | bigint, unit: string): string { const lower = unit.toLowerCase() const singular = lower.endsWith("s") ? lower.slice(0, -1) : lower - const normalized = value === 1 ? singular : `${singular}s` + const isOne = typeof value === "bigint" ? value === BIGINT_ONE : value === 1 + const normalized = isOne ? singular : `${singular}s` return normalized.charAt(0).toUpperCase() + normalized.slice(1) } @@ -47,33 +50,119 @@ export function formatTTL(value?: number, unit?: string): string { return `${value} ${formatDurationUnit(value, unit)}` } +export function formatInterval( + value: bigint | null, + unit: string | null, +): string { + if (value == null) return "Unknown" + if (value === BIGINT_ZERO || !unit) return "None" + return `${value.toLocaleString()} ${formatDurationUnit(value, unit)}` +} + +// String-based so the server's microsecond precision survives: Date only +// keeps milliseconds, and a START FROM NOW boundary is a microsecond value. +export function formatUtcTimestamp(timestamp: string): string { + const isoMatch = timestamp.match( + /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2})(?:\.(\d+))?Z$/, + ) + if (isoMatch) { + const [, date, time, fraction = ""] = isoMatch + const subSeconds = fraction.replace(/0+$/, "") + return `${date} ${time}${subSeconds ? `.${subSeconds}` : ""} UTC` + } + const date = new Date(timestamp) + if (isNaN(date.getTime())) return timestamp + return `${date + .toISOString() + .replace("T", " ") + .replace(/\.\d{3}Z$/, "")} UTC` +} + +export function formatTxnCount(count: bigint | null): string { + if (count == null) return "Unknown" + return `${count.toLocaleString()} txn${count === BIGINT_ONE ? "" : "s"}` +} + +export function formatMicrosDuration(micros: bigint): string { + const value = Number(micros) + if (value < 1_000_000) return `${Math.round(value / 1_000)} ms` + if (value < 60_000_000) return `${(value / 1_000_000).toFixed(1)} s` + if (value < 3_600_000_000) return `${(value / 60_000_000).toFixed(1)} min` + return `${(value / 3_600_000_000).toFixed(1)} h` +} + +export function formatBytes(bytes: bigint | null): string { + if (bytes == null) return "Unknown" + const value = Number(bytes) + if (value < 1024) return `${value} B` + if (value < 1024 ** 2) return `${(value / 1024).toFixed(1)} KiB` + if (value < 1024 ** 3) return `${(value / 1024 ** 2).toFixed(1)} MiB` + return `${(value / 1024 ** 3).toFixed(1)} GiB` +} + +export function getTrendSamplesForIssue( + field: string, + trendData: TrendData, +): TimestampedSample[] | undefined { + switch (field) { + case "transactionLag": + return trendData.transactionLag + case "pendingRows": + return trendData.walPendingRowCount + default: + return undefined + } +} + export type StoragePolicyClause = { action: string; duration: string } const STORAGE_POLICY_LABELS = [ - ["toParquet", "To Parquet"], - ["toRemote", "To Remote"], - ["dropLocal", "Drop Local"], - ["dropRemote", "Drop Remote"], + ["to_parquet", "To Parquet"], + ["to_remote", "To Remote"], + ["drop_local", "Drop Local"], + ["drop_remote", "Drop Remote"], ] as const -export function extractStoragePolicyClauses( - ddl: string, -): StoragePolicyClause[] { - let stmt: { storagePolicy?: StoragePolicy } | undefined - try { - stmt = parseOne(ddl) as { storagePolicy?: StoragePolicy } - } catch { - return [] +const formatStoragePolicyDuration = (duration: string): string => { + const match = duration.match(/^(\d+)([a-z]+)$/) + if (!match) return duration + + const value = Number(match[1]) + const unit = match[2] + if (unit === "h" && value % (24 * 7) === 0) { + const weeks = value / (24 * 7) + return `${weeks} ${formatDurationUnit(weeks, "week")}` } - const policy = stmt?.storagePolicy + if (unit === "h" && value % 24 === 0) { + const days = value / 24 + return `${days} ${formatDurationUnit(days, "day")}` + } + if (unit === "m" && value % 12 === 0) { + const years = value / 12 + return `${years} ${formatDurationUnit(years, "year")}` + } + + const unitName = { + h: "hour", + d: "day", + w: "week", + m: "month", + y: "year", + }[unit] + return unitName ? `${value} ${formatDurationUnit(value, unitName)}` : duration +} + +export function formatStoragePolicyClauses( + policy: StoragePolicy | null, +): StoragePolicyClause[] { if (!policy) return [] return STORAGE_POLICY_LABELS.flatMap(([key, label]) => { - const v = policy[key] - if (!v) return [] + const duration = policy[key] + if (!duration || /^0[a-z]+$/.test(duration)) return [] return [ { action: label, - duration: `${v.value} ${formatDurationUnit(v.value, v.unit)}`, + duration: formatStoragePolicyDuration(duration), }, ] }) diff --git a/src/scenes/Schema/VirtualTables/index.tsx b/src/scenes/Schema/VirtualTables/index.tsx index 1931d6672..b10012dee 100644 --- a/src/scenes/Schema/VirtualTables/index.tsx +++ b/src/scenes/Schema/VirtualTables/index.tsx @@ -36,10 +36,17 @@ import { TABLES_GROUP_KEY, MATVIEWS_GROUP_KEY, VIEWS_GROUP_KEY, + LIVEVIEWS_GROUP_KEY, } from "../localStorageUtils" import { useSchema } from "../SchemaContext" +import { getLiveViewFailure } from "../TableDetailsDrawer/healthCheck" import { QuestContext } from "../../../providers" -import { PartitionBy, SymbolColumnDetails } from "../../../utils/questdb/types" +import { + getTableKindLabel, + PartitionBy, + SymbolColumnDetails, + TableKind, +} from "../../../utils/questdb/types" import { useSelector, useDispatch } from "react-redux" import { selectors, actions } from "../../../store" import { @@ -63,6 +70,7 @@ type VirtualTablesProps = { tables: QuestDB.Table[] materializedViews?: QuestDB.MaterializedView[] views?: QuestDB.View[] + liveViews?: QuestDB.LiveView[] filterSuspendedOnly: boolean state: State loadingError: ErrorResult | null @@ -96,6 +104,7 @@ export type FlattenedTreeItem = { column?: TreeColumn matViewData?: QuestDB.MaterializedView viewData?: QuestDB.View + liveViewData?: QuestDB.LiveView walTableData?: QuestDB.WalTable parent?: string isExpanded?: boolean @@ -155,19 +164,6 @@ const Loader = styled(Loader3)` ${spinAnimation}; ` -export const getTableKindLabel = (kind: "table" | "matview" | "view") => { - switch (kind) { - case "table": - return "Table" - case "matview": - return "Materialized view" - case "view": - return "View" - default: - return "" - } -} - const Loading = () => { const [loaderShown, setLoaderShown] = useState(false) // Show the loader only for delayed fetching process @@ -183,6 +179,7 @@ const VirtualTables: FC = ({ tables, materializedViews, views, + liveViews, filterSuspendedOnly, state, loadingError, @@ -223,46 +220,57 @@ const VirtualTables: FC = ({ const wrapperRef = useRef(null) useRetainLastFocus({ virtuosoRef, focusedIndex, setFocusedIndex, wrapperRef }) - const [regularTables, matViewTables, viewTables] = useMemo(() => { - return tables - .reduce( - (acc, table: QuestDB.Table) => { - const normalizedTableName = table.table_name.toLowerCase() - const normalizedQuery = query.toLowerCase() - const tableNameMatches = normalizedTableName.includes(normalizedQuery) - const columnMatches = - !!query && - !!allColumns[table.table_name]?.some((col) => - col.column_name.toLowerCase().includes(normalizedQuery), - ) - const shownIfFilteredSuspendedOnly = filterSuspendedOnly - ? table.walEnabled && table.table_suspended - : true - const shownIfFilteredWithQuery = tableNameMatches || columnMatches - - if (shownIfFilteredSuspendedOnly && shownIfFilteredWithQuery) { - // Use table_type to categorize: 'T' = table, 'M' = matview, 'V' = view - // Default to 'T' (table) for backward compatibility with older servers - const tableType = - (table.table_type as "T" | "M" | "V" | undefined) ?? "T" - const categoryIndex = - tableType === "M" ? 1 : tableType === "V" ? 2 : 0 - acc[categoryIndex].push({ - ...table, - hasColumnMatches: columnMatches, - }) + const [regularTables, matViewTables, viewTables, liveViewTables] = + useMemo(() => { + return tables + .reduce( + (acc, table: QuestDB.Table) => { + const normalizedTableName = table.table_name.toLowerCase() + const normalizedQuery = query.toLowerCase() + const tableNameMatches = + normalizedTableName.includes(normalizedQuery) + const columnMatches = + !!query && + !!allColumns[table.table_name]?.some((col) => + col.column_name.toLowerCase().includes(normalizedQuery), + ) + const shownIfFilteredSuspendedOnly = filterSuspendedOnly + ? table.walEnabled && table.table_suspended + : true + const shownIfFilteredWithQuery = tableNameMatches || columnMatches + + if (shownIfFilteredSuspendedOnly && shownIfFilteredWithQuery) { + // Use table_type to categorize: 'T' = table, 'M' = matview, 'V' = view, 'L' = live view + // Default to 'T' (table) for backward compatibility with older servers + const tableType = table.table_type ?? "T" + const categoryIndex = + tableType === "M" + ? 1 + : tableType === "V" + ? 2 + : tableType === "L" + ? 3 + : 0 + acc[categoryIndex].push({ + ...table, + hasColumnMatches: columnMatches, + }) + return acc + } return acc - } - return acc - }, - [[], [], []] as (QuestDB.Table & { hasColumnMatches: boolean })[][], - ) - .map((tables) => - tables.sort((a, b) => - a.table_name.toLowerCase().localeCompare(b.table_name.toLowerCase()), - ), - ) - }, [tables, query, filterSuspendedOnly, allColumns]) + }, + [[], [], [], []] as (QuestDB.Table & { + hasColumnMatches: boolean + })[][], + ) + .map((tables) => + tables.sort((a, b) => + a.table_name + .toLowerCase() + .localeCompare(b.table_name.toLowerCase()), + ), + ) + }, [tables, query, filterSuspendedOnly, allColumns]) const flattenedItems = useMemo(() => { return Object.values(schemaTree).reduce((acc, node) => { @@ -273,35 +281,23 @@ const VirtualTables: FC = ({ const getTableSchema = async ( tableName: string, - kind: "table" | "matview" | "view", + kind: TableKind, ): Promise => { try { - const response = - kind === "matview" - ? await quest.showMatViewDDL(tableName) - : kind === "view" - ? await quest.showViewDDL(tableName) - : await quest.showTableDDL(tableName) + const response = await quest.showDDL(tableName, kind) if (response?.type === QuestDB.Type.DQL && response.data?.[0]?.ddl) { return response.data[0].ddl } } catch (_error) { - const kindLabel = - kind === "matview" - ? "materialized view" - : kind === "view" - ? "view" - : "table" - toast.error(`Cannot fetch schema for ${kindLabel} '${tableName}'`) + toast.error( + `Cannot fetch schema for ${getTableKindLabel(kind).toLowerCase()} '${tableName}'`, + ) } return null } - const handleCopyQuery = async ( - tableName: string, - kind: "table" | "matview" | "view", - ) => { + const handleCopyQuery = async (tableName: string, kind: TableKind) => { void trackEvent(ConsoleEvent.SCHEMA_CONTEXT_COPY_DDL, { kind }) const schema = await getTableSchema(tableName, kind) if (schema) { @@ -543,20 +539,22 @@ const VirtualTables: FC = ({ if ( item.id === TABLES_GROUP_KEY || item.id === MATVIEWS_GROUP_KEY || - item.id === VIEWS_GROUP_KEY + item.id === VIEWS_GROUP_KEY || + item.id === LIVEVIEWS_GROUP_KEY ) { - const isTable = item.id === TABLES_GROUP_KEY - const isMatView = item.id === MATVIEWS_GROUP_KEY - const isEmpty = isTable - ? regularTables.length === 0 - : isMatView - ? matViewTables.length === 0 - : viewTables.length === 0 - const hookLabel = isTable - ? "tables" - : isMatView - ? "materialized-views" - : "views" + const groupTables = { + [TABLES_GROUP_KEY]: regularTables, + [MATVIEWS_GROUP_KEY]: matViewTables, + [VIEWS_GROUP_KEY]: viewTables, + [LIVEVIEWS_GROUP_KEY]: liveViewTables, + }[item.id] + const isEmpty = groupTables.length === 0 + const hookLabel = { + [TABLES_GROUP_KEY]: "tables", + [MATVIEWS_GROUP_KEY]: "materialized-views", + [VIEWS_GROUP_KEY]: "views", + [LIVEVIEWS_GROUP_KEY]: "live-views", + }[item.id] return ( = ({ if ( item.kind === "table" || item.kind === "matview" || - item.kind === "view" + item.kind === "view" || + item.kind === "liveview" ) { - const canSuspend = item.kind !== "view" // Views cannot be suspended + // A view's WAL can suspend (its ALTER VIEW ... AS definition change is + // a WAL transaction), but no ALTER grammar can resume it + const suspendableKind = item.kind === "view" ? null : item.kind + const canSuspend = suspendableKind !== null + const liveViewFailure = item.liveViewData + ? getLiveViewFailure(item.liveViewData) + : null const handleOpenDetailsDrawer = () => { if ( activeSidebar?.type === "tableDetails" && @@ -606,11 +611,7 @@ const VirtualTables: FC = ({ dispatch( actions.console.pushSidebarHistory({ type: "tableDetails", - payload: { - tableName: item.name, - isMatView: item.kind === "matview", - isView: item.kind === "view", - }, + payload: { tableName: item.name, kind: item.kind as TableKind }, }), ) setTimeout(() => setFocusedIndex(index)) @@ -645,21 +646,22 @@ const VirtualTables: FC = ({ errors={[ ...(item.matViewData?.view_status === "invalid" ? [ - `Materialized view is invalid${item.matViewData?.invalidation_reason && `: ${item.matViewData?.invalidation_reason}`}`, + `Materialized view is invalid${item.matViewData.invalidation_reason ? `: ${item.matViewData.invalidation_reason}` : ""}`, ] : []), ...(item.viewData?.view_status === "invalid" ? [ - `View is invalid${item.viewData?.invalidation_reason && `: ${item.viewData?.invalidation_reason}`}`, + `View is invalid${item.viewData.invalidation_reason ? `: ${item.viewData.invalidation_reason}` : ""}`, ] : []), + ...(liveViewFailure ? [liveViewFailure.message] : []), ...(item.table?.table_suspended ? [`Suspended`] : []), ]} /> - {canSuspend && item.table?.table_suspended && ( + {suspendableKind && item.table?.table_suspended && ( { setOpenedSuspensionDialog(isOpen ? item.id : null) @@ -679,10 +681,7 @@ const VirtualTables: FC = ({ - await handleCopyQuery( - item.name, - item.kind as "table" | "matview" | "view", - ) + await handleCopyQuery(item.name, item.kind as TableKind) } icon={} > @@ -751,7 +750,7 @@ const VirtualTables: FC = ({ await handleExplainSchema( item.table.id, item.name, - item.kind as "table" | "matview" | "view", + item.kind as TableKind, { partitionBy: item.partitionBy, walEnabled: item.walEnabled, @@ -816,6 +815,7 @@ const VirtualTables: FC = ({ regularTables, matViewTables, viewTables, + liveViewTables, toggleNodeExpansion, openedContextMenu, openedSuspensionDialog, @@ -826,98 +826,66 @@ const VirtualTables: FC = ({ useEffect(() => { if (state.view === View.ready) { - const newTree: SchemaTree = { - [TABLES_GROUP_KEY]: { - id: TABLES_GROUP_KEY, - kind: "folder", - name: `Tables (${regularTables.length})`, - isExpanded: - regularTables.length === 0 - ? false - : getSectionExpanded(TABLES_GROUP_KEY), - children: regularTables.map((table) => { - const node = createTableNode( - table, - TABLES_GROUP_KEY, - false, - false, - materializedViews, - views, - allColumns[table.table_name] ?? [], - ) - if (table.hasColumnMatches) { - node.isExpanded = true - // Also mark the columns folder as expanded (but not persisted) - const columnsFolder = node.children.find((child) => - child.id.endsWith(":columns"), - ) - if (columnsFolder) { - columnsFolder.isExpanded = true - } - } - return node - }), - }, - [MATVIEWS_GROUP_KEY]: { - id: MATVIEWS_GROUP_KEY, - kind: "folder", - name: `Materialized views (${matViewTables.length})`, - isExpanded: - matViewTables.length === 0 - ? false - : getSectionExpanded(MATVIEWS_GROUP_KEY), - children: matViewTables.map((table) => { - const node = createTableNode( - table, - MATVIEWS_GROUP_KEY, - true, - false, - materializedViews, - views, - allColumns[table.table_name] ?? [], - ) - if (table.hasColumnMatches) { - node.isExpanded = true - const columnsFolder = node.children.find((child) => - child.id.endsWith(":columns"), - ) - if (columnsFolder) { - columnsFolder.isExpanded = true - } - } - return node - }), - }, - [VIEWS_GROUP_KEY]: { - id: VIEWS_GROUP_KEY, - kind: "folder", - name: `Views (${viewTables.length})`, - isExpanded: - viewTables.length === 0 - ? false - : getSectionExpanded(VIEWS_GROUP_KEY), - children: viewTables.map((table) => { - const node = createTableNode( - table, - VIEWS_GROUP_KEY, - false, - true, - materializedViews, - views, - allColumns[table.table_name] ?? [], + const createGroupNode = ( + groupKey: string, + groupName: string, + groupTables: (QuestDB.Table & { hasColumnMatches: boolean })[], + kind: TableKind, + ): TreeNode => ({ + id: groupKey, + kind: "folder", + name: `${groupName} (${groupTables.length})`, + isExpanded: + groupTables.length === 0 ? false : getSectionExpanded(groupKey), + children: groupTables.map((table) => { + const node = createTableNode( + table, + groupKey, + kind, + materializedViews, + views, + liveViews, + allColumns[table.table_name] ?? [], + ) + if (table.hasColumnMatches) { + node.isExpanded = true + // Also mark the columns folder as expanded (but not persisted) + const columnsFolder = node.children.find((child) => + child.id.endsWith(":columns"), ) - if (table.hasColumnMatches) { - node.isExpanded = true - const columnsFolder = node.children.find((child) => - child.id.endsWith(":columns"), - ) - if (columnsFolder) { - columnsFolder.isExpanded = true - } + if (columnsFolder) { + columnsFolder.isExpanded = true } - return node - }), - }, + } + return node + }), + }) + + const newTree: SchemaTree = { + [TABLES_GROUP_KEY]: createGroupNode( + TABLES_GROUP_KEY, + "Tables", + regularTables, + "table", + ), + [MATVIEWS_GROUP_KEY]: createGroupNode( + MATVIEWS_GROUP_KEY, + "Materialized views", + matViewTables, + "matview", + ), + [LIVEVIEWS_GROUP_KEY]: createGroupNode( + LIVEVIEWS_GROUP_KEY, + "Live views", + liveViewTables, + "liveview", + ), + [VIEWS_GROUP_KEY]: createGroupNode( + VIEWS_GROUP_KEY, + "Views", + viewTables, + "view", + ), } fetchedSymbolsRef.current.clear() @@ -928,9 +896,10 @@ const VirtualTables: FC = ({ regularTables, matViewTables, viewTables, + liveViewTables, materializedViews, views, - + liveViews, allColumns, ]) diff --git a/src/scenes/Schema/VirtualTables/utils.ts b/src/scenes/Schema/VirtualTables/utils.ts index 95e982686..4f12e614a 100644 --- a/src/scenes/Schema/VirtualTables/utils.ts +++ b/src/scenes/Schema/VirtualTables/utils.ts @@ -130,27 +130,30 @@ const createStorageDetailsNodes = ( export const createTableNode = ( table: QuestDB.Table, parentId: string, - isMatView: boolean = false, - isView: boolean = false, + kind: QuestDB.TableKind, materializedViews: QuestDB.MaterializedView[] | undefined, views: QuestDB.View[] | undefined, + liveViews: QuestDB.LiveView[] | undefined, tableColumns: InformationSchemaColumn[], ): TreeNode => { const tableId = `${parentId}:${table.table_name}` - const matViewData = isMatView - ? materializedViews?.find((mv) => mv.view_name === table.table_name) - : undefined - const viewData = isView - ? views?.find((v) => v.view_name === table.table_name) - : undefined + const matViewData = + kind === "matview" + ? materializedViews?.find((mv) => mv.view_name === table.table_name) + : undefined + const viewData = + kind === "view" + ? views?.find((v) => v.view_name === table.table_name) + : undefined + const liveViewData = + kind === "liveview" + ? liveViews?.find((lv) => lv.view_name === table.table_name) + : undefined const columnsId = `${tableId}:columns` const baseTablesId = `${tableId}:baseTables` const storageDetailsId = `${tableId}:storageDetails` - // Determine the kind - const kind = isMatView ? "matview" : isView ? "view" : "table" - const tableNode: TreeNode = { id: tableId, kind, @@ -158,6 +161,7 @@ export const createTableNode = ( table, matViewData, viewData, + liveViewData, parent: parentId, isExpanded: getSectionExpanded(tableId), partitionBy: table.partitionBy, @@ -173,8 +177,8 @@ export const createTableNode = ( isExpanded: getSectionExpanded(columnsId), children: createColumnNodes(table, columnsId, tableColumns), }, - // Only show storage details for tables and materialized views (not for regular views) - ...(!isView + // Only show storage details for stored kinds (not for regular views) + ...(kind !== "view" ? [ { id: storageDetailsId, @@ -189,7 +193,9 @@ export const createTableNode = ( ], } - if (isMatView && matViewData) { + const baseTableName = + matViewData?.base_table_name ?? liveViewData?.base_table_name + if (baseTableName) { tableNode.children.push({ id: baseTablesId, kind: "folder", @@ -198,9 +204,9 @@ export const createTableNode = ( isExpanded: getSectionExpanded(baseTablesId), children: [ { - id: `${baseTablesId}:${matViewData.base_table_name}`, + id: `${baseTablesId}:${baseTableName}`, kind: "detail", - name: matViewData.base_table_name, + name: baseTableName, parent: baseTablesId, children: [], }, diff --git a/src/scenes/Schema/index.tsx b/src/scenes/Schema/index.tsx index 391505b42..187e3e8e5 100644 --- a/src/scenes/Schema/index.tsx +++ b/src/scenes/Schema/index.tsx @@ -70,9 +70,7 @@ import { RefreshRate, } from "../../scenes/Editor/Metrics/utils" import type { Duration } from "../../scenes/Editor/Metrics/types" -import { useSchema } from "./SchemaContext" -import { SchemaProvider } from "./SchemaContext" -import { TreeNodeKind } from "./Row" +import { SchemaProvider, useSchema, type SelectedTable } from "./SchemaContext" import { toast } from "../../components/Toast" import { trackEvent } from "../../modules/ConsoleEventTracker" import { ConsoleEvent } from "../../modules/ConsoleEventTracker/events" @@ -143,6 +141,7 @@ const Schema = ({ const [materializedViews, setMaterializedViews] = useState() const [views, setViews] = useState() + const [liveViews, setLiveViews] = useState() const dispatch = useDispatch() const [filterSuspendedOnly, setFilterSuspendedOnly] = useState(false) const { autoRefreshTables, updateSettings } = useLocalStorage() @@ -170,6 +169,9 @@ const Schema = ({ if (data.some((t) => t.table_type === "V")) { void fetchViews() } + if (data.some((t) => t.table_type === "L")) { + void fetchLiveViews() + } dispatchState({ view: View.ready }) } else { dispatchState({ view: View.error }) @@ -183,9 +185,7 @@ const Schema = ({ const fetchMaterializedViews = async () => { try { - const matViewsResponse = await quest.query( - "materialized_views()", - ) + const matViewsResponse = await quest.showMaterializedViews() if (matViewsResponse && matViewsResponse.type === QuestDB.Type.DQL) { setMaterializedViews(matViewsResponse.data) } @@ -205,6 +205,17 @@ const Schema = ({ } } + const fetchLiveViews = async () => { + try { + const liveViewsResponse = await quest.showLiveViews() + if (liveViewsResponse && liveViewsResponse.type === QuestDB.Type.DQL) { + setLiveViews(liveViewsResponse.data) + } + } catch (error) { + // Fail silently + } + } + const fetchColumns = async () => { const queries = [ "information_schema.questdb_columns()", @@ -231,17 +242,11 @@ const Schema = ({ const copySchemasToClipboard = async () => { void trackEvent(ConsoleEvent.SCHEMA_COPY_MULTIPLE) if (!tables) return - const tablesWithError: { name: string; type: TreeNodeKind }[] = [] + const tablesWithError: SelectedTable[] = [] const ddls = await Promise.all( selectedTables.map(async (table) => { try { - // selectedTables only contains "table" | "matview" | "view" types from allSelectableTables - const response = - table.type === "matview" - ? await quest.showMatViewDDL(table.name) - : table.type === "view" - ? await quest.showViewDDL(table.name) - : await quest.showTableDDL(table.name) + const response = await quest.showDDL(table.name, table.type) if (response?.type === QuestDB.Type.DQL && response.data?.[0]?.ddl) { return response.data[0].ddl @@ -332,23 +337,27 @@ const Schema = ({ } }, [autoRefreshTables]) - const allSelectableTables = useMemo(() => { + const allSelectableTables = useMemo(() => { if (!tables) return [] // Default to 'T' (table) for backward compatibility with older servers const regularTables = tables .filter((t) => (t.table_type ?? "T") === "T") - .map((t) => ({ name: t.table_name, type: "table" as TreeNodeKind })) + .map((t) => ({ name: t.table_name, type: "table" as const })) const matViews = tables .filter((t) => t.table_type === "M") - .map((t) => ({ name: t.table_name, type: "matview" as TreeNodeKind })) + .map((t) => ({ name: t.table_name, type: "matview" as const })) + + const liveViewsList = tables + .filter((t) => t.table_type === "L") + .map((t) => ({ name: t.table_name, type: "liveview" as const })) const viewsList = tables .filter((t) => t.table_type === "V") - .map((t) => ({ name: t.table_name, type: "view" as TreeNodeKind })) + .map((t) => ({ name: t.table_name, type: "view" as const })) - return [...regularTables, ...matViews, ...viewsList] + return [...regularTables, ...matViews, ...liveViewsList, ...viewsList] }, [tables]) const suspendedTablesCount = useMemo( @@ -518,6 +527,7 @@ const Schema = ({ tables={tables ?? []} materializedViews={materializedViews} views={views} + liveViews={liveViews} filterSuspendedOnly={filterSuspendedOnly} state={state} loadingError={loadingError} diff --git a/src/scenes/Schema/localStorageUtils.ts b/src/scenes/Schema/localStorageUtils.ts index 9f00f65f0..7bca92b5c 100644 --- a/src/scenes/Schema/localStorageUtils.ts +++ b/src/scenes/Schema/localStorageUtils.ts @@ -2,6 +2,7 @@ const STORAGE_KEY_PREFIX = "questdb:expanded:" export const TABLES_GROUP_KEY = `${STORAGE_KEY_PREFIX}tables` export const MATVIEWS_GROUP_KEY = `${STORAGE_KEY_PREFIX}matviews` export const VIEWS_GROUP_KEY = `${STORAGE_KEY_PREFIX}views` +export const LIVEVIEWS_GROUP_KEY = `${STORAGE_KEY_PREFIX}liveviews` export const getItemFromStorage = (key: string): boolean => { try { diff --git a/src/scenes/Schema/table-icon.tsx b/src/scenes/Schema/table-icon.tsx index 1c3e03d2a..7d32c51ec 100644 --- a/src/scenes/Schema/table-icon.tsx +++ b/src/scenes/Schema/table-icon.tsx @@ -6,7 +6,7 @@ import { color } from "../../utils" import * as QuestDB from "../../utils/questdb" type TableIconProps = { - kind: "table" | "matview" | "view" + kind: QuestDB.TableKind walEnabled?: boolean partitionBy?: QuestDB.PartitionBy designatedTimestamp?: string @@ -85,6 +85,35 @@ export const MaterializedViewIcon = ({ ) +export const LiveViewIcon = ({ size = DEFAULT_SIZE }: { size?: string }) => ( + + + + + + + + + + +) + export const ViewIcon = ({ height, width, @@ -136,7 +165,7 @@ export const TableIcon: FC = ({ - {partitionText}, {timestampText}. + {QuestDB.getTableKindLabel(kind)}. {partitionText}, {timestampText}. } delay={1000} @@ -149,6 +178,24 @@ export const TableIcon: FC = ({ ) } + if (kind === "liveview") { + return ( + + {QuestDB.getTableKindLabel(kind)}. {partitionText}, {timestampText}. + + } + delay={1000} + placement="bottom" + > + + + + + ) + } + return ( + ({ table_name: name, table_type: tableType }) as Table + +const TABLES = [ + makeTable("btc_trades", "T"), + makeTable("btc_trades_mv", "M"), + makeTable("btc_trades_view", "V"), + makeTable("btc_trades_lv", "L"), +] + +describe("createModelToolsClient getTableSchema", () => { + it("should request DDL with the kind resolved from the table list", async () => { + // Given a quest client whose DDL requests are captured + const showDDL = vi + .fn<[string, string], Promise>() + .mockResolvedValue({ type: Type.DQL, data: [{ ddl: "CREATE ..." }] }) + const toolsClient = createModelToolsClient( + { showDDL } as unknown as Client, + TABLES, + ) + + // When the model asks for the schema of each table kind + await toolsClient.getTableSchema?.("btc_trades") + await toolsClient.getTableSchema?.("btc_trades_mv") + await toolsClient.getTableSchema?.("btc_trades_view") + await toolsClient.getTableSchema?.("btc_trades_lv") + + // Then every request carries the kind of its own table + expect(showDDL.mock.calls).toEqual([ + ["btc_trades", "table"], + ["btc_trades_mv", "matview"], + ["btc_trades_view", "view"], + ["btc_trades_lv", "liveview"], + ]) + }) + + it("should return the DDL from a successful response", async () => { + // Given a quest client that answers with DDL + const showDDL = vi.fn().mockResolvedValue({ + type: Type.DQL, + data: [{ ddl: "CREATE LIVE VIEW 'btc_trades_lv' ..." }], + }) + const toolsClient = createModelToolsClient( + { showDDL } as unknown as Client, + TABLES, + ) + + // When / Then + await expect(toolsClient.getTableSchema?.("btc_trades_lv")).resolves.toBe( + "CREATE LIVE VIEW 'btc_trades_lv' ...", + ) + }) + + it("should return null without a DDL request for an unknown table", async () => { + // Given a quest client and a table name that is not in the table list + const showDDL = vi.fn() + const toolsClient = createModelToolsClient( + { showDDL } as unknown as Client, + TABLES, + ) + + // When / Then + await expect(toolsClient.getTableSchema?.("nope")).resolves.toBeNull() + expect(showDDL).not.toHaveBeenCalled() + }) +}) diff --git a/src/utils/ai/aiAssistant.ts b/src/utils/ai/aiAssistant.ts index 157640543..45b7c80bc 100644 --- a/src/utils/ai/aiAssistant.ts +++ b/src/utils/ai/aiAssistant.ts @@ -190,9 +190,10 @@ export function createModelToolsClient( return null } - const ddlResponse = table.matView - ? await questClient.showMatViewDDL(tableName) - : await questClient.showTableDDL(tableName) + const ddlResponse = await questClient.showDDL( + tableName, + getTableKind(table), + ) if ( ddlResponse?.type === Type.DQL && diff --git a/src/utils/ai/executeAIFlow.ts b/src/utils/ai/executeAIFlow.ts index 26c69f188..15fdbaf31 100644 --- a/src/utils/ai/executeAIFlow.ts +++ b/src/utils/ai/executeAIFlow.ts @@ -135,8 +135,13 @@ type HealthIssueFlowConfig = BaseFlowConfig & { severity: "critical" | "warning" } tableDetails: string + diagnosticDetails?: { + source: "materialized_views()" | "live_views()" + data: string + } + issueGuidance?: string monitoringDocs: string - trendSamples?: Array<{ value: number; timestamp: number }> + trendSamples?: Array<{ value: bigint; timestamp: number }> } export type AIFlowConfig = @@ -309,6 +314,8 @@ function buildFlowSpecificUserMessage(config: AIFlowConfig): AIFlowUserMessage { tableName: config.tableName, issue: config.issue, tableDetails: config.tableDetails, + diagnosticDetails: config.diagnosticDetails, + issueGuidance: config.issueGuidance, monitoringDocs: config.monitoringDocs, trendSamples: config.trendSamples, }), diff --git a/src/utils/ai/prompts.test.ts b/src/utils/ai/prompts.test.ts new file mode 100644 index 000000000..545e65330 --- /dev/null +++ b/src/utils/ai/prompts.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest" + +import { getHealthIssuePrompt } from "./prompts" + +describe("getHealthIssuePrompt", () => { + it("should preserve unsafe LONG trend samples without locale separators", () => { + // Given + const unsafeLong = BigInt("9007199254740993") + + // When + const prompt = getHealthIssuePrompt({ + tableName: "trades_lv", + issue: { + id: "Y1", + field: "transactionLag", + message: "Transaction lag increasing", + }, + tableDetails: "{}", + monitoringDocs: "Documentation unavailable", + trendSamples: [{ value: unsafeLong, timestamp: Date.UTC(2026, 7, 26) }], + }) + + // Then the digits reach the model unseparated, in any locale + expect(prompt).toContain("9007199254740993") + }) + + it("includes kind-specific diagnostics and issue guidance when provided", () => { + const prompt = getHealthIssuePrompt({ + tableName: "trades_lv", + issue: { + id: "R5", + field: "viewStatus", + message: "Live view is invalid", + }, + tableDetails: '{"table_name":"trades_lv"}', + diagnosticDetails: { + source: "live_views()", + data: '{"view_status":"invalid","writer_stall_micros":"6000000"}', + }, + issueGuidance: "Drop and recreate the live view.", + monitoringDocs: "Documentation unavailable", + }) + + expect(prompt).toContain("Kind-specific Details (from live_views())") + expect(prompt).toContain('"writer_stall_micros":"6000000"') + expect(prompt).toContain("Issue-specific Guidance") + expect(prompt).toContain("Drop and recreate the live view.") + }) + + it("omits optional diagnostic sections for regular table issues", () => { + const prompt = getHealthIssuePrompt({ + tableName: "trades", + issue: { + id: "R1", + field: "walStatus", + message: "WAL suspended", + }, + tableDetails: '{"table_name":"trades"}', + monitoringDocs: "Documentation unavailable", + }) + + expect(prompt).not.toContain("Kind-specific Details") + expect(prompt).not.toContain("Issue-specific Guidance") + }) +}) diff --git a/src/utils/ai/prompts.ts b/src/utils/ai/prompts.ts index 00e600a5a..544fa2aee 100644 --- a/src/utils/ai/prompts.ts +++ b/src/utils/ai/prompts.ts @@ -57,9 +57,9 @@ export const getUnifiedPrompt = ( - Use the suggest_query tool to suggest a SQL query to the user. The query will be displayed as an accept/reject suggestion that updates the editor. This is the ONLY way to suggest SQL queries. ` const schemaAccess = grantSchemaAccess - ? `- Use the get_tables tool to retrieve all tables and materialized views in the database instance -- Use the get_table_schema tool to get detailed schema information for a specific table or a materialized view -- Use the get_table_details tool to get detailed information for a specific table or a materialized view. Each property is described in meta functions docs. + ? `- Use the get_tables tool to retrieve all tables, materialized views, views and live views in the database instance +- Use the get_table_schema tool to get detailed schema information for a specific table, materialized view, view or live view +- Use the get_table_details tool to get detailed information for a specific table, materialized view, view or live view. Each property is described in meta functions docs. ` : "" const permsBlock = perms @@ -157,13 +157,13 @@ ${schema} |--------|------|-------------| | column_name | \`data_type\` | Brief description | -3. If this is a table or materialized view (not a view), add a "## Storage Details" section with bullet points about: +3. If this is a table, materialized view or live view (not a regular view), add a "## Storage Details" section with bullet points about: - WAL enablement - Partitioning strategy - Designated timestamp column - Any other storage considerations -For views, skip the Storage Details section.` +For regular views, skip the Storage Details section.` export type HealthIssuePromptData = { tableName: string @@ -174,12 +174,25 @@ export type HealthIssuePromptData = { currentValue?: string } tableDetails: string + diagnosticDetails?: { + source: "materialized_views()" | "live_views()" + data: string + } + issueGuidance?: string monitoringDocs: string - trendSamples?: Array<{ value: number; timestamp: number }> + trendSamples?: Array<{ value: bigint; timestamp: number }> } export const getHealthIssuePrompt = (data: HealthIssuePromptData): string => { - const { tableName, issue, tableDetails, monitoringDocs, trendSamples } = data + const { + tableName, + issue, + tableDetails, + diagnosticDetails, + issueGuidance, + monitoringDocs, + trendSamples, + } = data let trendSection = "" if (trendSamples && trendSamples.length > 0) { @@ -189,10 +202,25 @@ export const getHealthIssuePrompt = (data: HealthIssuePromptData): string => { ### Trend Data (Recent Samples) | Timestamp | Value | |-----------|-------| -${recentSamples.map((s) => `| ${new Date(s.timestamp).toISOString()} | ${s.value.toLocaleString()} |`).join("\n")} +${recentSamples.map((s) => `| ${new Date(s.timestamp).toISOString()} | ${s.value.toString()} |`).join("\n")} ` } + const diagnosticDetailsSection = diagnosticDetails + ? ` + +## Kind-specific Details (from ${diagnosticDetails.source}) +\`\`\`json +${diagnosticDetails.data} +\`\`\`` + : "" + const issueGuidanceSection = issueGuidance + ? ` + +## Issue-specific Guidance +${issueGuidance}` + : "" + return `You are a QuestDB expert assistant helping diagnose and resolve table health issues. A user is viewing the health monitoring panel for their table and has asked for help with a detected issue. @@ -206,7 +234,7 @@ ${issue.currentValue ? `- **Current Value**: ${issue.currentValue}` : ""}${trend ## Table Details (from tables() function) \`\`\`json ${tableDetails} -\`\`\` +\`\`\`${diagnosticDetailsSection}${issueGuidanceSection} ## QuestDB Monitoring Documentation ${monitoringDocs} diff --git a/src/utils/ai/shared.ts b/src/utils/ai/shared.ts index 822849c9b..917d65842 100644 --- a/src/utils/ai/shared.ts +++ b/src/utils/ai/shared.ts @@ -8,6 +8,7 @@ import { } from "../questdbDocsRetrieval" import { jsonrepair } from "jsonrepair" import type { NotebookFreshness } from "../notebooks/notebookFreshness" +import { stringifyWithBigInts } from "../questdb/serialize" export class RefusalError extends Error { constructor(message: string) { @@ -116,19 +117,18 @@ export const executeTool = async ( if (result.length > MAX_TABLES) { const truncated = result.slice(0, MAX_TABLES) return { - content: JSON.stringify( + content: stringifyWithBigInts( { tables: truncated, total_count: result.length, truncated: true, message: `Showing ${MAX_TABLES} of ${result.length} tables. Use get_table_schema with a specific table name to get details if you are interested in a specific table.`, }, - null, 2, ), } } - return { content: JSON.stringify(result, null, 2) } + return { content: stringifyWithBigInts(result, 2) } } case "get_table_schema": { const tableName = (input as { table_name: string })?.table_name @@ -177,7 +177,7 @@ export const executeTool = async ( const result = await modelToolsClient.getTableDetails(tableName) return { content: result - ? JSON.stringify(result, null, 2) + ? stringifyWithBigInts(result, 2) : "Table details not found", is_error: !result, } diff --git a/src/utils/questdb/client.test.ts b/src/utils/questdb/client.test.ts index a4be22e92..d8f3738e0 100644 --- a/src/utils/questdb/client.test.ts +++ b/src/utils/questdb/client.test.ts @@ -1,7 +1,10 @@ import "../../test/stubBrowserGlobals" import { afterEach, describe, expect, it, vi } from "vitest" import { Client } from "./client" +import { stringifyWithBigInts } from "./serialize" import { Type } from "./types" +import type { QueryRawResult } from "./types" +import type { TableKind } from "./types" const response = (body: Record): Response => ({ @@ -14,6 +17,97 @@ afterEach(() => { vi.unstubAllGlobals() }) +const rawDqlResult = ( + columns: Array<{ name: string; type: string }>, + dataset: Array>, +): QueryRawResult => ({ + columns, + count: dataset.length, + dataset, + error: undefined, + notice: undefined, + query: "catalog()", + timings: { + compiler: 0, + authentication: 0, + count: 0, + execute: 0, + fetch: 0, + }, + type: Type.DQL, +}) + +describe("Client catalog LONG conversion", () => { + it("converts every LONG column to bigint without changing other columns", () => { + const raw = rawDqlResult( + [ + { name: "safe_long", type: "LONG" }, + { name: "max_long", type: "LONG" }, + { name: "min_long", type: "LONG" }, + { name: "nullable_long", type: "LONG" }, + { name: "ratio", type: "DOUBLE" }, + { name: "name", type: "STRING" }, + ], + [ + [ + "42", + "9223372036854775807", + "-9223372036854775807", + null, + 1.5, + "trades", + ], + ], + ) + + const result = Client.transformQueryRawResult>( + raw, + { convertLongsToBigInt: true }, + ) + + expect(result.type).toBe(Type.DQL) + if (result.type !== Type.DQL) throw new Error("expected DQL result") + expect(result.data[0]).toEqual({ + safe_long: BigInt(42), + max_long: BigInt("9223372036854775807"), + min_long: BigInt("-9223372036854775807"), + nullable_long: null, + ratio: 1.5, + name: "trades", + }) + }) + + it("leaves regular query LONG values in their existing wire form", () => { + const raw = rawDqlResult( + [{ name: "value", type: "LONG" }], + [["9007199254740993"]], + ) + + const result = Client.transformQueryRawResult>(raw) + + expect(result.type).toBe(Type.DQL) + if (result.type !== Type.DQL) throw new Error("expected DQL result") + expect(result.data[0].value).toBe("9007199254740993") + }) + + it("rejects a LONG number that has already lost integer precision", () => { + const raw = rawDqlResult( + [{ name: "value", type: "LONG" }], + [[Number("9007199254740993")]], + ) + + expect(() => + Client.transformQueryRawResult(raw, { convertLongsToBigInt: true }), + ).toThrow("Invalid LONG value for column value") + }) + + it("serializes bigint as a decimal string at a JSON boundary", () => { + expect(stringifyWithBigInts({ value: BigInt("9223372036854775807") })).toBe( + '{"value":"9223372036854775807"}', + ) + }) +}) + describe("Client queryRaw NOTICE timings", () => { it("adds fetch timing when the notice carries server timings", async () => { // Given a notice response with the regular query timing fields @@ -62,3 +156,95 @@ describe("Client queryRaw NOTICE timings", () => { expect(result).not.toHaveProperty("timings") }) }) + +describe("Client catalog method wiring", () => { + const catalogResponse = ( + tableName: string, + rowCount: string | null, + ): Response => + response({ + columns: [ + { name: "table_name", type: "STRING" }, + { name: "table_row_count", type: "LONG" }, + ], + count: 1, + dataset: [[tableName, rowCount]], + timings: { compiler: 0, authentication: 0, count: 0, execute: 0 }, + }) + + it("returns showTables catalog LONGs as bigint", async () => { + // Given a tables() response whose LONG arrives as a quoted decimal string + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(catalogResponse("trades", "9007199254740993")), + ) + + // When the schema catalog is listed + const result = await new Client().showTables() + + // Then the row count keeps its full 64-bit precision + expect(result.type).toBe(Type.DQL) + if (result.type !== Type.DQL) throw new Error("expected DQL result") + expect(result.data[0].table_row_count).toBe(BigInt("9007199254740993")) + }) + + it("returns getTableDetails catalog LONGs as bigint", async () => { + // Given a single-table tables() response + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(catalogResponse("trades", "9007199254740993")), + ) + + // When one table's details are fetched + const result = await new Client().getTableDetails("trades") + + // Then the row count keeps its full 64-bit precision + expect(result.type).toBe(Type.DQL) + if (result.type !== Type.DQL) throw new Error("expected DQL result") + expect(result.data[0].table_row_count).toBe(BigInt("9007199254740993")) + }) + + it("escapes single quotes in the table name it filters on", async () => { + // Given a table whose name carries a single quote + const fetchMock = vi.fn<[string], Promise>(() => + Promise.resolve(catalogResponse("o'brien", "1")), + ) + vi.stubGlobal("fetch", fetchMock) + + // When its details are fetched + await new Client().getTableDetails("o'brien") + + // Then the quote is doubled so the predicate stays a single string literal + expect(decodeURIComponent(fetchMock.mock.calls[0][0])).toContain( + "tables() where table_name = 'o''brien';", + ) + }) +}) + +describe("Client showDDL kind routing", () => { + it("sends the kind-specific SHOW CREATE statement for every table kind", async () => { + // Given a client whose requests are captured + const fetchMock = vi.fn<[string], Promise>(() => + Promise.resolve(response({ notice: "ok" })), + ) + vi.stubGlobal("fetch", fetchMock) + const client = new Client() + const kinds: TableKind[] = ["table", "matview", "view", "liveview"] + + // When DDL is requested for each kind + for (const kind of kinds) { + await client.showDDL("my_target", kind) + } + + // Then each kind maps to its own SHOW CREATE statement + const sentQueries = fetchMock.mock.calls.map(([url]) => + decodeURIComponent(url), + ) + expect(sentQueries[0]).toContain("SHOW CREATE TABLE 'my_target';") + expect(sentQueries[1]).toContain( + "SHOW CREATE MATERIALIZED VIEW 'my_target';", + ) + expect(sentQueries[2]).toContain("SHOW CREATE VIEW 'my_target';") + expect(sentQueries[3]).toContain("SHOW CREATE LIVE VIEW 'my_target';") + }) +}) diff --git a/src/utils/questdb/client.ts b/src/utils/questdb/client.ts index 165bcf05a..ffc517e4a 100644 --- a/src/utils/questdb/client.ts +++ b/src/utils/questdb/client.ts @@ -12,7 +12,6 @@ import { Table, Column, Options, - RawData, RawResult, Release, NewsItem, @@ -25,6 +24,9 @@ import { Permission, SymbolColumnDetails, View, + LiveView, + MaterializedView, + TableKind, ValidateQueryResult, ValidateQuerySuccessResult, ValidateQueryErrorResult, @@ -33,6 +35,22 @@ import { ssoAuthState } from "../../modules/OAuth2/ssoAuthState" export type QueryId = number +export const escapeSqlLiteral = (value: string) => value.replace(/'/g, "''") + +export const buildDDLQuery = (name: string, kind: TableKind): string => { + const escapedName = escapeSqlLiteral(name) + switch (kind) { + case "table": + return `SHOW CREATE TABLE '${escapedName}';` + case "matview": + return `SHOW CREATE MATERIALIZED VIEW '${escapedName}';` + case "view": + return `SHOW CREATE VIEW '${escapedName}';` + case "liveview": + return `SHOW CREATE LIVE VIEW '${escapedName}';` + } +} + export class Client { private _controllers = new Map() private _nextQueryId: QueryId = 1 @@ -121,20 +139,36 @@ export class Client { static transformQueryRawResult = >( result: QueryRawResult, + options?: { convertLongsToBigInt?: boolean }, ): QueryResult => { if (result.type === Type.DQL) { const { columns, count, dataset, timings } = result - const parsed = dataset.map( - (row) => - row.reduce( - (acc: RawData, val: Value, idx) => ({ - ...acc, - [columns[idx].name]: val, - }), - {}, - ) as RawData, - ) as unknown as T[] + const parsed = dataset.map((row) => + row.reduce>((acc, val: Value | null, idx) => { + const column = columns[idx] + let value: unknown = val + + if ( + options?.convertLongsToBigInt && + column.type === "LONG" && + val !== null + ) { + if (typeof val === "string" && /^-?\d+$/.test(val)) { + value = BigInt(val) + } else if (typeof val === "number" && Number.isSafeInteger(val)) { + value = BigInt(val) + } else { + throw new TypeError( + `Invalid LONG value for column ${column.name}: ${String(val)}`, + ) + } + } + + acc[column.name] = value + return acc + }, {}), + ) as T[] return { columns, @@ -158,6 +192,21 @@ export class Client { return Client.transformQueryRawResult(result) } + /** + * QuestDB returns LONG columns as decimal strings for console requests so + * their full 64-bit precision survives JSON. Catalog consumers opt into + * bigint conversion here; regular query results intentionally stay as-is. + */ + async queryCatalog>( + query: string, + options?: Options, + ): Promise> { + const result = await this.queryRaw(query, options) + return Client.transformQueryRawResult(result, { + convertLongsToBigInt: true, + }) + } + private removeController(queryId: QueryId) { this._controllers.delete(queryId) if (this._activeQueryId === queryId) { @@ -417,7 +466,7 @@ export class Client { } async showTables(): Promise> { - const response = await this.query
("tables();") + const response = await this.queryCatalog
("tables();") if (response.type === Type.DQL) { return { @@ -465,23 +514,28 @@ export class Client { } async getTableDetails(table: string): Promise> { - return await this.query
(`tables() where table_name = '${table}';`) - } - - async showMatViewDDL(table: string): Promise> { - return this.queryDDL(`SHOW CREATE MATERIALIZED VIEW '${table}';`) + return await this.queryCatalog
( + `tables() where table_name = '${escapeSqlLiteral(table)}';`, + ) } - async showViewDDL(viewName: string): Promise> { - return this.queryDDL(`SHOW CREATE VIEW '${viewName}';`) + async showMaterializedViews(): Promise> { + return await this.queryCatalog("materialized_views()") } async showViews(): Promise> { return await this.query("views();") } - async showTableDDL(table: string): Promise> { - return this.queryDDL(`SHOW CREATE TABLE '${table}';`) + async showLiveViews(): Promise> { + return await this.queryCatalog("live_views();") + } + + async showDDL( + name: string, + kind: TableKind, + ): Promise> { + return this.queryDDL(buildDDLQuery(name, kind)) } private async queryDDL(sql: string): Promise> { diff --git a/src/utils/questdb/index.ts b/src/utils/questdb/index.ts index ac307d424..f64356761 100644 --- a/src/utils/questdb/index.ts +++ b/src/utils/questdb/index.ts @@ -1,3 +1,4 @@ export * from "./types" export * from "./client" export * from "./queryExecutionManager" +export * from "./serialize" diff --git a/src/utils/questdb/serialize.ts b/src/utils/questdb/serialize.ts new file mode 100644 index 000000000..389774174 --- /dev/null +++ b/src/utils/questdb/serialize.ts @@ -0,0 +1,12 @@ +/** JSON has no bigint representation, so decimal strings are used only when + * catalog data crosses back into a JSON/text boundary. */ +export const stringifyWithBigInts = ( + value: unknown, + space?: string | number, +): string => + JSON.stringify( + value, + (_key, item: unknown) => + typeof item === "bigint" ? item.toString() : item, + space, + ) diff --git a/src/utils/questdb/types.test.ts b/src/utils/questdb/types.test.ts new file mode 100644 index 000000000..3e4440d8e --- /dev/null +++ b/src/utils/questdb/types.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest" +import { getTableKind, Table } from "./types" + +const makeTable = (overrides: Partial
): Table => + ({ table_name: "t", ...overrides }) as Table + +describe("getTableKind", () => { + it("should return table for table_type T", () => { + // Given + const table = makeTable({ table_type: "T" }) + + // When / Then + expect(getTableKind(table)).toBe("table") + }) + + it("should return table when table_type is missing on older servers", () => { + // Given + const table = makeTable({ table_type: undefined }) + + // When / Then + expect(getTableKind(table)).toBe("table") + }) + + it("should return matview for table_type M", () => { + // Given + const table = makeTable({ table_type: "M" }) + + // When / Then + expect(getTableKind(table)).toBe("matview") + }) + + it("should return matview for the legacy matView flag", () => { + // Given + const table = makeTable({ matView: true, table_type: undefined }) + + // When / Then + expect(getTableKind(table)).toBe("matview") + }) + + it("should return view for table_type V", () => { + // Given + const table = makeTable({ table_type: "V" }) + + // When / Then + expect(getTableKind(table)).toBe("view") + }) + + it("should return liveview for table_type L", () => { + // Given + const table = makeTable({ table_type: "L" }) + + // When / Then + expect(getTableKind(table)).toBe("liveview") + }) +}) diff --git a/src/utils/questdb/types.ts b/src/utils/questdb/types.ts index 326e798b1..15ed53aff 100644 --- a/src/utils/questdb/types.ts +++ b/src/utils/questdb/types.ts @@ -44,7 +44,7 @@ export type Timings = { export type Explain = { jitCompiled: boolean } -export type DatasetType = Array +export type DatasetType = Array export type RawDqlResult = { columns: ColumnDefinition[] @@ -166,16 +166,30 @@ export type ValidateQueryResult = export type PartitionBy = "HOUR" | "DAY" | "WEEK" | "MONTH" | "YEAR" | "NONE" -export type TableType = "T" | "M" | "V" // Table | MaterializedView | View +export type TableType = "T" | "M" | "V" | "L" // Table | MaterializedView | View | LiveView -export type TableKind = "table" | "matview" | "view" +export type TableKind = "table" | "matview" | "view" | "liveview" export const getTableKind = (table: Table): TableKind => { if (table.matView || table.table_type === "M") return "matview" if (table.table_type === "V") return "view" + if (table.table_type === "L") return "liveview" return "table" } +export const getTableKindLabel = (kind: TableKind): string => { + switch (kind) { + case "matview": + return "Materialized view" + case "view": + return "View" + case "liveview": + return "Live view" + default: + return "Table" + } +} + export type Table = { id: number table_name: string @@ -189,32 +203,42 @@ export type Table = { table_type?: TableType // Optional for backward compatibility with older servers directoryName: string maxUncommittedRows: number - o3MaxLag: number + o3MaxLag: bigint table_suspended: boolean - table_row_count: number | null + table_row_count: bigint | null table_last_write_timestamp: string | null table_max_timestamp: string | null - table_txn: number | null + table_txn: bigint | null table_memory_pressure_level: number | null - wal_pending_row_count: number | null - wal_txn: number | null - wal_tx_count: number | null + wal_pending_row_count: bigint | null + wal_txn: bigint | null + wal_tx_count: bigint | null wal_max_timestamp: string | null - wal_dedup_row_count_since_start: number | null - table_write_amp_count: number | null + wal_dedup_row_count_since_start: bigint | null + table_write_amp_count: bigint | null table_write_amp_p50: number | null table_write_amp_p90: number | null table_write_amp_p99: number | null table_write_amp_max: number | null - table_merge_rate_count: number | null - table_merge_rate_p50: number | null - table_merge_rate_p90: number | null - table_merge_rate_p99: number | null - table_merge_rate_max: number | null - wal_tx_size_p50: number | null - wal_tx_size_p90: number | null - wal_tx_size_p99: number | null - wal_tx_size_max: number | null + table_merge_rate_count: bigint | null + table_merge_rate_p50: bigint | null + table_merge_rate_p90: bigint | null + table_merge_rate_p99: bigint | null + table_merge_rate_max: bigint | null + wal_tx_size_p50: bigint | null + wal_tx_size_p90: bigint | null + wal_tx_size_p99: bigint | null + wal_tx_size_max: bigint | null +} + +export type StoragePolicy = { + table_dir_name: string + to_parquet: string + to_remote: string + drop_local: string + drop_remote: string + status: string + last_updated: string } export type View = { @@ -271,8 +295,8 @@ export type MaterializedView = { invalidation_reason: string | null view_status: "valid" | "refreshing" | "invalid" refresh_period_hi: string | null - refresh_base_table_txn: number - base_table_txn: number + refresh_base_table_txn: bigint + base_table_txn: bigint refresh_limit: number refresh_limit_unit: string | null timer_time_zone: string | null @@ -285,6 +309,36 @@ export type MaterializedView = { period_delay_unit: string | null } +export type LiveView = { + view_name: string + base_table_name: string | null + view_sql: string | null + view_table_dir_name: string + view_status: + | "creating" + | "active" + | "seeding" + | "invalid" + | "dropping" + | "version_unsupported" + | "state_unreadable" + invalidation_reason: string | null + flush_every_interval: bigint | null + flush_every_interval_unit: string | null + in_memory_interval: bigint | null + in_memory_interval_unit: string | null + view_lower_bound_timestamp: string | null + lag_seqtxn: bigint | null + lag_micros: bigint | null + writer_stall_micros: bigint | null + in_mem_rows: bigint | null + in_mem_bytes: bigint | null + below_lower_bound_count: bigint | null + o3_rejected_count: bigint | null + last_processed_seqtxn: bigint | null + seed_target_seqtxn: bigint | null +} + export type Column = { column: string indexed: boolean diff --git a/src/utils/tools/dispatch.test.ts b/src/utils/tools/dispatch.test.ts new file mode 100644 index 000000000..31522794f --- /dev/null +++ b/src/utils/tools/dispatch.test.ts @@ -0,0 +1,81 @@ +import "../../test/stubBrowserGlobals" +import { describe, expect, it } from "vitest" +import { dispatchTool } from "./dispatch" +import { executeTool } from "../ai/shared" +import type { ModelToolsClient, StatusCallback } from "../ai/aiAssistant" +import type { Table } from "../questdb/types" + +const UNSAFE_ROW_COUNT = BigInt("9007199254740993") + +const tableWithBigIntCounters = (): Table => + ({ + table_name: "trades", + table_row_count: UNSAFE_ROW_COUNT, + o3MaxLag: BigInt(300_000), + table_txn: BigInt(42), + }) as Table + +const clientReturning = (table: Table | null): ModelToolsClient => + ({ + getTableDetails: () => Promise.resolve(table), + }) as unknown as ModelToolsClient + +const ignoreStatus: StatusCallback = () => {} + +const toolSurfaces = [ + { + name: "dispatchTool", + run: (client: ModelToolsClient) => + dispatchTool( + "get_table_details", + { table_name: "trades" }, + client, + ignoreStatus, + ), + }, + { + name: "executeTool", + run: (client: ModelToolsClient) => + executeTool( + "get_table_details", + { table_name: "trades" }, + client, + ignoreStatus, + ), + }, +] + +for (const surface of toolSurfaces) { + describe(`${surface.name} get_table_details`, () => { + it("should serialize bigint catalog counters as decimal strings", async () => { + // Given a table whose catalog LONGs arrived as bigint + const client = clientReturning(tableWithBigIntCounters()) + + // When the model asks for its details + const result = await surface.run(client) + + // Then the tool answers with JSON that keeps every digit + expect(result.is_error).toBeFalsy() + expect(JSON.parse(result.content)).toMatchObject({ + table_name: "trades", + table_row_count: "9007199254740993", + o3MaxLag: "300000", + table_txn: "42", + }) + }) + + it("should report an error when the table has no details", async () => { + // Given a table the catalog does not know + const client = clientReturning(null) + + // When the model asks for its details + const result = await surface.run(client) + + // Then the tool reports the miss instead of an empty result + expect(result).toEqual({ + content: "Table details not found", + is_error: true, + }) + }) + }) +} diff --git a/src/utils/tools/dispatch.ts b/src/utils/tools/dispatch.ts index d6be52076..353da3465 100644 --- a/src/utils/tools/dispatch.ts +++ b/src/utils/tools/dispatch.ts @@ -27,6 +27,7 @@ import { type RunCellGate, } from "./permissions" import type { ValidateQueryResult } from "../questdb/types" +import { stringifyWithBigInts } from "../questdb/serialize" import { categoryFor, mutatesNotebook, @@ -331,19 +332,18 @@ export const dispatchTool = async ( if (result.length > MAX_TABLES) { const truncated = result.slice(0, MAX_TABLES) return { - content: JSON.stringify( + content: stringifyWithBigInts( { tables: truncated, total_count: result.length, truncated: true, message: `Showing ${MAX_TABLES} of ${result.length} tables. Use get_table_schema with a specific table name to get details if you are interested in a specific table.`, }, - null, 2, ), } } - return { content: JSON.stringify(result, null, 2) } + return { content: stringifyWithBigInts(result, 2) } } case "get_table_schema": { const tableName = (input as { table_name: string })?.table_name @@ -392,7 +392,7 @@ export const dispatchTool = async ( const result = await modelToolsClient.getTableDetails(tableName) return { content: result - ? JSON.stringify(result, null, 2) + ? stringifyWithBigInts(result, 2) : "Table details not found", is_error: !result, }