From 3a34293c10419231784e86ed7826e5d13586e002 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 21:10:37 +0000 Subject: [PATCH 1/4] feat: add endpoint describe panel (#97) Add an "Endpoint overview" panel that helps to understand a SPARQL endpoint while writing queries. It is opened with the new "Describe endpoint" button in the control bar (or F8) and docks next to the editor. It can be pinned, collapsed to a thin rail and resized, and on small screens it opens as a bottom sheet. - Reads the SPARQL Service Description (endpoint without ?query) and VoID (/.well-known/void or embedded in the SD) and shows features, extension functions, named graphs, dataset statistics and class/property partitions. Missing sources are reported, not errors. - Bounded overview queries for all categories of the issue: named graphs (paginated), triple counts, namespaces, classes, properties, SKOS schemes, SHACL/ShEx shapes, sample instances, hubs, languages, links and time/geo (incl. a bounding box computed from WKT samples). Queries only run on demand; expensive ones are flagged. - Results are cached per endpoint in their own storage key, so they survive new queries, tab switches and reloads, without risking the main configuration when the storage quota is exceeded. - Clicking an IRI inserts it (prefixed, adding the PREFIX) into the query; describe queries can be opened in a new tab. - Queries and categories are configurable via `endpointDescribe`. - Tab gets `runBackgroundQuery` / `getRequestInit`, and yasqe's `executeQuery` a `skipGraphArgs` option. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DEbQuDzCkbQKKsARAXTpKu --- .changeset/endpoint-describe-panel.md | 10 + docs/developer-guide.md | 79 ++ docs/user-guide.md | 46 + package-lock.json | 1 + packages/yasgui/package.json | 1 + packages/yasgui/src/Tab.ts | 38 + packages/yasgui/src/TabSettingsModal.ts | 22 + packages/yasgui/src/defaults.ts | 6 + .../src/endpointDescribe/DescribeStore.ts | 161 ++++ .../EndpointDescribePanel.scss | 435 +++++++++ .../endpointDescribe/EndpointDescribePanel.ts | 832 ++++++++++++++++++ .../src/endpointDescribe/describeQueries.ts | 452 ++++++++++ .../src/endpointDescribe/metadataSources.ts | 284 ++++++ packages/yasgui/src/index.ts | 19 +- packages/yasqe/src/sparql.ts | 19 +- test/endpoint-describe-browser.ts | 197 +++++ test/run.ts | 1 + test/unit/endpoint-describe-test.ts | 311 +++++++ test/unit/yasqe-background-query-test.ts | 44 + 19 files changed, 2955 insertions(+), 3 deletions(-) create mode 100644 .changeset/endpoint-describe-panel.md create mode 100644 packages/yasgui/src/endpointDescribe/DescribeStore.ts create mode 100644 packages/yasgui/src/endpointDescribe/EndpointDescribePanel.scss create mode 100644 packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts create mode 100644 packages/yasgui/src/endpointDescribe/describeQueries.ts create mode 100644 packages/yasgui/src/endpointDescribe/metadataSources.ts create mode 100644 test/endpoint-describe-browser.ts create mode 100644 test/unit/endpoint-describe-test.ts diff --git a/.changeset/endpoint-describe-panel.md b/.changeset/endpoint-describe-panel.md new file mode 100644 index 00000000..86036e68 --- /dev/null +++ b/.changeset/endpoint-describe-panel.md @@ -0,0 +1,10 @@ +--- +"@matdata/yasgui": minor +"@matdata/yasqe": minor +--- + +Add an "Endpoint overview" panel to get to know a SPARQL endpoint while writing queries (Describe endpoint button or `F8`). + +The panel shows the endpoint's SPARQL Service Description and VoID description (when published) and offers bounded overview queries for named graphs, triple counts, namespaces, classes, properties, SKOS schemes, shapes, sample instances, hubs, languages, links and time/geo properties. Results are remembered per endpoint, also when other queries are executed or the page is reloaded. The panel docks next to the editor, can be pinned, collapsed and resized, and follows the endpoint of the active tab. Queries and categories can be extended through the new `endpointDescribe` configuration. + +YASQE's `executeQuery` gets a `skipGraphArgs` option to leave out the tab's default/named graph arguments. diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 710a95aa..66f87c59 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -56,6 +56,7 @@ This comprehensive guide covers everything developers need to know to integrate, - [Examples](#examples) - [Security Best Practices](#security-best-practices) - [Endpoint Buttons Configuration](#endpoint-buttons-configuration) + - [Endpoint Describe Configuration](#endpoint-describe-configuration) - [Theme Configuration](#theme-configuration) - [API Reference](#api-reference) - [Yasgui Class](#yasgui-class) @@ -689,6 +690,9 @@ interface Config { // Layout orientation: 'vertical' or 'horizontal' orientation?: 'vertical' | 'horizontal'; // default: 'vertical' + + // "Describe endpoint" panel (see Endpoint Describe Configuration) + endpointDescribe: EndpointDescribeConfig; } ``` @@ -1955,6 +1959,81 @@ You can customize button appearance using CSS variables: } ``` +### Endpoint Describe Configuration + +The **Endpoint overview** panel shows the SPARQL service description and VoID description of the current endpoint, and a set of predefined overview queries (classes, properties, named graphs, languages, links, time and geo). Results are cached per endpoint under their own localStorage key (`_endpointDescribe`), separately from the main configuration. + +```typescript +interface EndpointDescribeConfig { + // Show the "Describe endpoint" button and panel + enabled: boolean; // default: true + + // Replace or extend the default describe queries + queries?: DescribeQuery[] | ((defaults: DescribeQuery[]) => DescribeQuery[]); + + // Replace or extend the default categories + categories?: DescribeCategory[] | ((defaults: DescribeCategory[]) => DescribeCategory[]); + + // Timeout of a single describe query, in milliseconds + timeoutMs: number; // default: 60000 + + // Maximum number of describe queries running at the same time + maxConcurrentQueries: number; // default: 2 + + // Fetch the service description and VoID when the panel opens + fetchMetadata: boolean; // default: true +} + +interface DescribeQuery { + id: string; + category: string; // id of a DescribeCategory + label: string; + description?: string; + // SPARQL SELECT query, or a function that builds it (use ctx.limit and ctx.offset for paginated queries) + query: string | ((ctx: { limit: number; offset: number; getResult: (queryId: string) => any }) => string); + paginated?: boolean; // show "Load more" + pageSize?: number; // default: 25 + expensive?: boolean; // show a "may be slow" hint + dependsOn?: string; // id of a query whose results are needed to build this one +} + +interface DescribeCategory { + id: string; + label: string; + icon: string; // Font Awesome icon class, e.g. "fa-train" +} +``` + +**Example**: add a category with endpoint specific queries: + +```javascript +const yasgui = new Yasgui(document.getElementById("yasgui"), { + endpointDescribe: { + categories: (defaults) => [ + ...defaults, + { id: "railway", label: "Railway infrastructure", icon: "fa-train" }, + ], + queries: (defaults) => [ + ...defaults, + { + id: "operational-points", + category: "railway", + label: "Operational points per country", + query: `PREFIX era: +SELECT ?country (COUNT(?op) AS ?operationalPoints) WHERE { + ?op a era:OperationalPoint ; era:inCountry ?country . +} +GROUP BY ?country +ORDER BY DESC(?operationalPoints) +LIMIT 50`, + }, + ], + }, +}); +``` + +Disable the panel with `endpointDescribe: { enabled: false }`. The panel can also be controlled programmatically through `yasgui.endpointDescribe` (`open()`, `close()`, `toggle()`, `collapse()`, `setPinned(pinned)`, `runQuery(queryId)`). + ### Theme Configuration YASGUI supports both light and dark themes with comprehensive customization options. diff --git a/docs/user-guide.md b/docs/user-guide.md index a81bbdf5..8e7ecd8c 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -33,6 +33,7 @@ - [Endpoint Quick Switch](#endpoint-quick-switch) - [Configuration Import/Export](#configuration-importexport) - [URI Explorer](#uri-explorer) + - [Endpoint Overview](#endpoint-overview) - [Query Tabs](#query-tabs) - [Settings Modal](#settings-modal) - [SPARQL Endpoints Management](#sparql-endpoints-management) @@ -726,6 +727,50 @@ Alternatively, use `Ctrl`+`Shift` and click on the URI to executes a CONSTRUCT q **Example:** Ctrl+clicking on `http://dbpedia.org/resource/European_Union` automatically queries for all triples where the European Union is the subject of. +### Endpoint Overview + +Get to know an unfamiliar SPARQL endpoint while you write your queries. Click the **Describe endpoint** button in the control bar or press `F8` to open the **Endpoint overview** panel next to the editor. + +**Service description and VoID** + +When the panel opens, MatGUI looks for metadata that the endpoint publishes about itself: + +- The [SPARQL Service Description](https://www.w3.org/TR/sparql11-service-description/), returned by the endpoint URL when it is requested without a query. It lists the supported SPARQL features, extension functions, result formats and named graphs. +- A [VoID](https://www.w3.org/TR/void/) dataset description at `/.well-known/void` (or embedded in the service description), with statistics such as the number of triples, classes, vocabularies and class/property partitions. + +Many endpoints publish neither (for example GraphDB has no service description). The panel then simply says so and you can use the overview queries instead. If you run an endpoint yourself, publishing a service description and VoID makes it much easier to explore. + +**Overview queries** + +The panel contains predefined queries, grouped in categories: + +| Category | Queries | +| ----------------------------- | ------------------------------------------------------------------------------------------ | +| Endpoint & dataset overview | Named graphs, triples per named graph, total number of triples, most used namespaces | +| Vocabulary & schema | Classes and properties with usage counts, SKOS concept schemes, SHACL/ShEx shapes | +| Instances | Sample instances of the most used classes, most connected nodes (hubs) | +| Labels, languages & literals | Labels per language, properties with language-tagged literals | +| Interlinking & external links | Referenced hosts (e.g. Wikidata, GeoNames, DBpedia), linking properties, linked entities | +| Time & geo | Temporal properties, geospatial properties, coordinate reference systems, bounding boxes | + +- Queries only run when you click **Run** (▶) or **Run all** for a category, so large endpoints are never queried unexpectedly. Queries marked **may be slow** scan the whole dataset and may time out on large endpoints. +- Every query is limited. Lists such as the named graphs are loaded page by page with **Load more**. +- Describe queries run in the background with the endpoint and authentication settings of the current tab. They do not change your query or the results view. The default and named graphs configured for the tab are not applied, so the whole endpoint is described. +- Click an IRI in the results to insert it into your query. MatGUI uses a prefixed name when it knows the prefix and adds the missing `PREFIX` declaration. +- Click the **Open query in a new tab** button next to a query to open it in a new tab, to adapt it further. + +**Remembered results** + +Results are remembered per endpoint, also when you execute other queries, switch tabs or reload the page. The panel always follows the endpoint of the active tab. Use the **Run again** button to refresh a result; the time it was fetched is shown next to it. + +**Keeping the panel out of the way** + +- **Move out of the way** (») collapses the panel to a thin bar at the side of the screen. Click the bar to expand it again. +- An unpinned panel collapses automatically when you click into the query editor. +- The **pin** button pins the panel: it stays open while you write queries and is restored when you reload the page. +- Drag the left edge of the panel to change its width. +- On small screens the panel opens as a sheet at the bottom of the screen. + ### Query Tabs Manage multiple queries simultaneously with tabs. @@ -1830,6 +1875,7 @@ Master Matgui with these keyboard shortcuts for faster querying. | `F11` | Toggle YASQE (editor) fullscreen | | `F10` | Toggle YASR (results) fullscreen | | `F9` | Switch between YASQE and YASR fullscreen | +| `F8` | Toggle the endpoint overview panel | | `Esc` | Exit fullscreen mode | ### Matgui Tabs diff --git a/package-lock.json b/package-lock.json index 70d47ae1..3467954f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9250,6 +9250,7 @@ "es6-object-assign": "^1.1.0", "jsuri": "^1.3.1", "lodash-es": "^4.17.15", + "n3": "^1.3.5", "sortablejs": "^1.10.2" }, "devDependencies": { diff --git a/packages/yasgui/package.json b/packages/yasgui/package.json index a81f0c66..cb77fee5 100644 --- a/packages/yasgui/package.json +++ b/packages/yasgui/package.json @@ -46,6 +46,7 @@ "es6-object-assign": "^1.1.0", "jsuri": "^1.3.1", "lodash-es": "^4.17.15", + "n3": "^1.3.5", "sortablejs": "^1.10.2", "@matdata/yasgui-graph-plugin": "^1.6.1", "@matdata/yasgui-table-plugin": "^1.3.0" diff --git a/packages/yasgui/src/Tab.ts b/packages/yasgui/src/Tab.ts index edb9950e..599cefed 100644 --- a/packages/yasgui/src/Tab.ts +++ b/packages/yasgui/src/Tab.ts @@ -458,6 +458,13 @@ export class Tab extends EventEmitter { } } + // F8 - Toggle the endpoint describe panel + if (event.key === "F8" && this.yasgui.endpointDescribe) { + event.preventDefault(); + this.yasgui.endpointDescribe.toggle(); + return; + } + // F11 - Toggle Yasqe fullscreen if (event.key === "F11") { event.preventDefault(); @@ -1581,6 +1588,37 @@ export class Tab extends EventEmitter { void this.executeBackgroundQuery(constructQuery); }; + /** + * Execute a SPARQL query against this tab's endpoint without touching the editor or the results view. + * The request configuration and authentication of the tab are used. + */ + public async runBackgroundQuery( + query: string, + options: { accept?: string; signal?: AbortSignal; skipGraphArgs?: boolean } = {}, + ): Promise { + if (!this.yasqe) throw new Error("No yasqe editor initialized"); + const tokenValid = await this.ensureOAuth2TokenValid(); + if (!tokenValid) throw new Error("OAuth 2.0 authentication failed"); + return Yasqe.Sparql.executeQuery(this.yasqe, undefined, { + customQuery: query, + customAccept: options.accept, + signal: options.signal, + silent: true, + skipGraphArgs: options.skipGraphArgs, + }); + } + + /** + * Request headers (including authentication) and credentials mode used for requests to this tab's endpoint. + */ + public async getRequestInit(): Promise<{ headers: { [key: string]: string }; withCredentials: boolean } | undefined> { + if (!this.yasqe) return undefined; + await this.ensureOAuth2TokenValid(); + const ajaxConfig = Yasqe.Sparql.getAjaxConfig(this.yasqe); + if (!ajaxConfig) return undefined; + return { headers: { ...(ajaxConfig.headers || {}) }, withCredentials: ajaxConfig.withCredentials }; + } + private async executeBackgroundQuery(query: string) { if (!this.yasqe || !this.yasr) return; diff --git a/packages/yasgui/src/TabSettingsModal.ts b/packages/yasgui/src/TabSettingsModal.ts index bd1fa79a..f4986ab8 100644 --- a/packages/yasgui/src/TabSettingsModal.ts +++ b/packages/yasgui/src/TabSettingsModal.ts @@ -99,6 +99,17 @@ export default class TabSettingsModal { controlBarEl.appendChild(this.mapButton); this.mapButton.onclick = (event: MouseEvent) => this.openMapWidget(event); + // Describe endpoint button + if (this.tab.yasgui.config.endpointDescribe?.enabled) { + const describeButton = document.createElement("button"); + describeButton.setAttribute("aria-label", "Describe endpoint"); + describeButton.title = "Describe endpoint (F8)"; + describeButton.innerHTML = ''; + addClass(describeButton, "tabContextButton", "describeEndpointButton", "desktopOnly"); + describeButton.onclick = () => this.tab.yasgui.endpointDescribe?.toggle(); + controlBarEl.appendChild(describeButton); + } + // Hamburger menu button and dropdown (mobile only) const hamburgerContainer = document.createElement("div"); addClass(hamburgerContainer, "hamburgerContainer", "mobileOnly"); @@ -167,6 +178,17 @@ export default class TabSettingsModal { }; this.hamburgerDropdown.appendChild(mapItem); + if (this.tab.yasgui.config.endpointDescribe?.enabled) { + const describeItem = document.createElement("button"); + addClass(describeItem, "hamburgerMenuItem"); + describeItem.innerHTML = 'Describe endpoint'; + describeItem.onclick = () => { + this.tab.yasgui.endpointDescribe?.toggle(); + this.closeHamburgerMenu(); + }; + this.hamburgerDropdown.appendChild(describeItem); + } + // Theme toggle menu item (if enabled) if (this.tab.yasgui.config.showThemeToggle) { const themeItem = document.createElement("button"); diff --git a/packages/yasgui/src/defaults.ts b/packages/yasgui/src/defaults.ts index 71efc0fb..894535e4 100644 --- a/packages/yasgui/src/defaults.ts +++ b/packages/yasgui/src/defaults.ts @@ -31,6 +31,12 @@ export default function initialize(): Config { showThemeToggle: true, orientation: "vertical", showSnippetsBar: true, + endpointDescribe: { + enabled: true, + timeoutMs: 60000, + maxConcurrentQueries: 2, + fetchMetadata: true, + }, endpointButtons: undefined, endpointCatalogueOptions: { getData: () => { diff --git a/packages/yasgui/src/endpointDescribe/DescribeStore.ts b/packages/yasgui/src/endpointDescribe/DescribeStore.ts new file mode 100644 index 00000000..590afc4c --- /dev/null +++ b/packages/yasgui/src/endpointDescribe/DescribeStore.ts @@ -0,0 +1,161 @@ +/** + * Per-endpoint cache of "describe endpoint" results and the UI state of the panel. + * + * The cache is stored under its own storage key (separate from the main Matgui config), + * so that a full localStorage quota never wipes the user's tabs because of cached describe results. + */ +import type { DescribeBinding } from "./describeQueries"; +import type { EndpointMetadata } from "./metadataSources"; + +export interface DescribeResult { + status: "done" | "error"; + vars: string[]; + bindings: DescribeBinding[]; + fetchedAt: number; + durationMs?: number; + error?: string; + /** True when a following page may exist */ + hasMore?: boolean; + /** True when rows were dropped before persisting */ + truncated?: boolean; +} + +export interface EndpointEntry { + lastUsed: number; + metadata?: EndpointMetadata; + results: { [queryId: string]: DescribeResult }; +} + +export interface PanelUiState { + open: boolean; + pinned: boolean; + collapsed: boolean; + width: number; + expandedCategories: string[]; +} + +export interface StoredDescribeState { + ui: PanelUiState; + endpoints: { [endpoint: string]: EndpointEntry }; +} + +export interface StorageAdapter { + get(): StoredDescribeState | undefined; + set(state: StoredDescribeState): void; +} + +export const MAX_ENDPOINTS = 20; +export const MAX_PERSISTED_ROWS = 200; +/** Rough maximum size of the serialized cache (in characters) */ +export const MAX_PERSISTED_SIZE = 1_500_000; + +export function getDefaultUiState(): PanelUiState { + return { open: false, pinned: false, collapsed: false, width: 420, expandedCategories: ["overview", "schema"] }; +} + +export default class DescribeStore { + private state: StoredDescribeState; + constructor(private storage?: StorageAdapter) { + let stored: StoredDescribeState | undefined; + try { + stored = storage?.get(); + } catch { + stored = undefined; + } + this.state = { + ui: { ...getDefaultUiState(), ...(stored?.ui || {}) }, + endpoints: stored?.endpoints && typeof stored.endpoints === "object" ? stored.endpoints : {}, + }; + } + + public getUiState(): PanelUiState { + return this.state.ui; + } + public setUiState(update: Partial) { + this.state.ui = { ...this.state.ui, ...update }; + this.persist(); + } + + public getEntry(endpoint: string): EndpointEntry | undefined { + return this.state.endpoints[endpoint]; + } + private getOrCreateEntry(endpoint: string): EndpointEntry { + let entry = this.state.endpoints[endpoint]; + if (!entry) { + entry = { lastUsed: Date.now(), results: {} }; + this.state.endpoints[endpoint] = entry; + } + entry.lastUsed = Date.now(); + return entry; + } + + public getResult(endpoint: string, queryId: string): DescribeResult | undefined { + return this.state.endpoints[endpoint]?.results[queryId]; + } + public setResult(endpoint: string, queryId: string, result: DescribeResult) { + this.getOrCreateEntry(endpoint).results[queryId] = result; + this.persist(); + } + public getMetadata(endpoint: string): EndpointMetadata | undefined { + return this.state.endpoints[endpoint]?.metadata; + } + public setMetadata(endpoint: string, metadata: EndpointMetadata) { + this.getOrCreateEntry(endpoint).metadata = metadata; + this.persist(); + } + public touch(endpoint: string) { + if (this.state.endpoints[endpoint]) this.state.endpoints[endpoint].lastUsed = Date.now(); + } + public clearEndpoint(endpoint: string) { + delete this.state.endpoints[endpoint]; + this.persist(); + } + public getEndpoints(): string[] { + return Object.keys(this.state.endpoints); + } + + /** + * Returns the state as it will be persisted: at most MAX_ENDPOINTS endpoints (least recently used are dropped), + * at most MAX_PERSISTED_ROWS rows per result, and a total size of roughly MAX_PERSISTED_SIZE. + */ + public getPersistableState(): StoredDescribeState { + const endpoints = Object.keys(this.state.endpoints) + .sort((a, b) => this.state.endpoints[b].lastUsed - this.state.endpoints[a].lastUsed) + .slice(0, MAX_ENDPOINTS); + // Evict from memory as well, so the in-memory cache doesn't grow unbounded + for (const endpoint of Object.keys(this.state.endpoints)) { + if (endpoints.indexOf(endpoint) < 0) delete this.state.endpoints[endpoint]; + } + const persisted: StoredDescribeState = { ui: this.state.ui, endpoints: {} }; + for (const endpoint of endpoints) { + const entry = this.state.endpoints[endpoint]; + const results: EndpointEntry["results"] = {}; + for (const queryId in entry.results) { + const result = entry.results[queryId]; + results[queryId] = + result.bindings.length > MAX_PERSISTED_ROWS + ? { ...result, bindings: result.bindings.slice(0, MAX_PERSISTED_ROWS), truncated: true, hasMore: false } + : result; + } + persisted.endpoints[endpoint] = { ...entry, results }; + } + // Drop least recently used endpoints until the serialized cache is small enough + let size = JSON.stringify(persisted).length; + while (size > MAX_PERSISTED_SIZE && endpoints.length > 1) { + const evicted = endpoints.pop()!; + delete persisted.endpoints[evicted]; + size = JSON.stringify(persisted).length; + } + return persisted; + } + + private persist() { + if (!this.storage) return; + try { + this.storage.set(this.getPersistableState()); + } catch (e) { + // Storage full or unavailable: the cache simply isn't persisted + console.warn("Could not persist endpoint describe cache", e); + } + } +} diff --git a/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.scss b/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.scss new file mode 100644 index 00000000..23a0724a --- /dev/null +++ b/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.scss @@ -0,0 +1,435 @@ +.yasgui { + // Row that holds the tab panels and the (docked) endpoint describe panel + > div.yasgui-main:not(.tabsList) { + flex-direction: row; + + > .yasgui-tabPanels { + flex: 1; + min-width: 0; + display: flex; + flex-direction: column; + } + } + + .yasgui-describe { + display: none; + position: relative; + flex: 0 0 auto; + min-height: 0; + border-left: 1px solid var(--yasgui-border-color, #e0e0e0); + background: var(--yasgui-bg-primary, white); + color: var(--yasgui-text-primary, #000); + + &.open { + display: flex; + } + + &.resizing { + user-select: none; + cursor: col-resize; + } + + &__resizer { + position: absolute; + top: 0; + bottom: 0; + left: -3px; + width: 6px; + cursor: col-resize; + z-index: 2; + + &:hover { + background: var(--yasgui-accent-color, #337ab7); + opacity: 0.4; + } + } + + &__rail { + display: none; + width: 32px; + padding: 8px 0; + border: none; + background: var(--yasgui-bg-secondary, #f7f7f7); + color: var(--yasgui-text-secondary, #666); + cursor: pointer; + flex-direction: column; + align-items: center; + gap: 8px; + font-size: 13px; + + span { + writing-mode: vertical-rl; + white-space: nowrap; + } + + &:hover { + color: var(--yasgui-text-primary, #000); + } + } + + &.collapsed { + .yasgui-describe__rail { + display: flex; + } + + .yasgui-describe__drawer, + .yasgui-describe__resizer { + display: none; + } + } + + &__drawer { + display: flex; + flex-direction: column; + min-height: 0; + width: 420px; + max-width: 70vw; + height: 100%; + } + + &__header { + border-bottom: 1px solid var(--yasgui-border-color, #e0e0e0); + padding: 8px 12px; + } + + &__header-top { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + } + + &__title { + margin: 0; + font-size: 15px; + font-weight: 600; + } + + &__header-buttons, + &__query-actions { + display: flex; + align-items: center; + gap: 2px; + flex-shrink: 0; + } + + &__icon-button { + background: none; + border: none; + padding: 4px 6px; + border-radius: 4px; + color: var(--yasgui-text-secondary, #666); + cursor: pointer; + + &:hover:not(:disabled) { + color: var(--yasgui-text-primary, #000); + background: var(--yasgui-bg-secondary, #f0f0f0); + } + + &:disabled { + opacity: 0.4; + cursor: default; + } + } + + &.pinned .yasgui-describe__pin { + color: var(--yasgui-accent-color, #337ab7); + } + + &__endpoint { + margin-top: 4px; + font-size: 12px; + color: var(--yasgui-text-secondary, #666); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + &__body { + flex: 1; + min-height: 0; + overflow-y: auto; + padding: 8px 12px 16px; + font-size: 13px; + } + + &__card { + border: 1px solid var(--yasgui-border-color, #e0e0e0); + border-radius: 6px; + padding: 8px 10px; + margin-bottom: 10px; + } + + &__card-heading { + display: flex; + justify-content: space-between; + align-items: baseline; + gap: 8px; + margin-bottom: 6px; + } + + &__status-list { + display: flex; + flex-wrap: wrap; + gap: 4px; + margin-bottom: 6px; + } + + &__badge { + font-size: 11px; + padding: 1px 6px; + border-radius: 10px; + border: 1px solid var(--yasgui-border-color, #e0e0e0); + + &.ok { + border-color: #2e7d32; + color: #2e7d32; + } + + &.unavailable { + color: var(--yasgui-text-secondary, #666); + } + } + + &__facts { + display: grid; + grid-template-columns: max-content 1fr; + gap: 2px 8px; + margin: 4px 0; + + dt { + color: var(--yasgui-text-secondary, #666); + } + + dd { + margin: 0; + min-width: 0; + overflow-wrap: anywhere; + } + } + + &__dataset { + border-top: 1px dashed var(--yasgui-border-color, #e0e0e0); + padding-top: 6px; + margin-top: 6px; + } + + &__dataset-title { + overflow-wrap: anywhere; + margin-bottom: 4px; + } + + &__partition > summary { + cursor: pointer; + margin: 4px 0; + } + + &__category { + border-top: 1px solid var(--yasgui-border-color, #e0e0e0); + padding: 4px 0; + } + + &__category-summary { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + padding: 6px 0; + cursor: pointer; + font-weight: 600; + list-style: none; + + &::-webkit-details-marker { + display: none; + } + + &::before { + content: "▸"; + margin-right: 4px; + color: var(--yasgui-text-secondary, #666); + } + } + + &__category[open] > .yasgui-describe__category-summary::before { + content: "▾"; + } + + &__category-label { + flex: 1; + } + + &__run-all, + &__button { + font-size: 12px; + font-weight: normal; + padding: 2px 8px; + border-radius: 4px; + border: 1px solid var(--yasgui-border-color, #ccc); + background: var(--yasgui-bg-primary, white); + color: var(--yasgui-text-primary, #000); + cursor: pointer; + + &:hover { + background: var(--yasgui-bg-secondary, #f0f0f0); + } + } + + &__button { + margin-top: 6px; + } + + &__query { + padding: 6px 0 8px 12px; + + & + & { + border-top: 1px dotted var(--yasgui-border-color, #e0e0e0); + } + } + + &__query-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + } + + &__query-label { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 6px; + font-weight: 500; + } + + &__slow { + font-size: 10px; + font-weight: normal; + padding: 0 5px; + border-radius: 8px; + background: #fff3cd; + color: #7a5b00; + } + + &__query-description, + &__query-meta, + &__muted { + color: var(--yasgui-text-secondary, #666); + font-size: 12px; + } + + &__query-meta { + margin-top: 4px; + } + + &__loading { + margin-top: 4px; + font-style: italic; + color: var(--yasgui-text-secondary, #666); + } + + &__error { + margin-top: 4px; + padding: 4px 6px; + border-radius: 4px; + background: rgba(211, 47, 47, 0.08); + color: var(--yasgui-error-color, #c62828); + font-size: 12px; + white-space: pre-wrap; + overflow-wrap: anywhere; + max-height: 120px; + overflow: auto; + } + + &__table-wrapper { + margin-top: 6px; + max-height: 320px; + overflow: auto; + border: 1px solid var(--yasgui-border-color, #e0e0e0); + border-radius: 4px; + } + + &__table { + width: 100%; + border-collapse: collapse; + font-size: 12px; + + th { + position: sticky; + top: 0; + background: var(--yasgui-bg-secondary, #f7f7f7); + text-align: left; + font-weight: 600; + } + + th, + td { + padding: 3px 6px; + border-bottom: 1px solid var(--yasgui-border-color, #eee); + vertical-align: top; + overflow-wrap: anywhere; + } + } + + &__iri { + background: none; + border: none; + padding: 0; + font: inherit; + text-align: left; + color: var(--yasgui-link-color, #337ab7); + cursor: pointer; + overflow-wrap: anywhere; + + &:hover { + text-decoration: underline; + } + } + + &__lang { + margin-left: 2px; + color: var(--yasgui-text-secondary, #666); + } + + // On small screens the panel becomes a bottom sheet + @media (max-width: 768px) { + &.open { + position: fixed; + left: 0; + right: 0; + bottom: 0; + height: 60vh; + z-index: 1001; + border-left: none; + border-top: 1px solid var(--yasgui-border-color, #e0e0e0); + box-shadow: 0 -4px 12px rgba(0, 0, 0, 0.15); + } + + &__drawer { + width: 100% !important; + max-width: none; + } + + &__resizer { + display: none; + } + + &.collapsed { + height: auto; + + .yasgui-describe__rail { + width: 100%; + flex-direction: row; + justify-content: center; + + span { + writing-mode: horizontal-tb; + } + + i { + transform: rotate(90deg); + } + } + } + } + } +} diff --git a/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts b/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts new file mode 100644 index 00000000..f27f407e --- /dev/null +++ b/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts @@ -0,0 +1,832 @@ +import { addClass, removeClass, Storage as YStorage } from "@matdata/yasgui-utils"; +import { Parser } from "@matdata/yasr"; +import type { Yasgui } from "../"; +import type Tab from "../Tab"; +import { storageNamespace } from "../PersistentConfig"; +import DescribeStore, { DescribeResult, StoredDescribeState } from "./DescribeStore"; +import { + buildQuery, + COMMON_PREFIXES, + DEFAULT_PAGE_SIZE, + defaultCategories, + defaultDescribeQueries, + DescribeBinding, + DescribeCategory, + DescribeQuery, + DescribeTable, +} from "./describeQueries"; +import { EndpointMetadata, fetchEndpointMetadata, VoidDataset } from "./metadataSources"; +import "./EndpointDescribePanel.scss"; + +export interface EndpointDescribeConfig { + /** Show the "Describe endpoint" button and panel */ + enabled: boolean; + /** Replace or extend the default describe queries */ + queries?: DescribeQuery[] | ((defaults: DescribeQuery[]) => DescribeQuery[]); + /** Replace or extend the default categories */ + categories?: DescribeCategory[] | ((defaults: DescribeCategory[]) => DescribeCategory[]); + /** Timeout of a single describe query, in milliseconds */ + timeoutMs: number; + /** Maximum number of describe queries running at the same time */ + maxConcurrentQueries: number; + /** Automatically fetch the SPARQL service description and VoID description when the panel opens */ + fetchMetadata: boolean; +} + +export const defaultEndpointDescribeConfig: EndpointDescribeConfig = { + enabled: true, + timeoutMs: 60000, + maxConcurrentQueries: 2, + fetchMetadata: true, +}; + +const MIN_WIDTH = 280; +const SPARQL_JSON = "application/sparql-results+json,application/sparql-results+xml;q=0.8"; + +interface RunningQuery { + controller: AbortController; + timedOut: boolean; +} + +function el( + tag: K, + className?: string, + text?: string, +): HTMLElementTagNameMap[K] { + const element = document.createElement(tag); + if (className) element.className = className; + if (text !== undefined) element.textContent = text; + return element; +} + +function iconButton(icon: string, title: string, className = ""): HTMLButtonElement { + const button = el("button", `yasgui-describe__icon-button ${className}`.trim()); + button.type = "button"; + button.title = title; + button.setAttribute("aria-label", title); + button.innerHTML = ``; + return button; +} + +export function formatRelativeTime(timestamp: number, now = Date.now()): string { + const seconds = Math.max(0, Math.round((now - timestamp) / 1000)); + if (seconds < 60) return "just now"; + const minutes = Math.round(seconds / 60); + if (minutes < 60) return `${minutes} min ago`; + const hours = Math.round(minutes / 60); + if (hours < 48) return `${hours} h ago`; + return `${Math.round(hours / 24)} days ago`; +} + +function formatNumber(value: number | undefined): string { + return value === undefined ? "–" : value.toLocaleString(); +} + +export default class EndpointDescribePanel { + private yasgui: Yasgui; + private store: DescribeStore; + private config: EndpointDescribeConfig; + private queries: DescribeQuery[]; + private categories: DescribeCategory[]; + + private rootEl: HTMLDivElement; + private resizerEl!: HTMLDivElement; + private railEl!: HTMLButtonElement; + private drawerEl!: HTMLDivElement; + private endpointEl!: HTMLDivElement; + private pinButton!: HTMLButtonElement; + private metadataEl!: HTMLDivElement; + private queryEls = new Map(); + + private endpoint: string | undefined; + private running = new Map(); + private queue: Array<{ key: string; start: () => Promise; cancel: () => void }> = []; + private activeCount = 0; + private metadataController: AbortController | undefined; + private renderedEndpoint: string | undefined | null = null; + + constructor(yasgui: Yasgui) { + this.yasgui = yasgui; + this.config = { ...defaultEndpointDescribeConfig, ...(yasgui.config.endpointDescribe || {}) }; + this.queries = + typeof this.config.queries === "function" + ? this.config.queries(defaultDescribeQueries) + : this.config.queries || defaultDescribeQueries; + this.categories = + typeof this.config.categories === "function" + ? this.config.categories(defaultCategories) + : this.config.categories || defaultCategories; + + const storageId = yasgui.getStorageId("endpointDescribe"); + const storage = new YStorage(storageNamespace); + this.store = new DescribeStore( + storageId + ? { + get: () => storage.get(storageId), + set: (state) => + storage.set(storageId, state, yasgui.config.persistencyExpire, () => { + // Quota exceeded: drop the cache instead of the whole Matgui configuration + storage.remove(storageId); + }), + } + : undefined, + ); + + this.rootEl = el("div", "yasgui-describe"); + this.rootEl.setAttribute("role", "complementary"); + this.rootEl.setAttribute("aria-label", "Endpoint overview"); + this.draw(); + this.registerListeners(); + + const ui = this.store.getUiState(); + // Only a pinned panel is restored on load + if (ui.pinned && ui.open) this.open(); + else this.applyState(false); + } + + public getElement(): HTMLDivElement { + return this.rootEl; + } + + public isOpen(): boolean { + return this.store.getUiState().open; + } + + public open() { + this.store.setUiState({ open: true, collapsed: false }); + this.applyState(); + this.syncEndpoint(); + } + + public close() { + this.store.setUiState({ open: false }); + this.applyState(); + } + + public toggle() { + const ui = this.store.getUiState(); + if (ui.open && !ui.collapsed) this.close(); + else this.open(); + } + + public collapse() { + this.store.setUiState({ collapsed: true }); + this.applyState(); + } + + public setPinned(pinned: boolean) { + this.store.setUiState({ pinned }); + this.applyState(false); + } + + /** + * Run a describe query for the current endpoint. + * @param loadMore fetch the next page and append it to the existing rows + */ + public runQuery(queryId: string, loadMore = false): Promise { + const query = this.queries.find((q) => q.id === queryId); + const tab = this.yasgui.getTab(); + const endpoint = this.endpoint; + if (!query || !tab || !endpoint) return Promise.resolve(); + const key = `${endpoint}\n${queryId}`; + if (this.running.has(key)) return Promise.resolve(); + + const running: RunningQuery = { controller: new AbortController(), timedOut: false }; + this.running.set(key, running); + this.renderQuery(queryId); + + return new Promise((resolve) => { + const finish = () => { + this.running.delete(key); + if (this.endpoint === endpoint) this.renderQuery(queryId); + resolve(); + }; + this.enqueue( + key, + async () => { + try { + await this.executeQuery(tab, endpoint, query, running, loadMore); + } finally { + finish(); + } + }, + finish, + ); + }); + } + + public cancelQuery(queryId: string) { + if (!this.endpoint) return; + const key = `${this.endpoint}\n${queryId}`; + const running = this.running.get(key); + if (running) running.controller.abort(); + const queued = this.queue.findIndex((item) => item.key === key); + if (queued >= 0) this.queue.splice(queued, 1)[0].cancel(); + } + + public runCategory(categoryId: string) { + for (const query of this.queries.filter((q) => q.category === categoryId)) { + void this.runQuery(query.id); + } + } + + public refreshMetadata() { + if (!this.endpoint) return; + void this.loadMetadata(this.endpoint, true); + } + + /** + * Called when the active tab or its endpoint may have changed + */ + public syncEndpoint(tab: Tab | undefined = this.yasgui.getTab()) { + const endpoint = tab?.getEndpoint() || undefined; + this.endpoint = endpoint; + if (!this.isOpen() || endpoint === this.renderedEndpoint) return; + this.renderedEndpoint = endpoint; + if (endpoint) this.store.touch(endpoint); + this.endpointEl.textContent = endpoint || "No endpoint selected"; + this.endpointEl.title = endpoint || ""; + this.renderMetadata(); + for (const query of this.queries) this.renderQuery(query.id); + if (endpoint && this.config.fetchMetadata && !this.store.getMetadata(endpoint)) { + void this.loadMetadata(endpoint, false); + } + } + + private registerListeners() { + this.yasgui.on("tabSelect", (_yasgui, tabId) => this.syncEndpoint(this.yasgui.getTab(tabId))); + // Endpoint changes are persisted through a tab change + this.yasgui.on("tabChange", (_yasgui, tab) => { + if (tab === this.yasgui.getTab() && tab.getEndpoint() !== this.endpoint) this.syncEndpoint(tab); + }); + // An unpinned panel moves out of the way when the user starts writing a query + this.yasgui.tabPanelsEl.addEventListener("focusin", (event) => { + const ui = this.store.getUiState(); + if (!ui.open || ui.pinned || ui.collapsed) return; + const target = event.target as HTMLElement | null; + if (target && target.closest && target.closest(".yasqe")) this.collapse(); + }); + } + + private applyState(refreshTab = true) { + const ui = this.store.getUiState(); + this.rootEl.classList.toggle("open", ui.open); + this.rootEl.classList.toggle("collapsed", ui.open && ui.collapsed); + this.rootEl.classList.toggle("pinned", ui.pinned); + this.drawerEl.style.width = `${Math.max(MIN_WIDTH, ui.width)}px`; + this.pinButton.setAttribute("aria-pressed", String(ui.pinned)); + this.pinButton.title = ui.pinned ? "Unpin panel" : "Pin panel (keep it open)"; + if (refreshTab) this.refreshActiveTab(); + } + + private refreshActiveTab() { + const tab = this.yasgui.getTab(); + if (!tab) return; + tab.getYasqe()?.refresh(); + tab.getYasr()?.refresh(); + } + + /** + * Drawing + */ + private draw() { + this.resizerEl = el("div", "yasgui-describe__resizer"); + this.resizerEl.title = "Drag to resize"; + this.resizerEl.addEventListener("mousedown", this.startResize); + this.rootEl.appendChild(this.resizerEl); + + this.railEl = el("button", "yasgui-describe__rail"); + this.railEl.type = "button"; + this.railEl.title = "Show endpoint overview"; + this.railEl.innerHTML = 'Endpoint overview'; + this.railEl.addEventListener("click", () => this.open()); + this.rootEl.appendChild(this.railEl); + + this.drawerEl = el("div", "yasgui-describe__drawer"); + this.rootEl.appendChild(this.drawerEl); + + const header = el("div", "yasgui-describe__header"); + const headerTop = el("div", "yasgui-describe__header-top"); + const title = el("h2", "yasgui-describe__title", "Endpoint overview"); + const buttons = el("div", "yasgui-describe__header-buttons"); + const refreshButton = iconButton("fa-rotate-right", "Reload service description and VoID"); + refreshButton.addEventListener("click", () => this.refreshMetadata()); + this.pinButton = iconButton("fa-thumbtack", "Pin panel (keep it open)", "yasgui-describe__pin"); + this.pinButton.addEventListener("click", () => this.setPinned(!this.store.getUiState().pinned)); + const collapseButton = iconButton("fa-angles-right", "Move out of the way"); + collapseButton.addEventListener("click", () => this.collapse()); + const closeButton = iconButton("fa-xmark", "Close"); + closeButton.addEventListener("click", () => this.close()); + buttons.append(refreshButton, this.pinButton, collapseButton, closeButton); + headerTop.append(title, buttons); + this.endpointEl = el("div", "yasgui-describe__endpoint"); + header.append(headerTop, this.endpointEl); + this.drawerEl.appendChild(header); + + const body = el("div", "yasgui-describe__body"); + this.metadataEl = el("div", "yasgui-describe__metadata"); + body.appendChild(this.metadataEl); + + const expanded = this.store.getUiState().expandedCategories; + for (const category of this.categories) { + const queries = this.queries.filter((q) => q.category === category.id); + if (queries.length === 0) continue; + const section = el("details", "yasgui-describe__category"); + section.open = expanded.indexOf(String(category.id)) >= 0; + section.addEventListener("toggle", () => { + const current = this.store.getUiState().expandedCategories.filter((id) => id !== category.id); + if (section.open) current.push(String(category.id)); + this.store.setUiState({ expandedCategories: current }); + }); + const summary = el("summary", "yasgui-describe__category-summary"); + const summaryLabel = el("span", "yasgui-describe__category-label"); + summaryLabel.innerHTML = ``; + summaryLabel.appendChild(document.createTextNode(" " + category.label)); + const runAll = el("button", "yasgui-describe__run-all", "Run all"); + runAll.type = "button"; + runAll.title = `Run all queries in "${category.label}"`; + runAll.addEventListener("click", (event) => { + event.preventDefault(); + section.open = true; + this.runCategory(String(category.id)); + }); + summary.append(summaryLabel, runAll); + section.appendChild(summary); + for (const query of queries) { + const queryEl = el("div", "yasgui-describe__query"); + queryEl.dataset.queryId = query.id; + this.queryEls.set(query.id, queryEl); + section.appendChild(queryEl); + } + body.appendChild(section); + } + this.drawerEl.appendChild(body); + } + + private renderMetadata() { + const container = this.metadataEl; + container.innerHTML = ""; + const endpoint = this.endpoint; + if (!endpoint) { + container.appendChild(el("p", "yasgui-describe__muted", "Select an endpoint to explore it.")); + return; + } + const metadata = this.store.getMetadata(endpoint); + const card = el("div", "yasgui-describe__card"); + const heading = el("div", "yasgui-describe__card-heading"); + heading.appendChild(el("strong", undefined, "Service description & VoID")); + if (metadata) heading.appendChild(el("span", "yasgui-describe__muted", formatRelativeTime(metadata.fetchedAt))); + card.appendChild(heading); + + if (!metadata) { + if (this.metadataController) { + card.appendChild(el("p", "yasgui-describe__muted", "Looking for a service description and VoID…")); + } else { + const load = el("button", "yasgui-describe__button", "Load service description & VoID"); + load.type = "button"; + load.addEventListener("click", () => this.refreshMetadata()); + card.appendChild(load); + } + container.appendChild(card); + return; + } + + const status = el("div", "yasgui-describe__status-list"); + status.append( + this.statusBadge("Service description", metadata.sdStatus === "ok", metadata.sdError), + this.statusBadge("VoID", metadata.voidStatus === "ok", metadata.voidError), + ); + card.appendChild(status); + + if (metadata.sd) { + const sd = metadata.sd; + const list = el("dl", "yasgui-describe__facts"); + this.addFact(list, "Languages", sd.languages); + this.addFact(list, "Features", sd.features); + this.addFact(list, "Extension functions", sd.extensionFunctions); + this.addFact(list, "Extension aggregates", sd.extensionAggregates); + this.addFact(list, "Entailment", sd.entailmentRegimes); + this.addFact(list, "Result formats", sd.resultFormats); + this.addFact(list, "Named graphs", sd.namedGraphs); + if (list.childElementCount) card.appendChild(list); + } + for (const dataset of metadata.datasets) card.appendChild(this.renderDataset(dataset)); + if (metadata.sdStatus !== "ok" && metadata.voidStatus !== "ok") { + card.appendChild( + el( + "p", + "yasgui-describe__muted", + "This endpoint does not publish a service description or VoID. Use the queries below to explore it.", + ), + ); + } + container.appendChild(card); + } + + private statusBadge(label: string, ok: boolean, error?: string): HTMLElement { + const badge = el("span", `yasgui-describe__badge ${ok ? "ok" : "unavailable"}`); + badge.textContent = `${label}: ${ok ? "available" : "not available"}`; + if (!ok && error) badge.title = error; + return badge; + } + + private addFact(list: HTMLElement, label: string, values: string[]) { + if (!values.length) return; + list.appendChild(el("dt", undefined, label)); + const dd = el("dd"); + values.slice(0, 50).forEach((value, i) => { + if (i > 0) dd.appendChild(document.createTextNode(", ")); + dd.appendChild(this.renderIri(value)); + }); + if (values.length > 50) dd.appendChild(document.createTextNode(` … (+${values.length - 50})`)); + list.appendChild(dd); + } + + private renderDataset(dataset: VoidDataset): HTMLElement { + const wrapper = el("div", "yasgui-describe__dataset"); + const title = el("div", "yasgui-describe__dataset-title"); + title.appendChild(el("i", "fas fa-database")); + title.appendChild(document.createTextNode(" ")); + if (dataset.title) title.appendChild(el("strong", undefined, dataset.title + " ")); + title.appendChild(this.renderIri(dataset.iri)); + wrapper.appendChild(title); + + const stats = el("dl", "yasgui-describe__facts yasgui-describe__facts--stats"); + const addStat = (label: string, value: number | undefined) => { + if (value === undefined) return; + stats.append(el("dt", undefined, label), el("dd", undefined, formatNumber(value))); + }; + addStat("Triples", dataset.triples); + addStat("Entities", dataset.entities); + addStat("Classes", dataset.classes); + addStat("Properties", dataset.properties); + addStat("Distinct subjects", dataset.distinctSubjects); + addStat("Distinct objects", dataset.distinctObjects); + if (stats.childElementCount) wrapper.appendChild(stats); + const vocabularies = el("dl", "yasgui-describe__facts"); + this.addFact(vocabularies, "Vocabularies", dataset.vocabularies); + if (vocabularies.childElementCount) wrapper.appendChild(vocabularies); + + const partitionTable = (label: string, key: string, rows: VoidDataset["classPartitions"]) => { + if (!rows.length) return; + const details = el("details", "yasgui-describe__partition"); + details.appendChild(el("summary", undefined, `${label} (${rows.length}, from VoID)`)); + const table: DescribeTable = { + vars: [key, "entities", "triples"].filter((v) => v === key || rows.some((r) => (r as any)[v] !== undefined)), + bindings: rows.map((row) => { + const binding: DescribeBinding = { [key]: { type: "uri", value: row.iri } }; + if (row.entities !== undefined) binding.entities = { type: "literal", value: String(row.entities) }; + if (row.triples !== undefined) binding.triples = { type: "literal", value: String(row.triples) }; + return binding; + }), + }; + details.appendChild(this.renderTable(table)); + wrapper.appendChild(details); + }; + partitionTable("Classes", "class", dataset.classPartitions); + partitionTable("Properties", "property", dataset.propertyPartitions); + if (dataset.linksets.length) { + const list = el("dl", "yasgui-describe__facts"); + this.addFact( + list, + "Linksets", + dataset.linksets.map((l) => l.target || l.iri), + ); + wrapper.appendChild(list); + } + return wrapper; + } + + private renderQuery(queryId: string) { + const container = this.queryEls.get(queryId); + const query = this.queries.find((q) => q.id === queryId); + if (!container || !query) return; + container.innerHTML = ""; + const endpoint = this.endpoint; + const running = endpoint ? this.running.has(`${endpoint}\n${queryId}`) : false; + const result = endpoint ? this.store.getResult(endpoint, queryId) : undefined; + + const header = el("div", "yasgui-describe__query-header"); + const label = el("div", "yasgui-describe__query-label"); + label.appendChild(el("span", undefined, query.label)); + if (query.expensive) { + const badge = el("span", "yasgui-describe__slow", "may be slow"); + badge.title = "This query scans the whole dataset and may be slow or time out on large endpoints"; + label.appendChild(badge); + } + header.appendChild(label); + + const actions = el("div", "yasgui-describe__query-actions"); + if (running) { + const cancel = iconButton("fa-stop", "Cancel"); + cancel.addEventListener("click", () => this.cancelQuery(queryId)); + actions.appendChild(cancel); + } else { + const run = iconButton(result ? "fa-rotate-right" : "fa-play", result ? "Run again" : "Run"); + run.disabled = !endpoint; + run.addEventListener("click", () => void this.runQuery(queryId)); + actions.appendChild(run); + } + const openInTab = iconButton("fa-arrow-up-right-from-square", "Open query in a new tab"); + openInTab.disabled = !endpoint; + openInTab.addEventListener("click", () => this.openInNewTab(query)); + actions.appendChild(openInTab); + header.appendChild(actions); + container.appendChild(header); + + if (query.description) container.appendChild(el("div", "yasgui-describe__query-description", query.description)); + + if (running) { + container.appendChild(el("div", "yasgui-describe__loading", "Running…")); + } + if (!result) return; + + const meta: string[] = [`fetched ${formatRelativeTime(result.fetchedAt)}`]; + if (result.durationMs !== undefined) meta.push(`${result.durationMs.toLocaleString()} ms`); + if (result.status === "done") meta.push(`${result.bindings.length.toLocaleString()} rows`); + if (result.truncated) meta.push("truncated"); + container.appendChild(el("div", "yasgui-describe__query-meta", meta.join(" · "))); + + if (result.error || result.status === "error") { + container.appendChild(el("div", "yasgui-describe__error", result.error || "Query failed")); + } + if (result.status === "error") return; + if (result.bindings.length === 0) { + container.appendChild(el("div", "yasgui-describe__muted", "No results")); + return; + } + container.appendChild(this.renderTable(result)); + if (query.paginated && result.hasMore && !running) { + const more = el("button", "yasgui-describe__button", "Load more"); + more.type = "button"; + more.addEventListener("click", () => void this.runQuery(queryId, true)); + container.appendChild(more); + } + } + + private renderTable(table: DescribeTable): HTMLElement { + const wrapper = el("div", "yasgui-describe__table-wrapper"); + const tableEl = el("table", "yasgui-describe__table"); + const thead = el("thead"); + const headRow = el("tr"); + for (const variable of table.vars) headRow.appendChild(el("th", undefined, variable)); + thead.appendChild(headRow); + const tbody = el("tbody"); + for (const binding of table.bindings) { + const row = el("tr"); + for (const variable of table.vars) { + const cell = el("td"); + const term = binding[variable]; + if (term) cell.appendChild(this.renderTerm(term)); + row.appendChild(cell); + } + tbody.appendChild(row); + } + tableEl.append(thead, tbody); + wrapper.appendChild(tableEl); + return wrapper; + } + + private renderTerm(term: DescribeBinding[string]): Node { + if (term.type === "uri") return this.renderIri(term.value); + if (term.type === "bnode") return document.createTextNode(`_:${term.value}`); + const value = term.value; + const span = el("span", "yasgui-describe__literal"); + span.textContent = /^-?\d{4,}$/.test(value) ? Number(value).toLocaleString() : value; + if (term["xml:lang"]) span.appendChild(el("span", "yasgui-describe__lang", `@${term["xml:lang"]}`)); + return span; + } + + private renderIri(iri: string): HTMLElement { + const shortened = this.shortenIri(iri); + const button = el("button", "yasgui-describe__iri", shortened ? shortened.prefixed : iri); + button.type = "button"; + button.title = `${iri}\nClick to insert into the query`; + button.addEventListener("click", () => this.insertIntoEditor(iri)); + return button; + } + + /** + * Prefix handling + */ + private getPrefixes(): { [prefix: string]: string } { + const prefixes: { [prefix: string]: string } = { ...COMMON_PREFIXES }; + // Saved prefixes (Settings > Prefixes) + const saved = this.yasgui.persistentConfig.getPrefixes() || ""; + for (const line of saved.split("\n")) { + const match = line.trim().match(/^PREFIX\s+(\w[\w.-]*|):\s*<([^>]+)>/i); + if (match) prefixes[match[1]] = match[2]; + } + // Prefixes declared in the current query take precedence + const fromQuery = this.yasgui.getTab()?.getYasqe()?.getPrefixesFromQuery() || {}; + return { ...prefixes, ...fromQuery }; + } + + public shortenIri(iri: string): { prefixed: string; prefix: string; namespace: string } | undefined { + const localPart = (namespace: string) => iri.substring(namespace.length); + const isValidLocal = (local: string) => /^[\w-]*$/.test(local) && !/^-/.test(local); + let best: { prefix: string; namespace: string } | undefined; + for (const [prefix, namespace] of Object.entries(this.getPrefixes())) { + if (iri.startsWith(namespace) && isValidLocal(localPart(namespace))) { + if (!best || namespace.length > best.namespace.length) best = { prefix, namespace }; + } + } + if (!best) return undefined; + return { ...best, prefixed: `${best.prefix}:${localPart(best.namespace)}` }; + } + + private insertIntoEditor(iri: string) { + const yasqe = this.yasgui.getTab()?.getYasqe(); + if (!yasqe) return; + const shortened = this.shortenIri(iri); + if (shortened) { + const declared = yasqe.getPrefixesFromQuery(); + if (declared[shortened.prefix] === undefined) { + yasqe.addPrefixes({ [shortened.prefix]: shortened.namespace }); + } else if (declared[shortened.prefix] !== shortened.namespace) { + yasqe.replaceSelection(`<${iri}>`); + return; + } + yasqe.replaceSelection(shortened.prefixed); + } else { + yasqe.replaceSelection(`<${iri}>`); + } + } + + private openInNewTab(query: DescribeQuery) { + const endpoint = this.endpoint; + if (!endpoint) return; + const queryString = buildQuery(query, { + limit: query.pageSize || DEFAULT_PAGE_SIZE, + offset: 0, + getResult: (id) => this.store.getResult(endpoint, id), + }); + const tab = this.yasgui.addTab(true, { + name: this.yasgui.createTabName(query.label), + requestConfig: { endpoint }, + } as any); + tab.setQuery(queryString); + } + + /** + * Query execution + */ + private enqueue(key: string, start: () => Promise, cancel: () => void) { + this.queue.push({ key, start, cancel }); + this.drainQueue(); + } + + private drainQueue() { + while (this.activeCount < Math.max(1, this.config.maxConcurrentQueries) && this.queue.length) { + const item = this.queue.shift()!; + this.activeCount++; + void item.start().finally(() => { + this.activeCount--; + this.drainQueue(); + }); + } + } + + private async executeQuery( + tab: Tab, + endpoint: string, + query: DescribeQuery, + running: RunningQuery, + loadMore: boolean, + ): Promise { + if (query.dependsOn && !this.store.getResult(endpoint, query.dependsOn)) { + // Run the dependency within this slot, so it can't deadlock on the concurrency limit + const dependency = this.queries.find((q) => q.id === query.dependsOn); + if (dependency) { + await this.executeQuery(tab, endpoint, dependency, running, false); + if (this.endpoint === endpoint) this.renderQuery(dependency.id); + } + } + if (running.controller.signal.aborted) return; + + const pageSize = query.pageSize || DEFAULT_PAGE_SIZE; + const previous = this.store.getResult(endpoint, query.id); + const offset = loadMore && previous?.status === "done" ? previous.bindings.length : 0; + const queryString = buildQuery(query, { + limit: pageSize, + offset, + getResult: (id) => this.store.getResult(endpoint, id), + }); + + const timeout = setTimeout(() => { + running.timedOut = true; + running.controller.abort(); + }, this.config.timeoutMs); + const start = Date.now(); + try { + const response = await tab.runBackgroundQuery(queryString, { + accept: SPARQL_JSON, + signal: running.controller.signal, + skipGraphArgs: true, + }); + const parser = new Parser(response, Date.now() - start); + if (parser.hasError()) { + const error = parser.getError(); + throw new Error(error?.text || error?.statusText || "Query failed"); + } + let table: DescribeTable = { + vars: parser.getVariables(), + bindings: (parser.getBindings() || []) as DescribeBinding[], + }; + if (!table.vars.length && parser.getBoolean() !== undefined) { + table = { vars: ["result"], bindings: [{ result: { type: "literal", value: String(parser.getBoolean()) } }] }; + } + const hasMore = !!query.paginated && table.bindings.length >= pageSize; + if (query.postProcess) table = query.postProcess(table); + const result: DescribeResult = { + status: "done", + vars: table.vars, + bindings: offset > 0 && previous ? [...previous.bindings, ...table.bindings] : table.bindings, + fetchedAt: Date.now(), + durationMs: Date.now() - start, + hasMore, + }; + this.store.setResult(endpoint, query.id, result); + } catch (e: any) { + if (running.controller.signal.aborted && !running.timedOut) return; // Cancelled by the user + let message = e instanceof Error ? e.message : String(e); + if (running.timedOut) message = `Timed out after ${Math.round(this.config.timeoutMs / 1000)} s`; + else if (e?.status) message = `HTTP ${e.status}${e.statusText ? " " + e.statusText : ""}: ${message}`; + // Keep the rows that were already loaded when loading another page fails + this.store.setResult(endpoint, query.id, { + ...(offset > 0 && previous ? previous : { vars: [], bindings: [] }), + status: offset > 0 && previous ? previous.status : "error", + error: message.length > 500 ? message.substring(0, 500) + "…" : message, + fetchedAt: Date.now(), + durationMs: Date.now() - start, + hasMore: false, + }); + } finally { + clearTimeout(timeout); + } + } + + private async loadMetadata(endpoint: string, force: boolean) { + if (!force && this.store.getMetadata(endpoint)) return; + this.metadataController?.abort(); + const controller = new AbortController(); + this.metadataController = controller; + this.renderMetadata(); + const timeout = setTimeout(() => controller.abort(), Math.min(this.config.timeoutMs, 20000)); + try { + const tab = this.yasgui.getTab(); + const init = (tab && tab.getEndpoint() === endpoint && (await tab.getRequestInit())) || {}; + const metadata: EndpointMetadata = await fetchEndpointMetadata(endpoint, { ...init, signal: controller.signal }); + this.store.setMetadata(endpoint, metadata); + } catch (e) { + // Superseded by a newer request: nothing to store + if (this.metadataController === controller) { + const message = controller.signal.aborted ? "Timed out" : e instanceof Error ? e.message : String(e); + this.store.setMetadata(endpoint, { + fetchedAt: Date.now(), + sdStatus: "unavailable", + voidStatus: "unavailable", + datasets: [], + sdError: message, + voidError: message, + }); + } + } finally { + clearTimeout(timeout); + if (this.metadataController === controller) this.metadataController = undefined; + if (this.endpoint === endpoint) this.renderMetadata(); + } + } + + /** + * Resizing + */ + private startResize = (event: MouseEvent) => { + event.preventDefault(); + addClass(this.rootEl, "resizing"); + document.documentElement.addEventListener("mousemove", this.doResize); + document.documentElement.addEventListener("mouseup", this.stopResize); + }; + + private doResize = (event: MouseEvent) => { + const right = this.rootEl.getBoundingClientRect().right; + const maxWidth = Math.max(MIN_WIDTH, this.yasgui.rootEl.getBoundingClientRect().width * 0.7); + const width = Math.min(maxWidth, Math.max(MIN_WIDTH, right - event.clientX)); + this.drawerEl.style.width = `${width}px`; + }; + + private stopResize = () => { + removeClass(this.rootEl, "resizing"); + document.documentElement.removeEventListener("mousemove", this.doResize); + document.documentElement.removeEventListener("mouseup", this.stopResize); + this.store.setUiState({ width: Math.round(this.drawerEl.getBoundingClientRect().width) }); + this.refreshActiveTab(); + }; + + public destroy() { + this.metadataController?.abort(); + for (const running of this.running.values()) running.controller.abort(); + for (const item of this.queue.splice(0)) item.cancel(); + this.rootEl.remove(); + } +} diff --git a/packages/yasgui/src/endpointDescribe/describeQueries.ts b/packages/yasgui/src/endpointDescribe/describeQueries.ts new file mode 100644 index 00000000..8d7ce32c --- /dev/null +++ b/packages/yasgui/src/endpointDescribe/describeQueries.ts @@ -0,0 +1,452 @@ +/** + * Catalogue of "describe endpoint" queries. + * + * Every query is bounded (LIMIT) so that exploring a very large endpoint never + * results in an unbounded request. Queries flagged as `expensive` require a full + * scan on most triple stores and are marked as such in the UI. + */ + +export type DescribeCategoryId = "overview" | "schema" | "instances" | "labels" | "links" | "timegeo"; + +export interface DescribeCategory { + id: DescribeCategoryId | string; + label: string; + icon: string; +} + +export interface DescribeBinding { + [variable: string]: { type: string; value: string; "xml:lang"?: string; datatype?: string }; +} + +export interface DescribeTable { + vars: string[]; + bindings: DescribeBinding[]; +} + +export interface DescribeQueryContext { + /** Maximum number of rows to fetch */ + limit: number; + /** Offset, used when loading more rows of a paginated query */ + offset: number; + /** Results of other describe queries for the same endpoint (used by dependent queries) */ + getResult: (queryId: string) => DescribeTable | undefined; +} + +export interface DescribeQuery { + id: string; + category: DescribeCategoryId | string; + label: string; + description?: string; + /** SPARQL SELECT query, or a function that builds it from the context */ + query: string | ((ctx: DescribeQueryContext) => string); + /** Number of rows fetched per page */ + pageSize?: number; + /** Whether more rows can be loaded with LIMIT/OFFSET */ + paginated?: boolean; + /** Query that must have results before this one can be built */ + dependsOn?: string; + /** Hint shown in the UI: this query may be slow on large endpoints */ + expensive?: boolean; + /** Transform the raw results (e.g. to compute aggregates client-side) */ + postProcess?: (table: DescribeTable) => DescribeTable; +} + +export const DEFAULT_PAGE_SIZE = 25; + +export const defaultCategories: DescribeCategory[] = [ + { id: "overview", label: "Endpoint & dataset overview", icon: "fa-database" }, + { id: "schema", label: "Vocabulary & schema", icon: "fa-sitemap" }, + { id: "instances", label: "Instances", icon: "fa-cubes" }, + { id: "labels", label: "Labels, languages & literals", icon: "fa-language" }, + { id: "links", label: "Interlinking & external links", icon: "fa-link" }, + { id: "timegeo", label: "Time & geo", icon: "fa-earth-europe" }, +]; + +export const COMMON_PREFIXES: { [prefix: string]: string } = { + dcat: "http://www.w3.org/ns/dcat#", + prov: "http://www.w3.org/ns/prov#", + sosa: "http://www.w3.org/ns/sosa/", + org: "http://www.w3.org/ns/org#", + void: "http://rdfs.org/ns/void#", + sd: "http://www.w3.org/ns/sparql-service-description#", + vcard: "http://www.w3.org/2006/vcard/ns#", + adms: "http://www.w3.org/ns/adms#", + locn: "http://www.w3.org/ns/locn#", + dbo: "http://dbpedia.org/ontology/", + dbr: "http://dbpedia.org/resource/", + dbp: "http://dbpedia.org/property/", + wd: "http://www.wikidata.org/entity/", + wdt: "http://www.wikidata.org/prop/direct/", + p: "http://www.wikidata.org/prop/", + ps: "http://www.wikidata.org/prop/statement/", + pq: "http://www.wikidata.org/prop/qualifier/", + wikibase: "http://wikiba.se/ontology#", + era: "http://data.europa.eu/949/", + euvoc: "http://publications.europa.eu/ontology/euvoc#", +}; + +const PREFIXES = { + rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + rdfs: "http://www.w3.org/2000/01/rdf-schema#", + owl: "http://www.w3.org/2002/07/owl#", + xsd: "http://www.w3.org/2001/XMLSchema#", + skos: "http://www.w3.org/2004/02/skos/core#", + sh: "http://www.w3.org/ns/shacl#", + shex: "http://www.w3.org/ns/shex#", + dcterms: "http://purl.org/dc/terms/", + dc: "http://purl.org/dc/elements/1.1/", + schema: "http://schema.org/", + foaf: "http://xmlns.com/foaf/0.1/", + geo: "http://www.opengis.net/ont/geosparql#", + wgs84: "http://www.w3.org/2003/01/geo/wgs84_pos#", + georss: "http://www.georss.org/georss/", +}; + +Object.assign(COMMON_PREFIXES, PREFIXES); + +function prefixes(...names: Array): string { + return names.map((name) => `PREFIX ${name}: <${PREFIXES[name]}>`).join("\n") + "\n"; +} + +function page(ctx: DescribeQueryContext): string { + return `LIMIT ${ctx.limit}` + (ctx.offset > 0 ? ` OFFSET ${ctx.offset}` : ""); +} + +function iriTerm(value: string): string { + return `<${value.replace(/[<>"{}|^`\\\s]/g, "")}>`; +} + +const NUMBER = "-?\\d+(?:\\.\\d+)?(?:[eE][-+]?\\d+)?"; +const COORDINATE_TUPLE = new RegExp(`${NUMBER}(?:\\s+${NUMBER})+`, "g"); + +/** + * Computes a bounding box and centroid from a sample of WKT literals. + * Only the first two numbers of every coordinate tuple are used (x/y), Z and M values are ignored. + */ +export function wktExtent(table: DescribeTable): DescribeTable { + let minX = Infinity, + minY = Infinity, + maxX = -Infinity, + maxY = -Infinity, + sumX = 0, + sumY = 0, + points = 0, + geometries = 0; + const crsSet = new Set(); + for (const binding of table.bindings) { + const wkt = binding.wkt?.value; + if (!wkt) continue; + const crsMatch = wkt.trim().match(/^<([^>]+)>/); + crsSet.add(crsMatch ? crsMatch[1] : "http://www.opengis.net/def/crs/OGC/1.3/CRS84"); + const body = crsMatch ? wkt.trim().substring(crsMatch[0].length) : wkt; + const tuples = body.match(COORDINATE_TUPLE); + if (!tuples) continue; + geometries++; + for (const tuple of tuples) { + const [x, y] = tuple.trim().split(/\s+/).map(Number); + if (!isFinite(x) || !isFinite(y)) continue; + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + sumX += x; + sumY += y; + points++; + } + } + const literal = (value: number | string) => ({ type: "literal", value: String(value) }); + if (points === 0) return { vars: ["sampledGeometries"], bindings: [{ sampledGeometries: literal(0) }] }; + return { + vars: ["minX", "minY", "maxX", "maxY", "centroidX", "centroidY", "sampledGeometries", "crs"], + bindings: [ + { + minX: literal(minX), + minY: literal(minY), + maxX: literal(maxX), + maxY: literal(maxY), + centroidX: literal(+(sumX / points).toFixed(6)), + centroidY: literal(+(sumY / points).toFixed(6)), + sampledGeometries: literal(geometries), + crs: literal(Array.from(crsSet).join(", ")), + }, + ], + }; +} + +export const defaultDescribeQueries: DescribeQuery[] = [ + /** + * 1) Endpoint & dataset overview + */ + { + id: "named-graphs", + category: "overview", + label: "Named graphs", + description: "Lists the named graphs, page by page, to see how the data is partitioned.", + paginated: true, + query: (ctx) => `SELECT DISTINCT ?graph WHERE {\n GRAPH ?graph { ?s ?p ?o }\n}\n${page(ctx)}`, + }, + { + id: "graph-sizes", + category: "overview", + label: "Triples per named graph", + description: "Number of triples in each named graph, largest first.", + paginated: true, + expensive: true, + query: (ctx) => + `SELECT ?graph (COUNT(*) AS ?triples) WHERE {\n GRAPH ?graph { ?s ?p ?o }\n}\nGROUP BY ?graph\nORDER BY DESC(?triples)\n${page( + ctx, + )}`, + }, + { + id: "triple-count", + category: "overview", + label: "Total number of triples", + description: "Number of triples in the default graph (which is the union of all graphs on many triple stores).", + expensive: true, + query: `SELECT (COUNT(*) AS ?triples) WHERE {\n ?s ?p ?o\n}`, + }, + { + id: "namespaces", + category: "overview", + label: "Most used namespaces", + description: "Namespaces of the predicates, ranked by the number of triples using them.", + expensive: true, + query: (ctx) => + `SELECT ?namespace (COUNT(*) AS ?triples) WHERE {\n ?s ?p ?o .\n BIND(REPLACE(STR(?p), "[^#/]*$", "") AS ?namespace)\n}\nGROUP BY ?namespace\nORDER BY DESC(?triples)\n${page( + ctx, + )}`, + paginated: true, + }, + + /** + * 2) Vocabulary & schema discovery + */ + { + id: "classes", + category: "schema", + label: "Classes", + description: "All classes with the number of instances, most used first.", + paginated: true, + query: (ctx) => + `SELECT ?class (COUNT(?instance) AS ?instances) WHERE {\n ?instance a ?class\n}\nGROUP BY ?class\nORDER BY DESC(?instances)\n${page( + ctx, + )}`, + }, + { + id: "properties", + category: "schema", + label: "Properties", + description: "All properties with their usage count, to learn the shape of the data.", + paginated: true, + expensive: true, + query: (ctx) => + `SELECT ?property (COUNT(*) AS ?uses) WHERE {\n ?s ?property ?o\n}\nGROUP BY ?property\nORDER BY DESC(?uses)\n${page( + ctx, + )}`, + }, + { + id: "skos-schemes", + category: "schema", + label: "SKOS concept schemes", + description: "SKOS concept schemes and the number of concepts in each.", + paginated: true, + query: (ctx) => + `${prefixes( + "skos", + )}SELECT ?scheme (SAMPLE(?schemeLabel) AS ?label) (COUNT(DISTINCT ?concept) AS ?concepts) WHERE {\n ?scheme a skos:ConceptScheme .\n OPTIONAL { ?concept skos:inScheme ?scheme }\n OPTIONAL { ?scheme skos:prefLabel ?schemeLabel }\n}\nGROUP BY ?scheme\nORDER BY DESC(?concepts)\n${page( + ctx, + )}`, + }, + { + id: "shapes", + category: "schema", + label: "SHACL / ShEx shapes", + description: "Shapes and constraints published in the endpoint, with their target class when available.", + paginated: true, + query: (ctx) => + `${prefixes( + "sh", + "shex", + )}SELECT ?shape ?type ?targetClass WHERE {\n VALUES ?type { sh:NodeShape sh:PropertyShape shex:Schema shex:ShapeDecl shex:Shape }\n ?shape a ?type .\n OPTIONAL { ?shape sh:targetClass ?targetClass }\n}\n${page( + ctx, + )}`, + }, + + /** + * 3) Instance-level exploration + */ + { + id: "class-samples", + category: "instances", + label: "Sample instances per class", + description: "A few instances of each of the 10 most used classes (runs the Classes query first).", + dependsOn: "classes", + query: (ctx) => { + const classes = (ctx.getResult("classes")?.bindings || []) + .map((b) => b.class) + .filter((term) => term && term.type === "uri") + .slice(0, 10); + if (classes.length === 0) { + return `SELECT ?class ?instance WHERE {\n ?instance a ?class\n}\nLIMIT ${ctx.limit}`; + } + const perClass = Math.max(1, Math.floor(50 / classes.length)); + const unions = classes + .map( + (term) => + ` {\n SELECT ?class ?instance WHERE {\n ?instance a ${iriTerm(term.value)} .\n BIND(${iriTerm( + term.value, + )} AS ?class)\n }\n LIMIT ${perClass}\n }`, + ) + .join("\n UNION\n"); + return `SELECT ?class ?instance WHERE {\n${unions}\n}`; + }, + }, + { + id: "hubs", + category: "instances", + label: "Most connected nodes", + description: "Resources with the highest number of incoming and outgoing statements (hubs).", + expensive: true, + query: (ctx) => + `SELECT ?node (COUNT(*) AS ?degree) WHERE {\n { ?node ?p ?o }\n UNION\n { ?s ?p ?node . FILTER(isIRI(?node)) }\n}\nGROUP BY ?node\nORDER BY DESC(?degree)\n${page( + ctx, + )}`, + }, + + /** + * 4) Labels, languages & literals + */ + { + id: "label-languages", + category: "labels", + label: "Labels per language", + description: "Number of labels and titles per language, to see the localization coverage.", + expensive: true, + query: `${prefixes( + "rdfs", + "skos", + "dcterms", + "dc", + "schema", + "foaf", + )}SELECT ?property ?language (COUNT(*) AS ?labels) WHERE {\n VALUES ?property { rdfs:label skos:prefLabel skos:altLabel dcterms:title dc:title schema:name foaf:name }\n ?s ?property ?label .\n BIND(LANG(?label) AS ?language)\n}\nGROUP BY ?property ?language\nORDER BY DESC(?labels)\nLIMIT 100`, + }, + { + id: "language-tagged-properties", + category: "labels", + label: "Properties with language-tagged literals", + description: "Properties that carry language-tagged literals and their language distribution.", + paginated: true, + expensive: true, + query: (ctx) => + `SELECT ?property ?language (COUNT(*) AS ?literals) WHERE {\n ?s ?property ?o .\n FILTER(isLiteral(?o) && LANG(?o) != "")\n BIND(LANG(?o) AS ?language)\n}\nGROUP BY ?property ?language\nORDER BY DESC(?literals)\n${page( + ctx, + )}`, + }, + + /** + * 5) Interlinking & external links + */ + { + id: "linked-hosts", + category: "links", + label: "Referenced hosts", + description: "Hosts of the IRIs used as objects (excluding rdf:type), e.g. Wikidata, GeoNames or DBpedia.", + paginated: true, + expensive: true, + query: (ctx) => + `${prefixes( + "rdf", + )}SELECT ?host (COUNT(*) AS ?links) WHERE {\n ?s ?p ?o .\n FILTER(isIRI(?o) && ?p != rdf:type)\n BIND(REPLACE(STR(?o), "^([a-zA-Z][a-zA-Z0-9+.-]*://[^/?#]+).*$", "$1") AS ?host)\n}\nGROUP BY ?host\nORDER BY DESC(?links)\n${page( + ctx, + )}`, + }, + { + id: "link-properties", + category: "links", + label: "Linking properties", + description: "Usage of common interlinking properties (owl:sameAs, SKOS mapping properties, rdfs:seeAlso, …).", + query: `${prefixes( + "owl", + "skos", + "rdfs", + "schema", + )}SELECT ?property (COUNT(*) AS ?links) (SAMPLE(?target) AS ?exampleTarget) WHERE {\n VALUES ?property { owl:sameAs skos:exactMatch skos:closeMatch skos:relatedMatch skos:broadMatch skos:narrowMatch rdfs:seeAlso schema:sameAs }\n ?entity ?property ?target .\n}\nGROUP BY ?property\nORDER BY DESC(?links)\nLIMIT 50`, + }, + { + id: "linked-entities", + category: "links", + label: "Entities with outbound links", + description: "Entities that link to other resources with an interlinking property.", + paginated: true, + query: (ctx) => + `${prefixes( + "owl", + "skos", + "rdfs", + "schema", + )}SELECT ?entity ?property ?target WHERE {\n VALUES ?property { owl:sameAs skos:exactMatch skos:closeMatch rdfs:seeAlso schema:sameAs }\n ?entity ?property ?target .\n FILTER(isIRI(?target))\n}\n${page( + ctx, + )}`, + }, + + /** + * 6) Time & geo dimensions + */ + { + id: "temporal-properties", + category: "timegeo", + label: "Temporal properties", + description: "Properties with date or time values, with their earliest and latest value.", + expensive: true, + query: `${prefixes( + "xsd", + )}SELECT ?property ?datatype (COUNT(*) AS ?values) (MIN(?o) AS ?earliest) (MAX(?o) AS ?latest) WHERE {\n ?s ?property ?o .\n FILTER(isLiteral(?o))\n BIND(DATATYPE(?o) AS ?datatype)\n FILTER(?datatype IN (xsd:date, xsd:dateTime, xsd:dateTimeStamp, xsd:gYear, xsd:gYearMonth, xsd:time))\n}\nGROUP BY ?property ?datatype\nORDER BY DESC(?values)\nLIMIT 50`, + }, + { + id: "geo-properties", + category: "timegeo", + label: "Geospatial properties", + description: "Usage of GeoSPARQL, W3C WGS84 and schema.org geo properties.", + query: `${prefixes( + "geo", + "wgs84", + "schema", + "georss", + )}SELECT ?property ?datatype (COUNT(*) AS ?values) (SAMPLE(?o) AS ?example) WHERE {\n VALUES ?property { geo:hasGeometry geo:hasDefaultGeometry geo:asWKT geo:asGML geo:asGeoJSON wgs84:lat wgs84:long wgs84:lat_long schema:geo schema:latitude schema:longitude georss:point }\n ?s ?property ?o .\n BIND(DATATYPE(?o) AS ?datatype)\n}\nGROUP BY ?property ?datatype\nORDER BY DESC(?values)\nLIMIT 50`, + }, + { + id: "wkt-crs", + category: "timegeo", + label: "Coordinate reference systems", + description: "CRS used in GeoSPARQL WKT literals (literals without an explicit CRS use CRS84).", + expensive: true, + query: `${prefixes( + "geo", + )}SELECT ?crs (COUNT(*) AS ?geometries) WHERE {\n ?geometry geo:asWKT ?wkt .\n BIND(IF(STRSTARTS(STR(?wkt), "<"), STRBEFORE(STRAFTER(STR(?wkt), "<"), ">"), "http://www.opengis.net/def/crs/OGC/1.3/CRS84") AS ?crs)\n}\nGROUP BY ?crs\nORDER BY DESC(?geometries)\nLIMIT 50`, + }, + { + id: "latlong-extent", + category: "timegeo", + label: "Bounding box (lat/long)", + description: "Bounding box and centroid of all resources with WGS84 or schema.org latitude/longitude.", + expensive: true, + query: `${prefixes( + "xsd", + "wgs84", + "schema", + )}SELECT (MIN(?lat) AS ?minLat) (MIN(?long) AS ?minLong) (MAX(?lat) AS ?maxLat) (MAX(?long) AS ?maxLong) (AVG(?lat) AS ?centroidLat) (AVG(?long) AS ?centroidLong) (COUNT(*) AS ?points) WHERE {\n { ?s wgs84:lat ?latValue ; wgs84:long ?longValue }\n UNION\n { ?s schema:latitude ?latValue ; schema:longitude ?longValue }\n BIND(xsd:decimal(?latValue) AS ?lat)\n BIND(xsd:decimal(?longValue) AS ?long)\n}`, + }, + { + id: "wkt-extent", + category: "timegeo", + label: "Bounding box (WKT sample)", + description: "Bounding box and centroid computed in the browser from a sample of 1000 GeoSPARQL WKT geometries.", + query: `${prefixes("geo")}SELECT ?wkt WHERE {\n ?geometry geo:asWKT ?wkt\n}\nLIMIT 1000`, + postProcess: wktExtent, + }, +]; + +export function buildQuery(query: DescribeQuery, ctx: DescribeQueryContext): string { + return typeof query.query === "function" ? query.query(ctx) : query.query; +} diff --git a/packages/yasgui/src/endpointDescribe/metadataSources.ts b/packages/yasgui/src/endpointDescribe/metadataSources.ts new file mode 100644 index 00000000..da9a0205 --- /dev/null +++ b/packages/yasgui/src/endpointDescribe/metadataSources.ts @@ -0,0 +1,284 @@ +/** + * Discovery of the SPARQL Service Description (SD) and VoID description of an endpoint. + * + * - SD: https://www.w3.org/TR/sparql11-service-description/ is returned by the endpoint + * itself when it is dereferenced without a query. + * - VoID: https://www.w3.org/TR/void/ is commonly published at /.well-known/void + * + * Both are optional. Failures (CORS, 404, HTML responses, unsupported formats) are expected + * and are reported as "unavailable". + */ +import * as N3 from "n3"; + +const SD = "http://www.w3.org/ns/sparql-service-description#"; +const VOID = "http://rdfs.org/ns/void#"; +const RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"; +const DCTERMS = "http://purl.org/dc/terms/"; +const RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label"; + +/** Maximum number of list items kept per metadata field */ +const MAX_ITEMS = 200; + +export const RDF_ACCEPT = "text/turtle, application/n-triples;q=0.9, application/n-quads;q=0.8, text/n3;q=0.7"; + +export type MetadataStatus = "ok" | "unavailable"; + +export interface ServiceDescription { + languages: string[]; + features: string[]; + extensionFunctions: string[]; + extensionAggregates: string[]; + resultFormats: string[]; + inputFormats: string[]; + entailmentRegimes: string[]; + namedGraphs: string[]; +} + +export interface VoidPartition { + iri: string; + entities?: number; + triples?: number; +} + +export interface VoidDataset { + iri: string; + title?: string; + triples?: number; + entities?: number; + classes?: number; + properties?: number; + distinctSubjects?: number; + distinctObjects?: number; + sparqlEndpoints: string[]; + vocabularies: string[]; + classPartitions: VoidPartition[]; + propertyPartitions: VoidPartition[]; + linksets: Array<{ iri: string; target?: string; triples?: number }>; +} + +export interface EndpointMetadata { + fetchedAt: number; + sdStatus: MetadataStatus; + voidStatus: MetadataStatus; + sd?: ServiceDescription; + datasets: VoidDataset[]; + /** Human readable reason why a source is unavailable */ + sdError?: string; + voidError?: string; +} + +export interface FetchInit { + headers?: { [key: string]: string }; + withCredentials?: boolean; + signal?: AbortSignal; +} + +export function parseRdf(content: string, contentType: string | null, baseIRI: string): N3.Quad[] { + const type = (contentType || "").split(";")[0].trim().toLowerCase(); + if (type.includes("html") || type.includes("json") || type.includes("xml")) { + throw new Error(`Unsupported content type ${type}`); + } + let format: string | undefined; + if (type === "application/n-triples") format = "N-Triples"; + else if (type === "application/n-quads") format = "N-Quads"; + else if (type === "text/n3") format = "N3"; + else if (type === "application/trig") format = "TriG"; + else format = "Turtle"; + const parser = new N3.Parser({ baseIRI, format }); + return parser.parse(content); +} + +function unique(values: string[]): string[] { + return Array.from(new Set(values)).slice(0, MAX_ITEMS); +} + +class QuadIndex { + private bySubject = new Map(); + constructor(public quads: N3.Quad[]) { + for (const quad of quads) { + const key = quad.subject.value; + if (!this.bySubject.has(key)) this.bySubject.set(key, []); + this.bySubject.get(key)!.push(quad); + } + } + objects(subject: string, predicate: string): N3.Term[] { + return (this.bySubject.get(subject) || []).filter((q) => q.predicate.value === predicate).map((q) => q.object); + } + values(subject: string, predicate: string): string[] { + return this.objects(subject, predicate).map((o) => o.value); + } + number(subject: string, predicate: string): number | undefined { + const value = this.values(subject, predicate)[0]; + if (value === undefined) return undefined; + const num = Number(value); + return isFinite(num) ? num : undefined; + } + subjectsOfType(type: string): string[] { + return unique( + this.quads.filter((q) => q.predicate.value === RDF_TYPE && q.object.value === type).map((q) => q.subject.value), + ); + } + subjectsWith(predicate: string): string[] { + return unique(this.quads.filter((q) => q.predicate.value === predicate).map((q) => q.subject.value)); + } +} + +export function extractServiceDescription(quads: N3.Quad[]): ServiceDescription | undefined { + const index = new QuadIndex(quads); + const services = unique([...index.subjectsOfType(SD + "Service"), ...index.subjectsWith(SD + "endpoint")]); + if (services.length === 0) return undefined; + const collect = (predicate: string) => unique(services.flatMap((s) => index.values(s, SD + predicate))); + + // Named graphs are reachable via sd:defaultDataset / sd:availableGraphs -> sd:namedGraph -> sd:name + const datasets = unique([ + ...services.flatMap((s) => index.values(s, SD + "defaultDataset")), + ...services.flatMap((s) => index.values(s, SD + "availableGraphs")), + ]); + const namedGraphs = unique( + datasets + .flatMap((d) => index.values(d, SD + "namedGraph")) + .flatMap((ng) => { + const names = index.values(ng, SD + "name"); + return names.length ? names : [ng]; + }), + ); + + return { + languages: collect("supportedLanguage"), + features: collect("feature"), + extensionFunctions: collect("extensionFunction"), + extensionAggregates: collect("extensionAggregate"), + resultFormats: collect("resultFormat"), + inputFormats: collect("inputFormat"), + entailmentRegimes: collect("defaultEntailmentRegime"), + namedGraphs, + }; +} + +function extractPartitions(index: QuadIndex, dataset: string, predicate: string, key: string): VoidPartition[] { + return index + .values(dataset, VOID + predicate) + .map((partition) => ({ + iri: index.values(partition, VOID + key)[0], + entities: index.number(partition, VOID + "entities"), + triples: index.number(partition, VOID + "triples"), + })) + .filter((p, i, all) => !!p.iri && all.findIndex((other) => other.iri === p.iri) === i) + .sort((a, b) => (b.entities ?? b.triples ?? 0) - (a.entities ?? a.triples ?? 0)) + .slice(0, MAX_ITEMS); +} + +export function extractVoidDatasets(quads: N3.Quad[]): VoidDataset[] { + const index = new QuadIndex(quads); + // Partitions are also typed void:Dataset; exclude those to only keep top-level datasets + const partitions = new Set( + quads + .filter((q) => q.predicate.value === VOID + "classPartition" || q.predicate.value === VOID + "propertyPartition") + .map((q) => q.object.value), + ); + const candidates = unique([ + ...index.subjectsOfType(VOID + "Dataset"), + ...index.subjectsWith(VOID + "triples"), + ...index.subjectsWith(VOID + "sparqlEndpoint"), + ]).filter((iri) => !partitions.has(iri) && index.values(iri, RDF_TYPE).indexOf(VOID + "Linkset") < 0); + + return candidates.map((iri) => ({ + iri, + title: index.values(iri, DCTERMS + "title")[0] || index.values(iri, RDFS_LABEL)[0], + triples: index.number(iri, VOID + "triples"), + entities: index.number(iri, VOID + "entities"), + classes: index.number(iri, VOID + "classes"), + properties: index.number(iri, VOID + "properties"), + distinctSubjects: index.number(iri, VOID + "distinctSubjects"), + distinctObjects: index.number(iri, VOID + "distinctObjects"), + sparqlEndpoints: index.values(iri, VOID + "sparqlEndpoint"), + vocabularies: unique(index.values(iri, VOID + "vocabulary")), + classPartitions: extractPartitions(index, iri, "classPartition", "class"), + propertyPartitions: extractPartitions(index, iri, "propertyPartition", "property"), + linksets: index + .subjectsWith(VOID + "subjectsTarget") + .filter((ls) => index.values(ls, VOID + "subjectsTarget").indexOf(iri) >= 0) + .map((ls) => ({ + iri: ls, + target: index.values(ls, VOID + "objectsTarget")[0], + triples: index.number(ls, VOID + "triples"), + })), + })); +} + +async function fetchRdf(url: string, init: FetchInit): Promise { + const response = await fetch(url, { + method: "GET", + headers: { ...(init.headers || {}), Accept: RDF_ACCEPT }, + credentials: init.withCredentials ? "include" : "same-origin", + mode: "cors", + signal: init.signal, + }); + if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`.trim()); + const content = await response.text(); + if (!content.trim()) throw new Error("Empty response"); + return parseRdf(content, response.headers.get("Content-Type"), response.url || url); +} + +export function getWellKnownVoidUrl(endpoint: string): string | undefined { + try { + return new URL("/.well-known/void", endpoint).toString(); + } catch { + return undefined; + } +} + +function errorMessage(e: unknown): string { + if (e instanceof Error) { + if (e.name === "TypeError") return "Not reachable (network or CORS error)"; + return e.message; + } + return String(e); +} + +/** + * Fetch the service description and VoID description of an endpoint. + * Never throws (except when aborted): unavailable sources are flagged in the result. + */ +export async function fetchEndpointMetadata(endpoint: string, init: FetchInit = {}): Promise { + const metadata: EndpointMetadata = { + fetchedAt: Date.now(), + sdStatus: "unavailable", + voidStatus: "unavailable", + datasets: [], + }; + const isAbort = (e: unknown) => e instanceof Error && e.name === "AbortError"; + + const voidUrl = getWellKnownVoidUrl(endpoint); + const [sdResult, voidResult] = await Promise.allSettled([ + fetchRdf(endpoint, init), + voidUrl ? fetchRdf(voidUrl, init) : Promise.reject(new Error("Invalid endpoint URL")), + ]); + if (init.signal?.aborted) throw new DOMException("Aborted", "AbortError"); + + let sdQuads: N3.Quad[] = []; + if (sdResult.status === "fulfilled") { + sdQuads = sdResult.value; + metadata.sd = extractServiceDescription(sdQuads); + if (metadata.sd) metadata.sdStatus = "ok"; + else metadata.sdError = "No service description found in the response"; + } else if (!isAbort(sdResult.reason)) { + metadata.sdError = errorMessage(sdResult.reason); + } + + let voidQuads: N3.Quad[] = []; + if (voidResult.status === "fulfilled") { + voidQuads = voidResult.value; + } else if (!isAbort(voidResult.reason)) { + metadata.voidError = errorMessage(voidResult.reason); + } + // A service description may embed VoID statistics as well + metadata.datasets = extractVoidDatasets([...sdQuads, ...voidQuads]); + if (metadata.datasets.length > 0) { + metadata.voidStatus = "ok"; + delete metadata.voidError; + } else if (!metadata.voidError) { + metadata.voidError = "No VoID dataset description found"; + } + return metadata; +} diff --git a/packages/yasgui/src/index.ts b/packages/yasgui/src/index.ts index 08b0ef32..48db9d28 100644 --- a/packages/yasgui/src/index.ts +++ b/packages/yasgui/src/index.ts @@ -18,6 +18,7 @@ import "@matdata/yasgui-graph-plugin/dist/yasgui-graph-plugin.min.css"; import "@matdata/yasgui-table-plugin/dist/yasgui-table-plugin.min.css"; import { ThemeManager, Theme } from "./ThemeManager"; import QueryBrowser from "./queryManagement/QueryBrowser"; +import EndpointDescribePanel, { EndpointDescribeConfig } from "./endpointDescribe/EndpointDescribePanel"; import "./index.scss"; import "./themes.scss"; import "./github-dark-theme.scss"; @@ -109,6 +110,10 @@ export interface Config { * Show code snippets bar in all tabs (global setting) */ showSnippetsBar?: boolean; + /** + * "Describe endpoint" panel: service description, VoID and overview queries of the current endpoint + */ + endpointDescribe: EndpointDescribeConfig; } export type PartialConfig = { [P in keyof Config]?: Config[P] extends object ? Partial : Config[P]; @@ -157,6 +162,8 @@ export class Yasgui extends EventEmitter { public persistentConfig: PersistentConfig; public themeManager: ThemeManager; public queryBrowser: QueryBrowser; + public endpointDescribe: EndpointDescribePanel | undefined; + public mainEl: HTMLDivElement; private recentTabIds: string[] = []; private navigationSnapshot: string[] | null = null; private navigationCursor = 0; @@ -194,6 +201,11 @@ export class Yasgui extends EventEmitter { this.tabElements = new TabElements(this); this.tabPanelsEl = document.createElement("div"); + addClass(this.tabPanelsEl, "yasgui-tabPanels"); + // Tab panels and the docked endpoint describe panel sit side by side + this.mainEl = document.createElement("div"); + addClass(this.mainEl, "yasgui-main"); + this.mainEl.appendChild(this.tabPanelsEl); this.queryBrowser = new QueryBrowser(this); this.on("tabClose", (_yasgui, tab) => { @@ -201,7 +213,7 @@ export class Yasgui extends EventEmitter { }); this.rootEl.appendChild(this.tabElements.drawTabsList()); - this.rootEl.appendChild(this.tabPanelsEl); + this.rootEl.appendChild(this.mainEl); this.rootEl.appendChild(this.queryBrowser.getElement()); let executeIdAfterInit: string | undefined; let optionsFromUrl: PersistedTabJson | undefined; @@ -259,6 +271,10 @@ export class Yasgui extends EventEmitter { // } } } + if (this.config.endpointDescribe?.enabled) { + this.endpointDescribe = new EndpointDescribePanel(this); + this.mainEl.appendChild(this.endpointDescribe.getElement()); + } } public hasFullscreen(fullscreen: boolean) { if (fullscreen) { @@ -503,6 +519,7 @@ export class Yasgui extends EventEmitter { } public destroy() { this.removeAllListeners(); + this.endpointDescribe?.destroy(); this.tabElements.destroy(); for (const tabId in this._tabs) { const tab = this._tabs[tabId]; diff --git a/packages/yasqe/src/sparql.ts b/packages/yasqe/src/sparql.ts index bda1b717..2590e129 100644 --- a/packages/yasqe/src/sparql.ts +++ b/packages/yasqe/src/sparql.ts @@ -181,8 +181,23 @@ export interface ExecuteQueryOptions { * Useful for background/plugin-driven queries that should not update main UI state. */ silent?: boolean; + /** + * Do not send the default/named graph arguments configured for the tab. + * Useful for queries that should describe the whole endpoint. + */ + skipGraphArgs?: boolean; } +const GRAPH_ARG_NAMES = [ + "default-graph-uri", + "named-graph-uri", + "using-graph-uri", + "using-named-graph-uri", + // getUrlArguments uses these (trailing space) names for update queries + "using-graph-uri ", + "using-named-graph-uri ", +]; + export async function executeQuery( yasqe: Yasqe, config?: YasqeAjaxConfig, @@ -256,10 +271,10 @@ export async function executeQuery( searchParams.append("query", options.customQuery); // Add other args except the query/update parameter - appendArgsToParams(populatedConfig.args, ["query", "update"]); + appendArgsToParams(populatedConfig.args, ["query", "update", ...(options.skipGraphArgs ? GRAPH_ARG_NAMES : [])]); } else { // Add all args from config - appendArgsToParams(populatedConfig.args); + appendArgsToParams(populatedConfig.args, options?.skipGraphArgs ? GRAPH_ARG_NAMES : []); } if (populatedConfig.reqMethod === "POST") { diff --git a/test/endpoint-describe-browser.ts b/test/endpoint-describe-browser.ts new file mode 100644 index 00000000..9669929e --- /dev/null +++ b/test/endpoint-describe-browser.ts @@ -0,0 +1,197 @@ +import * as path from "path"; +import * as http from "http"; +import * as chai from "chai"; +import * as puppeteer from "puppeteer"; +import { it, describe, before, beforeEach, after, afterEach } from "mocha"; +import { setup, destroy, closePage, getPage } from "./utils.js"; + +const expect = chai.expect; + +const ENDPOINT = "http://describe.test/sparql"; +const OTHER_ENDPOINT = "http://other.test/sparql"; + +const SD_TURTLE = ` +@prefix sd: . +@prefix void: . +<${ENDPOINT}#service> a sd:Service ; + sd:endpoint <${ENDPOINT}> ; + sd:supportedLanguage sd:SPARQL11Query ; + sd:defaultDataset <${ENDPOINT}#dataset> . +<${ENDPOINT}#dataset> a void:Dataset ; void:triples 4242 . +`; + +function sparqlJson(vars: string[], rows: string[][]) { + return JSON.stringify({ + head: { vars }, + results: { + bindings: rows.map((row) => + Object.fromEntries( + row.map((value, i) => [ + vars[i], + value.startsWith("http") ? { type: "uri", value } : { type: "literal", value }, + ]), + ), + ), + }, + }); +} + +/** + * Serve a fake SPARQL endpoint through request interception + */ +async function mockEndpoints(page: puppeteer.Page, queries: string[]) { + await page.setRequestInterception(true); + page.on("request", (request) => { + const url = new URL(request.url()); + if (url.host !== "describe.test" && url.host !== "other.test") { + void request.continue(); + return; + } + const headers = { "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "*" }; + if (request.method() === "OPTIONS") { + void request.respond({ status: 204, headers }); + return; + } + const params = new URLSearchParams(request.method() === "POST" ? request.postData() || "" : url.search); + const query = params.get("query"); + if (!query) { + if (url.pathname === "/sparql" && url.host === "describe.test") { + void request.respond({ status: 200, headers, contentType: "text/turtle", body: SD_TURTLE }); + } else { + void request.respond({ status: 404, headers, contentType: "text/plain", body: "Not found" }); + } + return; + } + queries.push(query); + let body = sparqlJson(["s"], [["http://example.org/main-result"]]); + if (/\?instance a \?class/.test(query)) { + body = sparqlJson( + ["class", "instances"], + [ + ["http://xmlns.com/foaf/0.1/Person", "12345"], + ["http://example.org/Thing", "3"], + ], + ); + } + void request.respond({ status: 200, headers, contentType: "application/sparql-results+json", body }); + }); +} + +async function waitUntil(condition: () => boolean, timeout = 5000) { + const start = Date.now(); + while (!condition()) { + if (Date.now() - start > timeout) throw new Error("Timed out waiting for condition"); + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + +describe("Endpoint describe panel", function () { + let browser: puppeteer.Browser; + let page: puppeteer.Page; + let server: http.Server | undefined; + let queries: string[]; + + before(async function () { + const refs = await setup(this, path.resolve("./build")); + browser = refs.browser; + server = refs.server; + }); + + beforeEach(async function () { + this.timeout(30000); + queries = []; + page = await getPage(browser, "yasgui.html"); + await page.evaluate(() => localStorage.clear()); + await page.reload({ waitUntil: "networkidle2" }); + await mockEndpoints(page, queries); + await page.evaluate((endpoint) => (window as any).yasgui.getTab().setEndpoint(endpoint), ENDPOINT); + }); + + afterEach(async () => { + await closePage(this, page); + }); + + after(async function () { + return destroy(browser, server); + }); + + async function openPanel() { + await page.click(".tabPanel.active .describeEndpointButton"); + await page.waitForSelector(".yasgui-describe.open .yasgui-describe__drawer", { visible: true }); + } + + async function runClasses() { + await page.click('.yasgui-describe__query[data-query-id="classes"] .yasgui-describe__icon-button'); + await page.waitForSelector('.yasgui-describe__query[data-query-id="classes"] table'); + } + + it("shows the service description and VoID statistics", async function () { + await openPanel(); + await page.waitForFunction( + () => + document.querySelector(".yasgui-describe__metadata")?.textContent?.includes("Service description: available"), + { timeout: 10000 }, + ); + const text = await page.$eval(".yasgui-describe__metadata", (el) => el.textContent || ""); + expect(text).to.contain("4,242"); + expect(await page.$eval(".yasgui-describe__endpoint", (el) => el.textContent)).to.equal(ENDPOINT); + }); + + it("keeps results when a regular query is executed", async function () { + await openPanel(); + await runClasses(); + const table = await page.$eval('.yasgui-describe__query[data-query-id="classes"] table', (el) => el.textContent); + expect(table).to.contain("foaf:Person"); + expect(table).to.contain("12,345"); + // The editor isn't touched by describe queries + const editorValue = await page.evaluate(() => (window as any).yasgui.getTab().getQuery()); + expect(editorValue).not.to.contain("?instance a ?class"); + + await page.evaluate(() => (window as any).yasgui.getTab().query()); + await waitUntil(() => queries.length >= 2); + expect(await page.$('.yasgui-describe__query[data-query-id="classes"] table')).to.not.equal(null); + }); + + it("inserts a prefixed IRI into the query when clicked", async function () { + await openPanel(); + await runClasses(); + await page.evaluate(() => (window as any).yasgui.getTab().setQuery("SELECT * WHERE { ?s a }")); + await page.click('.yasgui-describe__query[data-query-id="classes"] .yasgui-describe__iri'); + const value = await page.evaluate(() => (window as any).yasgui.getTab().getQuery()); + expect(value).to.contain("PREFIX foaf: "); + expect(value).to.contain("foaf:Person"); + }); + + it("follows the endpoint of the active tab", async function () { + await openPanel(); + await runClasses(); + await page.evaluate((endpoint) => (window as any).yasgui.getTab().setEndpoint(endpoint), OTHER_ENDPOINT); + await page.waitForFunction( + (endpoint) => document.querySelector(".yasgui-describe__endpoint")?.textContent === endpoint, + {}, + OTHER_ENDPOINT, + ); + // No results cached for the other endpoint + expect(await page.$('.yasgui-describe__query[data-query-id="classes"] table')).to.equal(null); + // Switching back shows the cached results again + await page.evaluate((endpoint) => (window as any).yasgui.getTab().setEndpoint(endpoint), ENDPOINT); + await page.waitForSelector('.yasgui-describe__query[data-query-id="classes"] table'); + }); + + it("restores a pinned panel and its results after a reload", async function () { + await openPanel(); + await runClasses(); + await page.click(".yasgui-describe__pin"); + await page.reload({ waitUntil: "networkidle2" }); + await page.waitForSelector(".yasgui-describe.open.pinned .yasgui-describe__drawer", { visible: true }); + await page.waitForSelector('.yasgui-describe__query[data-query-id="classes"] table'); + }); + + it("collapses an unpinned panel when the editor gets focus", async function () { + await openPanel(); + await page.click(".tabPanel.active .yasqe .cm-content"); + await page.waitForSelector(".yasgui-describe.collapsed .yasgui-describe__rail", { visible: true }); + await page.click(".yasgui-describe__rail"); + await page.waitForSelector(".yasgui-describe:not(.collapsed) .yasgui-describe__drawer", { visible: true }); + }); +}); diff --git a/test/run.ts b/test/run.ts index 1621b83d..2acdde1b 100644 --- a/test/run.ts +++ b/test/run.ts @@ -9,6 +9,7 @@ const expect = chai.expect; import Yasqe from "@matdata/yasqe"; //@ts-ignore ignore unused warning import { setup, destroy, closePage, getPage, makeScreenshot, inspectLive, wait } from "./utils.js"; +import "./endpoint-describe-browser.js"; declare var window: Window & { Yasqe: typeof Yasqe; diff --git a/test/unit/endpoint-describe-test.ts b/test/unit/endpoint-describe-test.ts new file mode 100644 index 00000000..c635aa73 --- /dev/null +++ b/test/unit/endpoint-describe-test.ts @@ -0,0 +1,311 @@ +import * as chai from "chai"; +import { describe, it } from "mocha"; + +import { + buildQuery, + defaultCategories, + defaultDescribeQueries, + DescribeQueryContext, + wktExtent, +} from "../../packages/yasgui/src/endpointDescribe/describeQueries.js"; +import { + extractServiceDescription, + extractVoidDatasets, + fetchEndpointMetadata, + getWellKnownVoidUrl, + parseRdf, +} from "../../packages/yasgui/src/endpointDescribe/metadataSources.js"; +import DescribeStore, { + MAX_ENDPOINTS, + MAX_PERSISTED_ROWS, + StoredDescribeState, +} from "../../packages/yasgui/src/endpointDescribe/DescribeStore.js"; + +const expect = chai.expect; + +function context(overrides: Partial = {}): DescribeQueryContext { + return { limit: 25, offset: 0, getResult: () => undefined, ...overrides }; +} + +function stripStrings(query: string) { + return query.replace(/"(?:[^"\\]|\\.)*"/g, '""'); +} + +describe("Endpoint describe queries", () => { + it("has unique ids and only uses known categories", () => { + const ids = defaultDescribeQueries.map((q) => q.id); + expect(new Set(ids).size).to.equal(ids.length); + const categories = defaultCategories.map((c) => c.id); + for (const query of defaultDescribeQueries) { + expect(categories, query.id).to.include(query.category); + } + // Every category from the issue has at least one query + for (const category of categories) { + expect( + defaultDescribeQueries.some((q) => q.category === category), + String(category), + ).to.equal(true); + } + }); + + it("builds well-formed, bounded queries", () => { + for (const query of defaultDescribeQueries) { + const text = buildQuery(query, context()); + const withoutStrings = stripStrings(text); + expect(withoutStrings.split("{").length, `${query.id}: braces`).to.equal(withoutStrings.split("}").length); + expect(withoutStrings.split("(").length, `${query.id}: parentheses`).to.equal(withoutStrings.split(")").length); + expect(text, query.id).to.match(/^(PREFIX [\w-]+: <[^>]+>\n)*SELECT /); + const isSingleAggregate = !/GROUP BY/.test(text) && /SELECT \(/.test(text); + if (!isSingleAggregate) expect(text, `${query.id}: needs a LIMIT`).to.match(/LIMIT \d+/); + // Every prefixed name that is used is declared + const declared = Array.from(text.matchAll(/PREFIX ([\w-]+):/g)).map((m) => m[1]); + const body = withoutStrings.replace(/PREFIX [\w-]+: <[^>]+>/g, "").replace(/<[^>]*>/g, ""); + for (const match of body.matchAll(/\b([a-z][\w-]*):[A-Za-z_]/g)) { + expect(declared, `${query.id}: prefix ${match[1]}`).to.include(match[1]); + } + } + }); + + it("paginates with LIMIT and OFFSET", () => { + for (const query of defaultDescribeQueries.filter((q) => q.paginated)) { + const firstPage = buildQuery(query, context({ limit: 10, offset: 0 })); + const secondPage = buildQuery(query, context({ limit: 10, offset: 20 })); + expect(firstPage, query.id).to.match(/LIMIT 10$/); + expect(secondPage, query.id).to.match(/LIMIT 10 OFFSET 20$/); + } + }); + + it("builds per-class samples from the classes result", () => { + const query = defaultDescribeQueries.find((q) => q.id === "class-samples")!; + expect(query.dependsOn).to.equal("classes"); + const text = buildQuery( + query, + context({ + getResult: (id) => + id === "classes" + ? { + vars: ["class", "instances"], + bindings: [ + { class: { type: "uri", value: "http://example.org/A" } }, + { class: { type: "uri", value: "http://example.org/B>" } }, + ], + } + : undefined, + }), + ); + expect(text).to.contain("?instance a "); + // IRIs are sanitized before being inlined + expect(text).to.contain(""); + expect(text).not.to.contain("B>>"); + expect(text.match(/UNION/g)?.length).to.equal(1); + }); + + it("computes a bounding box from WKT literals", () => { + const table = wktExtent({ + vars: ["wkt"], + bindings: [ + { wkt: { type: "literal", value: "POINT(4.35 50.85)" } }, + { + wkt: { + type: "literal", + value: " LINESTRING(1 2, 3 4 100)", + }, + }, + ], + }); + const row = table.bindings[0]; + expect(row.minX.value).to.equal("1"); + expect(row.minY.value).to.equal("2"); + expect(row.maxX.value).to.equal("4.35"); + expect(row.maxY.value).to.equal("50.85"); + expect(row.sampledGeometries.value).to.equal("2"); + expect(row.crs.value).to.contain("EPSG/0/4326"); + }); +}); + +const SD_TURTLE = ` +@prefix sd: . +@prefix void: . +@prefix dcterms: . + a sd:Service ; + sd:endpoint ; + sd:supportedLanguage sd:SPARQL11Query ; + sd:feature sd:UnionDefaultGraph ; + sd:extensionFunction ; + sd:resultFormat ; + sd:defaultDataset [ + sd:namedGraph [ sd:name ] , [ sd:name ] + ] . + a void:Dataset ; + dcterms:title "Example" ; + void:triples 1234 ; + void:classes 2 ; + void:vocabulary ; + void:classPartition [ void:class ; void:entities 10 ] , + [ void:class ; void:entities 50 ] ; + void:propertyPartition [ void:property ; void:triples 60 ] . +`; + +describe("Endpoint describe metadata (SD / VoID)", () => { + it("extracts the service description", () => { + const quads = parseRdf(SD_TURTLE, "text/turtle", "https://example.org/sparql"); + const sd = extractServiceDescription(quads)!; + expect(sd.languages).to.deep.equal(["http://www.w3.org/ns/sparql-service-description#SPARQL11Query"]); + expect(sd.features).to.deep.equal(["http://www.w3.org/ns/sparql-service-description#UnionDefaultGraph"]); + expect(sd.extensionFunctions).to.deep.equal(["http://jena.apache.org/text#query"]); + expect(sd.namedGraphs).to.have.members(["https://example.org/graph/1", "https://example.org/graph/2"]); + }); + + it("extracts VoID datasets and sorts partitions", () => { + const quads = parseRdf(SD_TURTLE, "text/turtle", "https://example.org/sparql"); + const datasets = extractVoidDatasets(quads); + expect(datasets).to.have.length(1); + const dataset = datasets[0]; + expect(dataset.title).to.equal("Example"); + expect(dataset.triples).to.equal(1234); + expect(dataset.classes).to.equal(2); + expect(dataset.vocabularies).to.deep.equal(["http://xmlns.com/foaf/0.1/"]); + expect(dataset.classPartitions.map((p) => p.iri)).to.deep.equal([ + "http://xmlns.com/foaf/0.1/Organization", + "http://xmlns.com/foaf/0.1/Person", + ]); + expect(dataset.propertyPartitions[0]).to.deep.include({ iri: "http://xmlns.com/foaf/0.1/name", triples: 60 }); + }); + + it("does not duplicate partitions described in both the SD and VoID", () => { + const quads = [ + ...parseRdf(SD_TURTLE, "text/turtle", "https://example.org/sparql"), + ...parseRdf(SD_TURTLE, "text/turtle", "https://example.org/.well-known/void"), + ]; + const datasets = extractVoidDatasets(quads); + expect(datasets).to.have.length(1); + expect(datasets[0].classPartitions).to.have.length(2); + }); + + it("rejects HTML responses", () => { + expect(() => parseRdf("", "text/html; charset=utf-8", "https://example.org/")).to.throw(); + }); + + it("builds the well-known VoID URL", () => { + expect(getWellKnownVoidUrl("https://example.org/repositories/test/sparql")).to.equal( + "https://example.org/.well-known/void", + ); + expect(getWellKnownVoidUrl("not a url")).to.equal(undefined); + }); + + it("reports unavailable sources without throwing (e.g. GraphDB without SD)", async () => { + const originalFetch = globalThis.fetch; + const requested: string[] = []; + try { + globalThis.fetch = async (input: any) => { + requested.push(String(input)); + if (String(input).endsWith("/.well-known/void")) return new Response("Not found", { status: 404 }); + return new Response("SPARQL form", { + status: 200, + headers: { "Content-Type": "text/html" }, + }); + }; + const metadata = await fetchEndpointMetadata("https://example.org/repositories/test"); + expect(requested).to.have.members([ + "https://example.org/repositories/test", + "https://example.org/.well-known/void", + ]); + expect(metadata.sdStatus).to.equal("unavailable"); + expect(metadata.voidStatus).to.equal("unavailable"); + expect(metadata.voidError).to.contain("404"); + expect(metadata.datasets).to.deep.equal([]); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("combines the service description and VoID", async () => { + const originalFetch = globalThis.fetch; + try { + globalThis.fetch = async (input: any, init: any) => { + expect(init.headers.Accept).to.contain("text/turtle"); + expect(init.headers.Authorization).to.equal("Bearer token"); + if (String(input).endsWith("/.well-known/void")) return new Response("Not found", { status: 404 }); + return new Response(SD_TURTLE, { status: 200, headers: { "Content-Type": "text/turtle" } }); + }; + const metadata = await fetchEndpointMetadata("https://example.org/sparql", { + headers: { Authorization: "Bearer token" }, + }); + expect(metadata.sdStatus).to.equal("ok"); + // VoID embedded in the service description is used as well + expect(metadata.voidStatus).to.equal("ok"); + expect(metadata.datasets[0].triples).to.equal(1234); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); + +describe("Endpoint describe store", () => { + function memoryStorage() { + let stored: StoredDescribeState | undefined; + return { + get: () => (stored ? JSON.parse(JSON.stringify(stored)) : undefined), + set: (state: StoredDescribeState) => { + stored = JSON.parse(JSON.stringify(state)); + }, + raw: () => stored, + }; + } + + it("persists results and UI state", () => { + const storage = memoryStorage(); + const store = new DescribeStore(storage); + store.setUiState({ open: true, pinned: true, width: 500 }); + store.setResult("https://example.org/sparql", "classes", { + status: "done", + vars: ["class"], + bindings: [{ class: { type: "uri", value: "http://example.org/A" } }], + fetchedAt: 1, + }); + const restored = new DescribeStore(storage); + expect(restored.getUiState()).to.include({ open: true, pinned: true, width: 500 }); + expect(restored.getResult("https://example.org/sparql", "classes")?.bindings).to.have.length(1); + }); + + it("truncates large results when persisting", () => { + const storage = memoryStorage(); + const store = new DescribeStore(storage); + const bindings = Array.from({ length: MAX_PERSISTED_ROWS + 50 }, (_, i) => ({ + s: { type: "literal", value: String(i) }, + })); + store.setResult("https://example.org/sparql", "big", { status: "done", vars: ["s"], bindings, fetchedAt: 1 }); + // In memory, all rows are kept + expect(store.getResult("https://example.org/sparql", "big")?.bindings).to.have.length(MAX_PERSISTED_ROWS + 50); + const persisted = storage.raw()!.endpoints["https://example.org/sparql"].results.big; + expect(persisted.bindings).to.have.length(MAX_PERSISTED_ROWS); + expect(persisted.truncated).to.equal(true); + }); + + it("evicts the least recently used endpoints", async () => { + const storage = memoryStorage(); + const store = new DescribeStore(storage); + for (let i = 0; i < MAX_ENDPOINTS + 3; i++) { + store.setResult(`https://example.org/${i}`, "q", { status: "done", vars: [], bindings: [], fetchedAt: i }); + // Make sure lastUsed differs + await new Promise((resolve) => setTimeout(resolve, 2)); + } + const endpoints = Object.keys(storage.raw()!.endpoints); + expect(endpoints).to.have.length(MAX_ENDPOINTS); + expect(endpoints).not.to.include("https://example.org/0"); + expect(endpoints).to.include(`https://example.org/${MAX_ENDPOINTS + 2}`); + }); + + it("keeps working when storage fails", () => { + const store = new DescribeStore({ + get: () => { + throw new Error("broken"); + }, + set: () => { + throw new Error("quota"); + }, + }); + store.setResult("https://example.org/sparql", "q", { status: "done", vars: [], bindings: [], fetchedAt: 1 }); + expect(store.getResult("https://example.org/sparql", "q")).to.not.equal(undefined); + }); +}); diff --git a/test/unit/yasqe-background-query-test.ts b/test/unit/yasqe-background-query-test.ts index 3c5a5406..9cc3e68f 100644 --- a/test/unit/yasqe-background-query-test.ts +++ b/test/unit/yasqe-background-query-test.ts @@ -134,4 +134,48 @@ describe("Yasqe background query execution", () => { globalThis.fetch = originalFetch; } }); + + it("omits the tab's graph arguments when skipGraphArgs is set", async () => { + const originalFetch = globalThis.fetch; + const bodies: string[] = []; + + try { + globalThis.fetch = async (input: any) => { + bodies.push(await (input as Request).text()); + return new Response('{"head":{"vars":[]},"results":{"bindings":[]}}', { + status: 200, + headers: { "Content-Type": "application/sparql-results+json" }, + }); + }; + + const yasqeMock = { + config: { requestConfig: {} }, + emit: () => true, + getQueryMode: () => "query", + getQueryType: () => "SELECT", + getValue: () => "SELECT * WHERE { ?s ?p ?o } LIMIT 1", + } as any; + const config = { + endpoint: "https://example.org/sparql", + method: "POST", + queryArgument: "query", + acceptHeaderSelect: "application/sparql-results+json", + defaultGraphs: ["https://example.org/g1"], + namedGraphs: ["https://example.org/g2"], + } as any; + + await executeQuery(yasqeMock, config, { customQuery: "ASK {}", silent: true }); + await executeQuery(yasqeMock, config, { customQuery: "ASK {}", silent: true, skipGraphArgs: true }); + + const withGraphs = new URLSearchParams(bodies[0]); + const withoutGraphs = new URLSearchParams(bodies[1]); + expect(withGraphs.get("default-graph-uri")).to.equal("https://example.org/g1"); + expect(withGraphs.get("named-graph-uri")).to.equal("https://example.org/g2"); + expect(withoutGraphs.get("default-graph-uri")).to.equal(null); + expect(withoutGraphs.get("named-graph-uri")).to.equal(null); + expect(withoutGraphs.get("query")).to.equal("ASK {}"); + } finally { + globalThis.fetch = originalFetch; + } + }); }); From 62e8a46da0ea9e6c393f9ae125fb2b4efce38318 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 22:00:06 +0000 Subject: [PATCH 2/4] docs: document the endpoint overview panel in README, intro and API reference Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DEbQuDzCkbQKKsARAXTpKu --- README.md | 2 ++ docs/developer-guide.md | 41 +++++++++++++++++++++++++++++++++++++++++ website/docs/intro.md | 1 + 3 files changed, 44 insertions(+) diff --git a/README.md b/README.md index f0276abf..86b6a2ae 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ Matgui provides a complete SPARQL development environment with powerful features - **[Query Formatting](https://github.com/Matdata-eu/Matgui/blob/main/docs/user-guide.md#query-formatting)** - One-click query beautification with configurable formatters - **[Prefix Management](https://github.com/Matdata-eu/Matgui/blob/main/docs/user-guide.md#prefix-management)** - Auto-capture and reuse PREFIX declarations - **[URI Explorer](https://github.com/Matdata-eu/Matgui/blob/main/docs/user-guide.md#uri-explorer)** - Ctrl+Click URIs to explore connections +- **[Endpoint Overview](https://github.com/Matdata-eu/Matgui/blob/main/docs/user-guide.md#endpoint-overview)** - Explore an unfamiliar endpoint: service description, VoID and overview queries (classes, properties, graphs, languages, links, geo) in a side panel - **[Keyboard Shortcuts](https://github.com/Matdata-eu/Matgui/blob/main/docs/user-guide.md#keyboard-shortcuts)** - Efficient query development workflow ### 📊 Powerful Visualizations @@ -248,6 +249,7 @@ const yasgui = new Yasgui(document.getElementById('yasgui'), { theme: 'dark', // 'light' or 'dark' orientation: 'horizontal', // 'horizontal' or 'vertical' showSnippetsBar: true, // Show code snippets + endpointDescribe: { enabled: true }, // "Describe endpoint" panel (F8) // Persistence persistenceId: 'my-yasgui-instance', // Custom storage ID diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 66f87c59..e9b81cd5 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -70,6 +70,7 @@ This comprehensive guide covers everything developers need to know to integrate, - [`setTheme(theme: 'light' | 'dark'): void`](#setthemetheme-light--dark-void) - [`getTheme(): 'light' | 'dark'`](#gettheme-light--dark) - [`toggleTheme(): 'light' | 'dark'`](#toggletheme-light--dark) + - [`endpointDescribe: EndpointDescribePanel | undefined`](#endpointdescribe-endpointdescribepanel--undefined) - [Tab Class](#tab-class) - [Methods](#methods-1) - [`getName(): string`](#getname-string) @@ -80,6 +81,8 @@ This comprehensive guide covers everything developers need to know to integrate, - [`query(): Promise`](#query-promisevoid) - [`setQuery(query: string): void`](#setqueryquery-string-void) - [`getQuery(): string`](#getquery-string) + - [`runBackgroundQuery(query: string, options?): Promise`](#runbackgroundqueryquery-string-options-promiseany) + - [`getRequestInit(): Promise<{ headers, withCredentials } | undefined>`](#getrequestinit-promise-headers-withcredentials---undefined) - [Yasqe Class](#yasqe-class) - [Methods](#methods-2) - [`getValue(): string`](#getvalue-string) @@ -2231,6 +2234,18 @@ const newTheme = yasgui.toggleTheme(); console.log('Switched to:', newTheme); ``` +##### `endpointDescribe: EndpointDescribePanel | undefined` + +The endpoint overview panel (see [Endpoint Describe Configuration](#endpoint-describe-configuration)). It is `undefined` when the panel is disabled with `endpointDescribe: { enabled: false }`. + +```javascript +yasgui.endpointDescribe?.open(); // open (or expand) the panel +yasgui.endpointDescribe?.setPinned(true); // keep it open, also after a reload +await yasgui.endpointDescribe?.runQuery("classes"); // run a describe query for the current endpoint +yasgui.endpointDescribe?.collapse(); // collapse it to a thin bar +yasgui.endpointDescribe?.close(); +``` + ### Tab Class Represents a query tab. @@ -2306,6 +2321,32 @@ const query = tab.getQuery(); console.log('Current query:', query); ``` +##### `runBackgroundQuery(query: string, options?): Promise` + +Execute a SPARQL query against the tab's endpoint without changing the editor or the results view. The tab's request configuration and authentication are used (an expired OAuth 2.0 token is refreshed first) and no query events are emitted. Rejects on HTTP errors. + +Options: `accept` (Accept header), `signal` (an `AbortSignal`) and `skipGraphArgs` (don't send the tab's default/named graphs). + +```javascript +const response = await tab.runBackgroundQuery("SELECT (COUNT(*) AS ?n) WHERE { ?s ?p ?o }", { + accept: "application/sparql-results+json", + skipGraphArgs: true, +}); +const count = JSON.parse(response.content).results.bindings[0].n.value; +``` + +##### `getRequestInit(): Promise<{ headers, withCredentials } | undefined>` + +Get the request headers (including authentication headers) and credentials mode used for requests to the tab's endpoint, for example to fetch other resources from the same server. + +```javascript +const init = await tab.getRequestInit(); +const response = await fetch(tab.getEndpoint(), { + headers: { ...init.headers, Accept: "text/turtle" }, + credentials: init.withCredentials ? "include" : "same-origin", +}); +``` + ### Yasqe Class SPARQL query editor. diff --git a/website/docs/intro.md b/website/docs/intro.md index a1ca71c6..a3844bfd 100644 --- a/website/docs/intro.md +++ b/website/docs/intro.md @@ -49,6 +49,7 @@ If you want to **integrate YASGUI** into your application: - Smart autocomplete - Query formatting - Prefix management +- Endpoint overview (service description, VoID and exploration queries) 📊 **Powerful Visualizations** - Interactive tables From 1cfd044f915fc3b5aefe0de7f8734a7977b9ba23 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 23:34:12 +0000 Subject: [PATCH 3/4] fix: harden endpoint describe panel against real-world endpoints Findings from running the panel against DBpedia, Wikidata and data.europa.eu: - The namespaces query used a REPLACE() pattern that matches the empty string, which Virtuoso rejects. Use "[^#/]+$" and add a regression test. - Virtuoso "anytime queries" (DBpedia) answer HTTP 206 / X-SQL-State S1TAT with incomplete, often empty results. Such results are now marked as partial instead of looking like "no data". - Error pages are summarized: HTML bodies (Wikimedia 429, gateway 504) are reduced to their title, 429 explains the rate limiting (with Retry-After), and Java stack traces (Blazegraph) are reduced to their root cause. - Wikidata's service description is ~19 MB. SD/VoID documents are now read up to 5 MB, parsed up to the last complete statement and flagged as truncated; partition de-duplication no longer is quadratic. - Blank-node VoID datasets are no longer shown as IRIs and datasets without any information are dropped. - SKOS concept schemes is flagged as possibly slow (timed out on data.europa.eu). - runQuery() now uses the endpoint of the active tab even when the panel is closed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DEbQuDzCkbQKKsARAXTpKu --- docs/user-guide.md | 4 + .../src/endpointDescribe/DescribeStore.ts | 2 + .../EndpointDescribePanel.scss | 9 ++ .../endpointDescribe/EndpointDescribePanel.ts | 31 ++++- .../src/endpointDescribe/describeQueries.ts | 3 +- .../src/endpointDescribe/metadataSources.ts | 103 +++++++++++++--- .../src/endpointDescribe/responseUtils.ts | 75 ++++++++++++ test/endpoint-describe-browser.ts | 44 +++++++ test/unit/endpoint-describe-test.ts | 112 ++++++++++++++++++ 9 files changed, 360 insertions(+), 23 deletions(-) create mode 100644 packages/yasgui/src/endpointDescribe/responseUtils.ts diff --git a/docs/user-guide.md b/docs/user-guide.md index 8e7ecd8c..b329ffc7 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -738,6 +738,8 @@ When the panel opens, MatGUI looks for metadata that the endpoint publishes abou - The [SPARQL Service Description](https://www.w3.org/TR/sparql11-service-description/), returned by the endpoint URL when it is requested without a query. It lists the supported SPARQL features, extension functions, result formats and named graphs. - A [VoID](https://www.w3.org/TR/void/) dataset description at `/.well-known/void` (or embedded in the service description), with statistics such as the number of triples, classes, vocabularies and class/property partitions. +Some descriptions are very large (Wikidata's includes statistics for every class and property). Only the first 5 MB are read; the panel mentions it when a description was cut off. + Many endpoints publish neither (for example GraphDB has no service description). The panel then simply says so and you can use the overview queries instead. If you run an endpoint yourself, publishing a service description and VoID makes it much easier to explore. **Overview queries** @@ -755,6 +757,8 @@ The panel contains predefined queries, grouped in categories: - Queries only run when you click **Run** (▶) or **Run all** for a category, so large endpoints are never queried unexpectedly. Queries marked **may be slow** scan the whole dataset and may time out on large endpoints. - Every query is limited. Lists such as the named graphs are loaded page by page with **Load more**. +- Some endpoints stop long-running queries at a time limit and return whatever they found so far (for example Virtuoso's "anytime queries", used by DBpedia). Such results are marked as **Partial result**: counts may be too low and an empty result does not mean there is no such data. +- Public endpoints limit how many (expensive) queries you can run. When an endpoint answers that you are sending too many requests, wait a moment before running more queries. Errors of the endpoint are shown below the query (e.g. "estimated execution time exceeds the limit"). - Describe queries run in the background with the endpoint and authentication settings of the current tab. They do not change your query or the results view. The default and named graphs configured for the tab are not applied, so the whole endpoint is described. - Click an IRI in the results to insert it into your query. MatGUI uses a prefixed name when it knows the prefix and adds the missing `PREFIX` declaration. - Click the **Open query in a new tab** button next to a query to open it in a new tab, to adapt it further. diff --git a/packages/yasgui/src/endpointDescribe/DescribeStore.ts b/packages/yasgui/src/endpointDescribe/DescribeStore.ts index 590afc4c..c1cbc3bf 100644 --- a/packages/yasgui/src/endpointDescribe/DescribeStore.ts +++ b/packages/yasgui/src/endpointDescribe/DescribeStore.ts @@ -18,6 +18,8 @@ export interface DescribeResult { hasMore?: boolean; /** True when rows were dropped before persisting */ truncated?: boolean; + /** True when the endpoint returned an incomplete result (e.g. Virtuoso anytime query, HTTP 206) */ + partial?: boolean; } export interface EndpointEntry { diff --git a/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.scss b/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.scss index 23a0724a..72a8e075 100644 --- a/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.scss +++ b/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.scss @@ -321,6 +321,15 @@ margin-top: 4px; } + &__partial { + margin-top: 4px; + padding: 4px 6px; + border-radius: 4px; + background: #fff3cd; + color: #7a5b00; + font-size: 12px; + } + &__loading { margin-top: 4px; font-style: italic; diff --git a/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts b/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts index f27f407e..41c176d1 100644 --- a/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts +++ b/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts @@ -16,6 +16,7 @@ import { DescribeTable, } from "./describeQueries"; import { EndpointMetadata, fetchEndpointMetadata, VoidDataset } from "./metadataSources"; +import { describeQueryError, isPartialResponse } from "./responseUtils"; import "./EndpointDescribePanel.scss"; export interface EndpointDescribeConfig { @@ -186,6 +187,8 @@ export default class EndpointDescribePanel { public runQuery(queryId: string, loadMore = false): Promise { const query = this.queries.find((q) => q.id === queryId); const tab = this.yasgui.getTab(); + // The panel may be closed, so make sure the endpoint of the active tab is used + this.syncEndpoint(tab); const endpoint = this.endpoint; if (!query || !tab || !endpoint) return Promise.resolve(); const key = `${endpoint}\n${queryId}`; @@ -411,6 +414,15 @@ export default class EndpointDescribePanel { if (list.childElementCount) card.appendChild(list); } for (const dataset of metadata.datasets) card.appendChild(this.renderDataset(dataset)); + if (metadata.truncated) { + card.appendChild( + el( + "p", + "yasgui-describe__muted", + "The description is very large: only its first part was read, so some partitions may be missing.", + ), + ); + } if (metadata.sdStatus !== "ok" && metadata.voidStatus !== "ok") { card.appendChild( el( @@ -448,7 +460,8 @@ export default class EndpointDescribePanel { title.appendChild(el("i", "fas fa-database")); title.appendChild(document.createTextNode(" ")); if (dataset.title) title.appendChild(el("strong", undefined, dataset.title + " ")); - title.appendChild(this.renderIri(dataset.iri)); + if (!dataset.blank) title.appendChild(this.renderIri(dataset.iri)); + else if (!dataset.title) title.appendChild(el("strong", undefined, "Dataset")); wrapper.appendChild(title); const stats = el("dl", "yasgui-describe__facts yasgui-describe__facts--stats"); @@ -546,6 +559,14 @@ export default class EndpointDescribePanel { if (result.status === "done") meta.push(`${result.bindings.length.toLocaleString()} rows`); if (result.truncated) meta.push("truncated"); container.appendChild(el("div", "yasgui-describe__query-meta", meta.join(" · "))); + if (result.partial) { + const warning = el( + "div", + "yasgui-describe__partial", + "Partial result: the endpoint stopped this query at its time limit. Counts and lists may be incomplete, and an empty result does not mean there is no such data.", + ); + container.appendChild(warning); + } if (result.error || result.status === "error") { container.appendChild(el("div", "yasgui-describe__error", result.error || "Query failed")); @@ -736,6 +757,7 @@ export default class EndpointDescribePanel { if (!table.vars.length && parser.getBoolean() !== undefined) { table = { vars: ["result"], bindings: [{ result: { type: "literal", value: String(parser.getBoolean()) } }] }; } + const partial = isPartialResponse(response); const hasMore = !!query.paginated && table.bindings.length >= pageSize; if (query.postProcess) table = query.postProcess(table); const result: DescribeResult = { @@ -745,18 +767,17 @@ export default class EndpointDescribePanel { fetchedAt: Date.now(), durationMs: Date.now() - start, hasMore, + partial: partial || undefined, }; this.store.setResult(endpoint, query.id, result); } catch (e: any) { if (running.controller.signal.aborted && !running.timedOut) return; // Cancelled by the user - let message = e instanceof Error ? e.message : String(e); - if (running.timedOut) message = `Timed out after ${Math.round(this.config.timeoutMs / 1000)} s`; - else if (e?.status) message = `HTTP ${e.status}${e.statusText ? " " + e.statusText : ""}: ${message}`; + const message = describeQueryError(e, { timedOut: running.timedOut, timeoutMs: this.config.timeoutMs }); // Keep the rows that were already loaded when loading another page fails this.store.setResult(endpoint, query.id, { ...(offset > 0 && previous ? previous : { vars: [], bindings: [] }), status: offset > 0 && previous ? previous.status : "error", - error: message.length > 500 ? message.substring(0, 500) + "…" : message, + error: message, fetchedAt: Date.now(), durationMs: Date.now() - start, hasMore: false, diff --git a/packages/yasgui/src/endpointDescribe/describeQueries.ts b/packages/yasgui/src/endpointDescribe/describeQueries.ts index 8d7ce32c..1b309669 100644 --- a/packages/yasgui/src/endpointDescribe/describeQueries.ts +++ b/packages/yasgui/src/endpointDescribe/describeQueries.ts @@ -212,7 +212,7 @@ export const defaultDescribeQueries: DescribeQuery[] = [ description: "Namespaces of the predicates, ranked by the number of triples using them.", expensive: true, query: (ctx) => - `SELECT ?namespace (COUNT(*) AS ?triples) WHERE {\n ?s ?p ?o .\n BIND(REPLACE(STR(?p), "[^#/]*$", "") AS ?namespace)\n}\nGROUP BY ?namespace\nORDER BY DESC(?triples)\n${page( + `SELECT ?namespace (COUNT(*) AS ?triples) WHERE {\n ?s ?p ?o .\n BIND(REPLACE(STR(?p), "[^#/]+$", "") AS ?namespace)\n}\nGROUP BY ?namespace\nORDER BY DESC(?triples)\n${page( ctx, )}`, paginated: true, @@ -250,6 +250,7 @@ export const defaultDescribeQueries: DescribeQuery[] = [ label: "SKOS concept schemes", description: "SKOS concept schemes and the number of concepts in each.", paginated: true, + expensive: true, query: (ctx) => `${prefixes( "skos", diff --git a/packages/yasgui/src/endpointDescribe/metadataSources.ts b/packages/yasgui/src/endpointDescribe/metadataSources.ts index da9a0205..d5310143 100644 --- a/packages/yasgui/src/endpointDescribe/metadataSources.ts +++ b/packages/yasgui/src/endpointDescribe/metadataSources.ts @@ -19,6 +19,13 @@ const RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label"; /** Maximum number of list items kept per metadata field */ const MAX_ITEMS = 200; +/** + * Maximum number of bytes read from a service description or VoID document. + * Some endpoints publish very large descriptions (e.g. Wikidata: ~19 MB with all partitions); + * only the first part is read and parsed. + */ +export const MAX_METADATA_BYTES = 5_000_000; + export const RDF_ACCEPT = "text/turtle, application/n-triples;q=0.9, application/n-quads;q=0.8, text/n3;q=0.7"; export type MetadataStatus = "ok" | "unavailable"; @@ -42,6 +49,8 @@ export interface VoidPartition { export interface VoidDataset { iri: string; + /** The dataset is a blank node: its identifier is not meaningful */ + blank?: boolean; title?: string; triples?: number; entities?: number; @@ -65,6 +74,8 @@ export interface EndpointMetadata { /** Human readable reason why a source is unavailable */ sdError?: string; voidError?: string; + /** The description was larger than MAX_METADATA_BYTES and only partially read */ + truncated?: boolean; } export interface FetchInit { @@ -73,6 +84,14 @@ export interface FetchInit { signal?: AbortSignal; } +/** + * Cut a truncated Turtle/N-Triples document after its last complete statement. + */ +export function cutAtLastStatement(content: string): string { + const match = /[\s\S]*\s\.[ \t]*(\r?\n|$)/.exec(content); + return match ? match[0] : ""; +} + export function parseRdf(content: string, contentType: string | null, baseIRI: string): N3.Quad[] { const type = (contentType || "").split(";")[0].trim().toLowerCase(); if (type.includes("html") || type.includes("json") || type.includes("xml")) { @@ -94,9 +113,11 @@ function unique(values: string[]): string[] { class QuadIndex { private bySubject = new Map(); + public blankNodes = new Set(); constructor(public quads: N3.Quad[]) { for (const quad of quads) { const key = quad.subject.value; + if (quad.subject.termType === "BlankNode") this.blankNodes.add(key); if (!this.bySubject.has(key)) this.bySubject.set(key, []); this.bySubject.get(key)!.push(quad); } @@ -156,16 +177,20 @@ export function extractServiceDescription(quads: N3.Quad[]): ServiceDescription } function extractPartitions(index: QuadIndex, dataset: string, predicate: string, key: string): VoidPartition[] { - return index - .values(dataset, VOID + predicate) - .map((partition) => ({ - iri: index.values(partition, VOID + key)[0], + const seen = new Set(); + const partitions: VoidPartition[] = []; + for (const partition of index.values(dataset, VOID + predicate)) { + const iri = index.values(partition, VOID + key)[0]; + // The same partition may be described in both the service description and the VoID document + if (!iri || seen.has(iri)) continue; + seen.add(iri); + partitions.push({ + iri, entities: index.number(partition, VOID + "entities"), triples: index.number(partition, VOID + "triples"), - })) - .filter((p, i, all) => !!p.iri && all.findIndex((other) => other.iri === p.iri) === i) - .sort((a, b) => (b.entities ?? b.triples ?? 0) - (a.entities ?? a.triples ?? 0)) - .slice(0, MAX_ITEMS); + }); + } + return partitions.sort((a, b) => (b.entities ?? b.triples ?? 0) - (a.entities ?? a.triples ?? 0)).slice(0, MAX_ITEMS); } export function extractVoidDatasets(quads: N3.Quad[]): VoidDataset[] { @@ -181,9 +206,11 @@ export function extractVoidDatasets(quads: N3.Quad[]): VoidDataset[] { ...index.subjectsWith(VOID + "triples"), ...index.subjectsWith(VOID + "sparqlEndpoint"), ]).filter((iri) => !partitions.has(iri) && index.values(iri, RDF_TYPE).indexOf(VOID + "Linkset") < 0); + const linksets = index.subjectsWith(VOID + "subjectsTarget"); - return candidates.map((iri) => ({ + const datasets: VoidDataset[] = candidates.map((iri) => ({ iri, + blank: index.blankNodes.has(iri) || undefined, title: index.values(iri, DCTERMS + "title")[0] || index.values(iri, RDFS_LABEL)[0], triples: index.number(iri, VOID + "triples"), entities: index.number(iri, VOID + "entities"), @@ -195,8 +222,7 @@ export function extractVoidDatasets(quads: N3.Quad[]): VoidDataset[] { vocabularies: unique(index.values(iri, VOID + "vocabulary")), classPartitions: extractPartitions(index, iri, "classPartition", "class"), propertyPartitions: extractPartitions(index, iri, "propertyPartition", "property"), - linksets: index - .subjectsWith(VOID + "subjectsTarget") + linksets: linksets .filter((ls) => index.values(ls, VOID + "subjectsTarget").indexOf(iri) >= 0) .map((ls) => ({ iri: ls, @@ -204,9 +230,49 @@ export function extractVoidDatasets(quads: N3.Quad[]): VoidDataset[] { triples: index.number(ls, VOID + "triples"), })), })); + // Only keep datasets that tell something about the data + return datasets.filter( + (d) => + d.title || + d.triples !== undefined || + d.entities !== undefined || + d.classes !== undefined || + d.properties !== undefined || + d.distinctSubjects !== undefined || + d.distinctObjects !== undefined || + d.vocabularies.length || + d.classPartitions.length || + d.propertyPartitions.length || + d.linksets.length, + ); +} + +/** + * Read a response body, stopping after maxBytes + */ +export async function readLimited(response: Response, maxBytes: number): Promise<{ text: string; truncated: boolean }> { + const reader = response.body && typeof response.body.getReader === "function" ? response.body.getReader() : undefined; + if (!reader) { + const text = await response.text(); + return text.length > maxBytes ? { text: text.substring(0, maxBytes), truncated: true } : { text, truncated: false }; + } + const decoder = new TextDecoder(); + let text = ""; + let bytes = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + bytes += value.byteLength; + text += decoder.decode(value, { stream: true }); + if (bytes >= maxBytes) { + void reader.cancel().catch(() => undefined); + return { text, truncated: true }; + } + } + return { text: text + decoder.decode(), truncated: false }; } -async function fetchRdf(url: string, init: FetchInit): Promise { +async function fetchRdf(url: string, init: FetchInit): Promise<{ quads: N3.Quad[]; truncated: boolean }> { const response = await fetch(url, { method: "GET", headers: { ...(init.headers || {}), Accept: RDF_ACCEPT }, @@ -215,9 +281,10 @@ async function fetchRdf(url: string, init: FetchInit): Promise { signal: init.signal, }); if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`.trim()); - const content = await response.text(); - if (!content.trim()) throw new Error("Empty response"); - return parseRdf(content, response.headers.get("Content-Type"), response.url || url); + const { text, truncated } = await readLimited(response, MAX_METADATA_BYTES); + const content = truncated ? cutAtLastStatement(text) : text; + if (!content.trim()) throw new Error(truncated ? "Description too large" : "Empty response"); + return { quads: parseRdf(content, response.headers.get("Content-Type"), response.url || url), truncated }; } export function getWellKnownVoidUrl(endpoint: string): string | undefined { @@ -258,7 +325,8 @@ export async function fetchEndpointMetadata(endpoint: string, init: FetchInit = let sdQuads: N3.Quad[] = []; if (sdResult.status === "fulfilled") { - sdQuads = sdResult.value; + sdQuads = sdResult.value.quads; + if (sdResult.value.truncated) metadata.truncated = true; metadata.sd = extractServiceDescription(sdQuads); if (metadata.sd) metadata.sdStatus = "ok"; else metadata.sdError = "No service description found in the response"; @@ -268,7 +336,8 @@ export async function fetchEndpointMetadata(endpoint: string, init: FetchInit = let voidQuads: N3.Quad[] = []; if (voidResult.status === "fulfilled") { - voidQuads = voidResult.value; + voidQuads = voidResult.value.quads; + if (voidResult.value.truncated) metadata.truncated = true; } else if (!isAbort(voidResult.reason)) { metadata.voidError = errorMessage(voidResult.reason); } diff --git a/packages/yasgui/src/endpointDescribe/responseUtils.ts b/packages/yasgui/src/endpointDescribe/responseUtils.ts new file mode 100644 index 00000000..92d2b4cc --- /dev/null +++ b/packages/yasgui/src/endpointDescribe/responseUtils.ts @@ -0,0 +1,75 @@ +/** + * Helpers to interpret describe query responses of real-world endpoints. + */ + +const MAX_ERROR_LENGTH = 500; + +function getHeader(response: any, name: string): string | undefined { + const headers = response?.headers; + if (!headers) return undefined; + if (typeof headers.get === "function") return headers.get(name) ?? undefined; + const key = Object.keys(headers).find((k) => k.toLowerCase() === name.toLowerCase()); + return key ? headers[key] : undefined; +} + +/** + * Whether the endpoint returned an incomplete result, e.g. Virtuoso "anytime queries" + * which stop at a time limit and answer with HTTP 206 and X-SQL-State S1TAT. + */ +export function isPartialResponse(response: any): boolean { + return response?.status === 206 || getHeader(response, "X-SQL-State") === "S1TAT"; +} + +/** + * Turn an HTML error page into a short text: its title (or heading), otherwise its text content. + */ +export function htmlToText(content: string): string { + const title = + content.match(/]*>([\s\S]*?)<\/title>/i)?.[1] || content.match(/]*>([\s\S]*?)<\/h1>/i)?.[1]; + const text = title ?? content.replace(/<(script|style)[^>]*>[\s\S]*?<\/\1>/gi, " ").replace(/<[^>]+>/g, " "); + return text + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/\s+/g, " ") + .trim(); +} + +/** + * Extract the root cause from a Java stack trace (e.g. Blazegraph / Wikidata errors), if any. + */ +export function javaRootCause(message: string): string | undefined { + const pattern = /[\w.$]+(?:Exception|Error): /g; + let last: RegExpExecArray | null = null; + for (let match = pattern.exec(message); match; match = pattern.exec(message)) last = match; + if (!last) return undefined; + const cause = message + .substring(last.index + last[0].length) + .split("\n")[0] + .trim(); + return cause || undefined; +} + +/** + * A readable error message for a failed describe query. + */ +export function describeQueryError(error: any, options: { timedOut?: boolean; timeoutMs?: number } = {}): string { + if (options.timedOut) return `Timed out after ${Math.round((options.timeoutMs || 0) / 1000)} s`; + let message = error instanceof Error ? error.message : String(error ?? "Query failed"); + if (/^\s*<(!doctype|html|\?xml)/i.test(message) || /<\/(html|body|title)>/i.test(message)) { + message = htmlToText(message); + } + if (/\n\s*(at |Caused by: )|java\.\w/.test(message)) message = javaRootCause(message) ?? message; + const status: number | undefined = error?.status; + if (status === 429) { + const retryAfter = getHeader(error?.response, "Retry-After"); + message = + "The endpoint is rate limiting requests. Wait a moment before running more queries" + + (retryAfter && /^\d+$/.test(retryAfter) ? ` (retry after ${retryAfter} s).` : "."); + } else if (status === 502 || status === 503 || status === 504) { + message = `The endpoint (or a gateway in front of it) did not answer in time. ${message}`.trim(); + } + if (status) message = `HTTP ${status}${error?.statusText ? " " + error.statusText : ""}: ${message}`; + return message.length > MAX_ERROR_LENGTH ? message.substring(0, MAX_ERROR_LENGTH) + "…" : message; +} diff --git a/test/endpoint-describe-browser.ts b/test/endpoint-describe-browser.ts index 9669929e..671ce5e8 100644 --- a/test/endpoint-describe-browser.ts +++ b/test/endpoint-describe-browser.ts @@ -73,6 +73,30 @@ async function mockEndpoints(page: puppeteer.Page, queries: string[]) { ], ); } + if (/LANG\(\?label\)/.test(query)) { + // Virtuoso "anytime query": interrupted at the time limit, incomplete (here: empty) result + void request.respond({ + status: 206, + headers: { + ...headers, + "Access-Control-Expose-Headers": "X-SQL-State", + "X-SQL-State": "S1TAT", + "X-SQL-Message": "RC...: Returning incomplete results, query interrupted by result timeout.", + }, + contentType: "application/sparql-results+json", + body: sparqlJson(["property", "language", "labels"], []), + }); + return; + } + if (/\?geometry geo:asWKT \?wkt \.\s*BIND/.test(query)) { + void request.respond({ + status: 429, + headers: { ...headers, "Retry-After": "30", "Access-Control-Expose-Headers": "Retry-After" }, + contentType: "text/html", + body: 'Wikimedia ErrorToo many requests', + }); + return; + } void request.respond({ status: 200, headers, contentType: "application/sparql-results+json", body }); }); } @@ -187,6 +211,26 @@ describe("Endpoint describe panel", function () { await page.waitForSelector('.yasgui-describe__query[data-query-id="classes"] table'); }); + it("flags partial results and explains rate limiting", async function () { + await openPanel(); + await page.evaluate(() => (window as any).yasgui.endpointDescribe.runQuery("label-languages")); + const partial = await page.$eval( + '.yasgui-describe__query[data-query-id="label-languages"]', + (el) => el.querySelector(".yasgui-describe__partial")?.textContent || "", + ); + expect(partial).to.contain("Partial result"); + + await page.evaluate(() => (window as any).yasgui.endpointDescribe.runQuery("wkt-crs")); + const error = await page.$eval( + '.yasgui-describe__query[data-query-id="wkt-crs"]', + (el) => el.querySelector(".yasgui-describe__error")?.textContent || "", + ); + expect(error).to.contain("HTTP 429"); + expect(error).to.contain("rate limiting"); + expect(error).to.contain("retry after 30 s"); + expect(error).not.to.contain(" { } }); + it("does not use REPLACE patterns that match the empty string (rejected by Virtuoso)", () => { + for (const query of defaultDescribeQueries) { + const text = buildQuery(query, context()); + for (const match of text.matchAll(/REPLACE\([^,]+,\s*"((?:[^"\\]|\\.)*)"/g)) { + expect(new RegExp(match[1]).test(""), `${query.id}: ${match[1]}`).to.equal(false); + } + } + }); + it("paginates with LIMIT and OFFSET", () => { for (const query of defaultDescribeQueries.filter((q) => q.paginated)) { const firstPage = buildQuery(query, context({ limit: 10, offset: 0 })); @@ -182,6 +199,44 @@ describe("Endpoint describe metadata (SD / VoID)", () => { expect(datasets[0].classPartitions).to.have.length(2); }); + it("flags blank-node datasets and drops datasets without information", () => { + const quads = parseRdf( + `@prefix void: . + _:graph a void:Dataset ; void:triples 42 . + _:empty a void:Dataset .`, + "text/turtle", + "https://example.org/sparql", + ); + const datasets = extractVoidDatasets(quads); + expect(datasets).to.have.length(1); + expect(datasets[0].blank).to.equal(true); + expect(datasets[0].triples).to.equal(42); + }); + + it("cuts a truncated document after its last complete statement", () => { + const doc = ' "1" .\n [ "2" ] .\n "unfin'; + const cut = cutAtLastStatement(doc); + expect(parseRdf(cut, "text/turtle", "https://example.org/")).to.have.length(3); + expect(cutAtLastStatement("no statement")).to.equal(""); + }); + + it("stops reading large descriptions", async () => { + const chunk = new TextEncoder().encode("x".repeat(1_000_000)); + let pulled = 0; + const stream = new ReadableStream({ + pull(controller) { + pulled++; + controller.enqueue(chunk); + }, + }); + const { text, truncated } = await readLimited(new Response(stream), MAX_METADATA_BYTES); + expect(truncated).to.equal(true); + expect(text.length).to.be.at.least(MAX_METADATA_BYTES); + expect(pulled).to.be.below(10); + const small = await readLimited(new Response(" ."), MAX_METADATA_BYTES); + expect(small).to.deep.equal({ text: " .", truncated: false }); + }); + it("rejects HTML responses", () => { expect(() => parseRdf("", "text/html; charset=utf-8", "https://example.org/")).to.throw(); }); @@ -309,3 +364,60 @@ describe("Endpoint describe store", () => { expect(store.getResult("https://example.org/sparql", "q")).to.not.equal(undefined); }); }); + +describe("Endpoint describe response handling", () => { + it("detects partial results of Virtuoso anytime queries", () => { + expect(isPartialResponse({ status: 206, headers: new Headers({ "X-SQL-State": "S1TAT" }) })).to.equal(true); + expect(isPartialResponse({ status: 200, headers: new Headers({ "X-SQL-State": "S1TAT" }) })).to.equal(true); + expect(isPartialResponse({ status: 200, headers: new Headers() })).to.equal(false); + expect(isPartialResponse(undefined)).to.equal(false); + }); + + it("summarizes HTML error pages", () => { + const wikimedia = + '\n\n\nWikimedia Error\n\nToo many requests'; + expect(htmlToText(wikimedia)).to.equal("Wikimedia Error"); + expect(htmlToText("

504 Gateway Time-out

")).to.equal("504 Gateway Time-out"); + expect(htmlToText("
a & b
")).to.equal("a & b"); + }); + + it("explains rate limiting, gateway timeouts and client timeouts", () => { + const rateLimited: any = new Error("Wikimedia Error"); + rateLimited.status = 429; + rateLimited.statusText = "Too Many Requests"; + rateLimited.response = { headers: new Headers({ "Retry-After": "30" }) }; + expect(describeQueryError(rateLimited)).to.equal( + "HTTP 429 Too Many Requests: The endpoint is rate limiting requests. Wait a moment before running more queries (retry after 30 s).", + ); + + const gateway: any = new Error( + 'Gateway Timeout', + ); + gateway.status = 504; + expect(describeQueryError(gateway)).to.equal( + "HTTP 504: The endpoint (or a gateway in front of it) did not answer in time. Gateway Timeout", + ); + + const virtuoso: any = new Error( + "Virtuoso 42000 Error The estimated execution time 6910 (sec) exceeds the limit of 60 (sec).", + ); + virtuoso.status = 500; + expect(describeQueryError(virtuoso)).to.equal( + "HTTP 500: Virtuoso 42000 Error The estimated execution time 6910 (sec) exceeds the limit of 60 (sec).", + ); + + const blazegraph: any = new Error( + "SPARQL-QUERY: queryStr=SELECT DISTINCT ?graph WHERE { GRAPH ?graph { ?s ?p ?o } }\njava.util.concurrent.ExecutionException: java.util.concurrent.ExecutionException: com.bigdata.rdf.sparql.ast.QuadsOperationInTriplesModeException: Use of WITH and GRAPH constructs in query body is not supported in triples mode.\nCaused by: com.bigdata.rdf.sparql.ast.QuadsOperationInTriplesModeException: Use of WITH and GRAPH constructs in query body is not supported in triples mode.\n\tat com.bigdata.Foo.bar(Foo.java:1)", + ); + blazegraph.status = 400; + blazegraph.statusText = "Bad Request"; + expect(describeQueryError(blazegraph)).to.equal( + "HTTP 400 Bad Request: Use of WITH and GRAPH constructs in query body is not supported in triples mode.", + ); + + expect(describeQueryError(new Error("aborted"), { timedOut: true, timeoutMs: 60000 })).to.equal( + "Timed out after 60 s", + ); + expect(describeQueryError(new Error("x".repeat(600)))).to.have.length(501); + }); +}); From ffab656cbc358858450e105d8c4fe062422b500e Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Wed, 30 Sep 2026 21:57:21 +0200 Subject: [PATCH 4/4] Modify textContent assignment to handle gYear datatype Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts b/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts index 41c176d1..c0006653 100644 --- a/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts +++ b/packages/yasgui/src/endpointDescribe/EndpointDescribePanel.ts @@ -613,7 +613,8 @@ export default class EndpointDescribePanel { if (term.type === "bnode") return document.createTextNode(`_:${term.value}`); const value = term.value; const span = el("span", "yasgui-describe__literal"); - span.textContent = /^-?\d{4,}$/.test(value) ? Number(value).toLocaleString() : value; + const isYear = term.datatype === "http://www.w3.org/2001/XMLSchema#gYear"; + span.textContent = !isYear && /^-?\d{4,}$/.test(value) ? Number(value).toLocaleString() : value; if (term["xml:lang"]) span.appendChild(el("span", "yasgui-describe__lang", `@${term["xml:lang"]}`)); return span; }