diff --git a/deploy/workflow-cfd-studio.yaml b/deploy/workflow-cfd-studio.yaml index 4d7dc19..4c85217 100644 --- a/deploy/workflow-cfd-studio.yaml +++ b/deploy/workflow-cfd-studio.yaml @@ -356,8 +356,10 @@ permissions: type: string optional: true tooltip: | - Comma-separated names of platform workflows to offer as tiles on the - Workflows tab, in order. Each viewer runs them under their own account. + Comma-separated workflows to offer as tiles on the Workflows tab, in + order: account workflow names, marketplace/, or + github.com//[/path][@ref]. Each viewer runs them under + their own account. Administrators can also pick them in Settings, Workflows, from the workflows on the account this Studio is deployed under. build_gufi: diff --git a/deploy/workflow.yaml b/deploy/workflow.yaml index 9de7353..75ec453 100644 --- a/deploy/workflow.yaml +++ b/deploy/workflow.yaml @@ -353,8 +353,10 @@ permissions: type: string optional: true tooltip: | - Comma-separated names of platform workflows to offer as tiles on the - Workflows tab, in order. Each viewer runs them under their own account. + Comma-separated workflows to offer as tiles on the Workflows tab, in + order: account workflow names, marketplace/, or + github.com//[/path][@ref]. Each viewer runs them under + their own account. Administrators can also pick them in Settings, Workflows, from the workflows on the account this Studio is deployed under. build_gufi: diff --git a/docs/WORKFLOWS.md b/docs/WORKFLOWS.md index 2e6ec3a..7f4db7c 100644 --- a/docs/WORKFLOWS.md +++ b/docs/WORKFLOWS.md @@ -6,7 +6,17 @@ The tab appears only when the Studio can reach a platform, through the viewer's ## Choosing the set -An administrator picks workflows in Settings, Workflows. The list to pick from is every workflow on the platform account the Studio is deployed under, so a site deployment offers from that site's workflows. Order in the list is the order of the tiles. A name that is no longer on the account is flagged there and shown as an unavailable tile. +An entry is one of three kinds: + +| Entry | Runs | +|---|---| +| `name` | a workflow saved on the account; a viewer without it can add a copy from its tile | +| `marketplace/` | the marketplace workflow, by slug, with no copy in anyone's account | +| `github.com//[/path][@ref]` | the workflow.yaml in that repository (or directory, or named file), with no copy in anyone's account | + +The form for a GitHub entry is read from the repository's raw file, so it shows for public repositories; a private repository still runs, through the platform's connected account. + +An administrator picks workflows in Settings, Workflows, from the workflows on the platform account the Studio is deployed under (a site deployment offers from that site's workflows), from the marketplace, or by adding a GitHub repository, which is checked before it is added. Order in the list is the order of the tiles. A name that is no longer on the account is flagged there and shown as an unavailable tile. At deploy time, the Offered Workflows input (environment variable `STUDIO_WORKFLOWS`, comma-separated names) sets the initial list. A list saved in Settings takes its place; clearing it in Settings returns to the deploy-time list. @@ -36,9 +46,10 @@ Recent runs from the tab are listed below the tiles with their state. | Route | Purpose | |---|---| | `GET /api/workflows/collection` | the tiles, as the viewer has them | -| `GET /api/workflows/catalog` | the deployment account's workflows, for Settings | -| `GET /api/workflows/:name/form` | the converted form and saved configurations | -| `POST /api/workflows/:name/run` | `{inputs, dryRun}`; validate or submit | -| `POST /api/workflows/:name/install` | copy the deployment's definition into the viewer's account | +| `GET /api/workflows/catalog` | the deployment account's workflows and the marketplace's, for Settings | +| `GET /api/workflows/item/form?w=` | the converted form and saved configurations | +| `GET /api/workflows/item/icon?w=` | the tile's icon | +| `POST /api/workflows/item/run?w=` | `{inputs, dryRun}`; validate or submit | +| `POST /api/workflows/item/install?w=` | copy the deployment's definition of an account workflow into the viewer's account | | `GET /api/workflows/runs` | recent runs | | `GET /api/platform/clusters`, `/partitions?cluster=`, `/buckets` | data for the pickers | diff --git a/server/src/settings.ts b/server/src/settings.ts index 71254e4..11e5791 100644 --- a/server/src/settings.ts +++ b/server/src/settings.ts @@ -1,4 +1,5 @@ import fs from 'node:fs' +import { parseWorkflowEntry } from './workflowEntries.js' import fsp from 'node:fs/promises' import path from 'node:path' import type { FastifyInstance } from 'fastify' @@ -188,7 +189,7 @@ export async function settingsRoutes(app: FastifyInstance): Promise { if (body.kbLabel !== undefined) next.kbLabel = String(body.kbLabel).slice(0, 60) || undefined if (body.sections !== undefined) next.sections = Array.isArray(body.sections) && body.sections.length ? body.sections.map(String).slice(0, 20) : undefined if (body.workflowCollection !== undefined) { - const list = Array.isArray(body.workflowCollection) ? body.workflowCollection.map(String).filter(n => /^[A-Za-z0-9_.-]{1,120}$/.test(n)) : [] + const list = Array.isArray(body.workflowCollection) ? body.workflowCollection.map(n => parseWorkflowEntry(String(n))?.entry).filter((n): n is string => !!n) : [] next.workflowCollection = list.length ? [...new Set(list)].slice(0, 60) : undefined } if (body.theme !== undefined) next.theme = body.theme === 'dark' ? 'dark' : 'light' diff --git a/server/src/workflowEntries.ts b/server/src/workflowEntries.ts new file mode 100644 index 0000000..eb2269c --- /dev/null +++ b/server/src/workflowEntries.ts @@ -0,0 +1,56 @@ +/** + * An entry in a Studio's workflow collection is one of: + * + * name a workflow saved on the account + * marketplace/ a marketplace workflow, run by slug + * github.com//[/path][@ref] a workflow.yaml in a GitHub repository + * + * The last two run without being added to anyone's account; `pw workflows + * run` accepts both forms as they are written here. + */ + +export type WorkflowEntry = + | { kind: 'account'; entry: string; name: string } + | { kind: 'marketplace'; entry: string; slug: string } + | { kind: 'github'; entry: string; owner: string; repo: string; path: string; ref: string | null } + +const NAME = /^[A-Za-z0-9_.-]{1,120}$/ +const SEG = '[A-Za-z0-9_.-]+' +const GITHUB = new RegExp(`^github\\.com/(${SEG})/(${SEG})((?:/${SEG})*)(?:@([A-Za-z0-9_./-]{1,120}))?$`) + +export function parseWorkflowEntry(raw: string): WorkflowEntry | null { + const s = String(raw ?? '').trim() + if (!s || s.length > 300 || s.includes('..')) return null + if (NAME.test(s)) return { kind: 'account', entry: s, name: s } + const mp = /^marketplace\/([A-Za-z0-9_.-]{1,120})$/.exec(s) + if (mp) return { kind: 'marketplace', entry: s, slug: mp[1] } + const gh = GITHUB.exec(s.replace(/^https?:\/\//, '')) + if (gh) { + const entry = s.replace(/^https?:\/\//, '') + return { kind: 'github', entry, owner: gh[1], repo: gh[2], path: gh[3].replace(/^\//, ''), ref: gh[4] ?? null } + } + return null +} + +/** The raw file URLs to try for a GitHub entry: the named file, or + * workflow.yaml then workflow.yml in the named directory. */ +export function githubYamlUrls(e: Extract): string[] { + const base = `https://raw.githubusercontent.com/${e.owner}/${e.repo}/${e.ref ?? 'HEAD'}` + if (/\.ya?ml$/.test(e.path)) return [`${base}/${e.path}`] + const dir = e.path ? `${base}/${e.path}` : base + return [`${dir}/workflow.yaml`, `${dir}/workflow.yml`] +} + +/** Where a GitHub workflow's thumbnail would be, by the platform's convention. */ +export function githubThumbnail(e: Extract): string { + const dir = /\.ya?ml$/.test(e.path) ? e.path.replace(/\/?[^/]+$/, '') : e.path + return `https://raw.githubusercontent.com/${e.owner}/${e.repo}/${e.ref ?? 'HEAD'}/${dir ? `${dir}/` : ''}thumbnail.png` +} + +/** A readable default title: the repository, or its last path segment. */ +export function entryTitle(e: WorkflowEntry): string { + if (e.kind === 'account') return e.name + if (e.kind === 'marketplace') return e.slug + const tail = e.path.replace(/\/?workflow\.ya?ml$/, '').split('/').filter(Boolean).pop() + return tail || e.repo +} diff --git a/server/src/workflowsTab.ts b/server/src/workflowsTab.ts index d182938..d17e40c 100644 --- a/server/src/workflowsTab.ts +++ b/server/src/workflowsTab.ts @@ -24,6 +24,7 @@ import { gatewayKey } from './chat/gateway.js' import { resolveUserCred } from './credentials.js' import { effectiveSettings } from './settings.js' import { KbError } from './kb.js' +import { entryTitle, githubThumbnail, githubYamlUrls, parseWorkflowEntry, type WorkflowEntry } from './workflowEntries.js' import { recordRun, listRuns, markRunEnded, isTerminalRunState } from './runs.js' import { withWorkspace } from './workspace.js' @@ -106,11 +107,6 @@ export function curatedNames(): string[] { return (effectiveSettings().workflowCollection ?? []).filter(Boolean) } -const SAFE_NAME = /^[A-Za-z0-9_.-]{1,120}$/ -function checkName(name: string): string { - if (!SAFE_NAME.test(name)) throw new KbError(400, 'invalid workflow name') - return name -} /** * The cluster and bucket pickers hold an object so conditional fields can @@ -127,6 +123,54 @@ export function resolveStudioRefs(v: unknown): unknown { return v } + +interface MarketRow { slug: string; name?: string; description?: string; imageUrl?: string; type?: string } +const marketCache = new Map() +async function marketplaceFor(key: string | null): Promise { + const k = (key ?? '').slice(-12) + const hit = marketCache.get(k) + if (hit && Date.now() - hit.at < 300_000) return hit.rows + const raw = firstJson(await call(['marketplace', 'ls', '-o', 'json'], key, 90_000)) + const rows = ((Array.isArray(raw) ? raw : (raw as { items?: unknown[] }).items ?? []) as MarketRow[]).filter(r => !r.type || r.type === 'workflow') + marketCache.set(k, { at: Date.now(), rows }) + return rows +} + +function entryOf(req: { query: unknown }): WorkflowEntry { + const e = parseWorkflowEntry(String((req.query as { w?: string }).w ?? '')) + if (!e) throw new KbError(400, 'invalid workflow: use a name, marketplace/, or github.com//[/path][@ref]') + return e +} + +interface Definition { yaml: any; displayName: string; description: string; configurations: { name: string; inputs: Record }[] } + +/** A workflow's YAML and details, wherever it is defined. */ +async function definitionOf(e: WorkflowEntry, key: string | null): Promise { + if (e.kind === 'account') { + const doc = firstJson(await call(['workflows', 'get', e.name, '-o', 'json'], key, 60_000)) + return { + yaml: typeof doc?.yaml === 'string' ? parseYaml(doc.yaml) : doc?.yaml, + displayName: doc?.displayName || e.name, + description: doc?.description ?? '', + configurations: (doc?.configurations ?? []).map((c: any) => ({ name: c.name ?? c.id ?? '', inputs: c.inputs ?? {} })), + } + } + if (e.kind === 'marketplace') { + const text = await call(['marketplace', 'get', e.slug, '--yaml'], key, 60_000) + const m = (await marketplaceFor(key).catch(() => [])).find(x => x.slug === e.slug) + return { yaml: parseYaml(text), displayName: m?.name || e.slug, description: m?.description ?? '', configurations: [] } + } + let lastStatus = 0 + for (const url of githubYamlUrls(e)) { + const res = await fetch(url, { signal: AbortSignal.timeout(20_000) }).catch(() => null) + if (res?.ok) return { yaml: parseYaml(await res.text()), displayName: entryTitle(e), description: e.entry, configurations: [] } + lastStatus = res?.status ?? 0 + } + throw new KbError(lastStatus === 404 ? 404 : 502, lastStatus === 404 + ? `No workflow.yaml at ${e.entry}. Private repositories still run, but their form cannot be shown here.` + : `Could not read ${e.entry} from GitHub.`) +} + const runChecked = new Map() const iconCache = new Map() @@ -151,8 +195,17 @@ export async function workflowsTabRoutes(app: FastifyInstance): Promise { if (!rows && mine.own && mine.key) { rows = await listFor(mine.key); source = 'viewer' } if (!rows) throw depError ?? new KbError(409, 'No platform credential: add your ACTIVATE API key under Settings, Model access.') const curated = new Set(curatedNames()) + // Marketplace workflows, listed with whichever key worked; a failure + // here leaves the account list standing. + const market = await marketplaceFor(source === 'viewer' ? mine.key : dep).catch(() => []) return { source, + marketplace: market.map(m => ({ + name: `marketplace/${m.slug}`, + displayName: m.name || m.slug, + description: m.description ?? '', + curated: curated.has(`marketplace/${m.slug}`), + })).sort((a, b) => a.displayName.localeCompare(b.displayName)), workflows: rows.map(w => ({ name: w.name, displayName: w.displayName || w.name, @@ -160,34 +213,54 @@ export async function workflowsTabRoutes(app: FastifyInstance): Promise { tags: (w.tags ?? []).filter(Boolean), curated: curated.has(w.name), })).sort((a, b) => a.displayName.localeCompare(b.displayName)), - missing: [...curated].filter(n => !rows.some(w => w.name === n)), + missing: [...curated].filter(n => parseWorkflowEntry(n)?.kind === 'account' && !rows.some(w => w.name === n)), } }) - // The tiles: the curated set, as this viewer has it. + // The tiles: the curated set, as this viewer has it. Account workflows + // come from the viewer's list (or the owner's, until they add a copy); + // marketplace and GitHub entries run as they are and need no copy. app.get('/api/workflows/collection', async req => { const names = curatedNames() if (!names.length) return { configured: false, workflows: [] } const v = viewer(req) - const mine = await listFor(v.key) + const entries = names.map(n => parseWorkflowEntry(n)).filter((e): e is WorkflowEntry => !!e) + const accountNames = entries.filter(e => e.kind === 'account').map(e => e.entry) + const mine = accountNames.length ? await listFor(v.key) : [] const byName = new Map(mine.map(w => [w.name, w])) - // A viewer on their own key may lack a curated record; the owner's copy - // supplies its title and icon until they add it. let owner: Map | null = null - if (v.own && names.some(n => !byName.has(n)) && gatewayKey()) { + if (v.own && accountNames.some(n => !byName.has(n)) && gatewayKey()) { owner = new Map((await listFor(gatewayKey()).catch(() => [])).map(w => [w.name, w])) } - const workflows = names.map(name => { - const w = byName.get(name) ?? owner?.get(name) + const market = entries.some(e => e.kind === 'marketplace') ? await marketplaceFor(v.key).catch(() => null) : null + const iconOf = (entry: string) => `/api/workflows/item/icon?w=${encodeURIComponent(entry)}` + const workflows = entries.map(e => { + if (e.kind === 'account') { + const w = byName.get(e.name) ?? owner?.get(e.name) + return { + name: e.entry, kind: e.kind, + displayName: w?.displayName || e.name, + description: w?.description ?? '', + tags: (w?.tags ?? []).filter(Boolean), + icon: w?.imageUrl ? iconOf(e.entry) : null, + configurations: (byName.get(e.name)?.configurations ?? []).map(c => c.name ?? c.id ?? '').filter(Boolean), + installed: byName.has(e.name), + available: !!w, + } + } + if (e.kind === 'marketplace') { + const m = market?.find(x => x.slug === e.slug) + return { + name: e.entry, kind: e.kind, + displayName: m?.name || e.slug, description: m?.description ?? '', tags: [], + icon: m?.imageUrl ? iconOf(e.entry) : null, configurations: [], + installed: true, available: market ? !!m : true, + } + } return { - name, - displayName: w?.displayName || name, - description: w?.description ?? '', - tags: (w?.tags ?? []).filter(Boolean), - icon: w?.imageUrl ? `/api/workflows/${encodeURIComponent(name)}/icon` : null, - configurations: (byName.get(name)?.configurations ?? []).map(c => c.name ?? c.id ?? '').filter(Boolean), - installed: byName.has(name), - available: !!w, + name: e.entry, kind: e.kind, + displayName: entryTitle(e), description: e.entry, tags: [], + icon: iconOf(e.entry), configurations: [], installed: true, available: true, } }) return { configured: true, workflows } @@ -195,43 +268,42 @@ export async function workflowsTabRoutes(app: FastifyInstance): Promise { // A tile's icon. Platform blobs need the credential, so they are served // through here; outside URLs are fetched once and cached. - app.get('/api/workflows/:name/icon', async (req, reply) => { - const name = checkName((req.params as { name: string }).name) - const hit = iconCache.get(name) + app.get('/api/workflows/item/icon', async (req, reply) => { + const e = entryOf(req) + const hit = iconCache.get(e.entry) if (hit && Date.now() - hit.at < 3_600_000) return reply.type(hit.type).header('Cache-Control', 'private, max-age=3600').send(hit.body) const v = viewer(req) - const rows = [...await listFor(v.key), ...(v.own && gatewayKey() ? await listFor(gatewayKey()).catch(() => []) : [])] - const url = rows.find(w => w.name === name)?.imageUrl + let url: string | undefined + if (e.kind === 'account') { + const rows = [...await listFor(v.key), ...(v.own && gatewayKey() ? await listFor(gatewayKey()).catch(() => []) : [])] + url = rows.find(w => w.name === e.name)?.imageUrl + } else if (e.kind === 'marketplace') { + url = (await marketplaceFor(v.key).catch(() => [])).find(x => x.slug === e.slug)?.imageUrl + } else url = githubThumbnail(e) if (!url) return reply.status(404).send({ error: 'no icon' }) const host = new URL(GATEWAY_BASE).origin const abs = url.startsWith('/') ? host + url : url if (!/^https:\/\//.test(abs)) return reply.status(404).send({ error: 'no icon' }) - const res = await fetch(abs, { headers: abs.startsWith(host) ? { Authorization: `Bearer ${v.key}` } : {}, signal: AbortSignal.timeout(15_000) }) - if (!res.ok) return reply.status(404).send({ error: 'icon unavailable' }) + const res = await fetch(abs, { headers: abs.startsWith(host) ? { Authorization: `Bearer ${v.key}` } : {}, signal: AbortSignal.timeout(15_000) }).catch(() => null) + if (!res?.ok) return reply.status(404).send({ error: 'icon unavailable' }) const type = res.headers.get('content-type') || 'image/png' if (!/^image\//.test(type)) return reply.status(404).send({ error: 'not an image' }) const body = Buffer.from(await res.arrayBuffer()) - iconCache.set(name, { at: Date.now(), type, body }) + iconCache.set(e.entry, { at: Date.now(), type, body }) return reply.type(type).header('Cache-Control', 'private, max-age=3600').send(body) }) // The form: the workflow's inputs in the platform's dynamic-form shape, - // and its saved configurations as presets. - app.get('/api/workflows/:name/form', async req => { - const name = checkName((req.params as { name: string }).name) + // and its saved configurations as presets. Also how Settings checks a + // GitHub entry before adding it. + app.get('/api/workflows/item/form', async req => { + const e = entryOf(req) const v = viewer(req) - const doc = firstJson(await call(['workflows', 'get', name, '-o', 'json'], v.key, 60_000)) - const yamlDoc = typeof doc?.yaml === 'string' ? parseYaml(doc.yaml) : doc?.yaml - const inputs = yamlDoc?.on?.execute?.inputs + const d = await definitionOf(e, v.key) + const inputs = d.yaml?.on?.execute?.inputs let form: unknown = null - try { form = inputs && workflowHasUserInputs(yamlDoc) ? convertToDynamicForm(inputs) : {} } catch { form = null } - return { - name, - displayName: doc?.displayName || name, - description: doc?.description ?? '', - form, - configurations: (doc?.configurations ?? []).map((c: any) => ({ name: c.name ?? c.id ?? '', inputs: c.inputs ?? {} })), - } + try { form = inputs && workflowHasUserInputs(d.yaml) ? convertToDynamicForm(inputs) : {} } catch { form = null } + return { name: e.entry, kind: e.kind, displayName: d.displayName, description: d.description, form, configurations: d.configurations } }) // Clusters for the cluster picker, as the pw:// reference the platform @@ -283,46 +355,47 @@ export async function workflowsTabRoutes(app: FastifyInstance): Promise { // Validate or run. Values arrive as the form holds them; the platform's // own submission logic drops hidden and ignored fields before they go. - app.post('/api/workflows/:name/run', async req => { - const name = checkName((req.params as { name: string }).name) + app.post('/api/workflows/item/run', async req => { + const e = entryOf(req) const body = req.body as { inputs?: Record; dryRun?: boolean } const v = viewer(req) - if (!curatedNames().includes(name)) throw new KbError(403, `${name} is not in this Studio's workflow collection`) - const doc = firstJson(await call(['workflows', 'get', name, '-o', 'json'], v.key, 60_000)) - const yamlDoc = typeof doc?.yaml === 'string' ? parseYaml(doc.yaml) : doc?.yaml + if (!curatedNames().includes(e.entry)) throw new KbError(403, `${e.entry} is not in this Studio's workflow collection`) + const d = await definitionOf(e, v.key) let values: Record = body.inputs ?? {} - try { values = prepareSubmittableValues(yamlDoc, values, v.username ?? '') } catch { /* submit as given */ } + try { values = prepareSubmittableValues(d.yaml, values, v.username ?? '') } catch { /* submit as given */ } values = resolveStudioRefs(values) as Record - const args = ['workflows', 'run', name, '-i', JSON.stringify(values), '-o', 'json'] + const args = ['workflows', 'run', e.entry, '-i', JSON.stringify(values), '-o', 'json'] if (body.dryRun) args.push('--dry-run') let out: string try { // Raw CLI errors here: withWorkspace recognizes a stopped workspace by its text. out = (await withWorkspace({ cli: (a, t) => cli(a, v.key, t) }, () => cli(args, v.key, 180_000))).value - } catch (e) { - const msg = String((e as Error).message ?? e).replace(/^\S+Z \[(ERROR|WARN)\] /gm, '') + } catch (err) { + const msg = String((err as Error).message ?? err).replace(/^\S+Z \[(ERROR|WARN)\] /gm, '') return { ok: false, dryRun: !!body.dryRun, message: msg.slice(0, 2000) } } if (body.dryRun) return { ok: true, dryRun: true, message: out.trim().slice(0, 2000) || 'Validation passed.' } let slug: string | null = null try { const r = firstJson(out); slug = String((r?.run ?? r)?.slug ?? (r?.run ?? r)?.id ?? '') || null } catch { /* plain output */ } - if (slug) recordRun({ slug, workflow: name, owner: v.username }) + if (slug) recordRun({ slug, workflow: e.entry, owner: v.username }) return { ok: true, dryRun: false, slug, message: slug ? `Run ${slug} submitted.` : out.trim().slice(0, 2000) } }) - // First use: copy the deployment owner's record into the viewer's account. - app.post('/api/workflows/:name/install', async req => { - const name = checkName((req.params as { name: string }).name) + // First use of an account workflow: copy the deployment owner's record + // into the viewer's account. Marketplace and GitHub entries need none. + app.post('/api/workflows/item/install', async req => { + const e = entryOf(req) const v = viewer(req) - if (!curatedNames().includes(name)) throw new KbError(403, `${name} is not in this Studio's workflow collection`) + if (!curatedNames().includes(e.entry)) throw new KbError(403, `${e.entry} is not in this Studio's workflow collection`) + if (e.kind !== 'account') throw new KbError(409, 'Marketplace and GitHub workflows run without a copy in your account.') const ownerKey = gatewayKey() if (!v.own || !ownerKey) throw new KbError(409, 'Nothing to add: this Studio runs workflows with its own credential.') - const doc = firstJson(await call(['workflows', 'get', name, '-o', 'json'], ownerKey, 60_000)) + const doc = firstJson(await call(['workflows', 'get', e.name, '-o', 'json'], ownerKey, 60_000)) const text = typeof doc?.yaml === 'string' ? doc.yaml : dumpYaml(doc?.yaml ?? {}) - const tmp = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'wf-')), `${name}.yaml`) + const tmp = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'wf-')), `${e.name}.yaml`) fs.writeFileSync(tmp, text) try { - const args = ['workflows', 'create', '--yaml', tmp, name] + const args = ['workflows', 'create', '--yaml', tmp, e.name] if (doc?.displayName) args.splice(2, 0, '--display-name', String(doc.displayName)) await call(args, v.key, 60_000) } finally { fs.rmSync(path.dirname(tmp), { recursive: true, force: true }) } diff --git a/server/test/workflowEntries.test.mjs b/server/test/workflowEntries.test.mjs new file mode 100644 index 0000000..15e4c70 --- /dev/null +++ b/server/test/workflowEntries.test.mjs @@ -0,0 +1,106 @@ +// Collection entries beyond account workflows: marketplace slugs and +// workflow.yaml files in GitHub repositories, shown and run from the tab. +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +process.env.KB_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'we-kb-')) +process.env.INDEX_BASE = fs.mkdtempSync(path.join(os.tmpdir(), 'we-ix-')) +process.env.PW_API_KEY = 'deployment-key' +process.env.STUDIO_WORKFLOWS = 'marketplace/jupyter,github.com/example-org/batch-flow@v2' + +const { parseWorkflowEntry, githubYamlUrls, githubThumbnail } = await import('../dist/workflowEntries.js') + +test('entries parse into their three kinds, and anything else is refused', () => { + assert.deepEqual(parseWorkflowEntry('activatebatch'), { kind: 'account', entry: 'activatebatch', name: 'activatebatch' }) + assert.deepEqual(parseWorkflowEntry('marketplace/jupyter'), { kind: 'marketplace', entry: 'marketplace/jupyter', slug: 'jupyter' }) + assert.deepEqual(parseWorkflowEntry('https://github.com/example-org/flows/examples/slurm@main'), + { kind: 'github', entry: 'github.com/example-org/flows/examples/slurm@main', owner: 'example-org', repo: 'flows', path: 'examples/slurm', ref: 'main' }) + for (const bad of ['', 'github.com/only-owner', 'github.com/a/b/../c', 'gitlab.com/a/b', 'marketplace/', 'a b', 'x;rm -rf /']) { + assert.equal(parseWorkflowEntry(bad), null, bad) + } +}) + +test('GitHub entries resolve to raw file URLs for the form and the thumbnail', () => { + const dir = parseWorkflowEntry('github.com/o/r/sub@v1') + assert.deepEqual(githubYamlUrls(dir), ['https://raw.githubusercontent.com/o/r/v1/sub/workflow.yaml', 'https://raw.githubusercontent.com/o/r/v1/sub/workflow.yml']) + assert.equal(githubThumbnail(dir), 'https://raw.githubusercontent.com/o/r/v1/sub/thumbnail.png') + const file = parseWorkflowEntry('github.com/o/r/flows/a.yaml') + assert.deepEqual(githubYamlUrls(file), ['https://raw.githubusercontent.com/o/r/HEAD/flows/a.yaml']) +}) + +const YAML = 'on:\n execute:\n inputs:\n message:\n type: string\n label: Message\njobs:\n main:\n steps:\n - run: echo hi\n' +const calls = [] +const { workflowsTabRoutes, setWorkflowsCli, resetWorkflowsTabForTests } = await import('../dist/workflowsTab.js') +setWorkflowsCli(async args => { + calls.push(args) + if (args[0] === 'marketplace' && args[1] === 'ls') return JSON.stringify([{ slug: 'jupyter', name: 'Jupyter', description: 'Notebook server', type: 'workflow' }, { slug: 'a-bucket', type: 'storage' }]) + if (args[0] === 'marketplace' && args[1] === 'get') return YAML + if (args[0] === 'workflows' && args[1] === 'run') return JSON.stringify({ run: { slug: 'run-00001' } }) + throw new Error(`unexpected ${args.join(' ')}`) +}) +const realFetch = globalThis.fetch +globalThis.fetch = async (url, init) => { + if (String(url) === 'https://raw.githubusercontent.com/example-org/batch-flow/v2/workflow.yaml') return new Response(YAML, { status: 200 }) + if (String(url).startsWith('https://raw.githubusercontent.com/')) return new Response('nope', { status: 404 }) + return realFetch(url, init) +} + +const { default: Fastify } = await import('fastify') +const { sanitizedErrorHandler } = await import('../dist/routes.js') +const app = Fastify() +app.setErrorHandler(sanitizedErrorHandler(app)) +await app.register(workflowsTabRoutes) +await app.ready() +const j = async (method, url, body) => { + const res = await app.inject({ method, url, ...(body ? { payload: body } : {}) }) + return { status: res.statusCode, body: res.json() } +} +const q = s => encodeURIComponent(s) + +test('tiles for marketplace and GitHub entries need no copy in the account', async () => { + resetWorkflowsTabForTests() + const { body } = await j('GET', '/api/workflows/collection') + assert.deepEqual(body.workflows.map(w => [w.name, w.kind, w.displayName, w.installed, w.available]), [ + ['marketplace/jupyter', 'marketplace', 'Jupyter', true, true], + ['github.com/example-org/batch-flow@v2', 'github', 'batch-flow', true, true], + ]) +}) + +test('the form comes from the marketplace YAML and from the repository file', async () => { + const mp = await j('GET', `/api/workflows/item/form?w=${q('marketplace/jupyter')}`) + assert.equal(mp.status, 200) + assert.ok(mp.body.form && Object.keys(mp.body.form).length) + const gh = await j('GET', `/api/workflows/item/form?w=${q('github.com/example-org/batch-flow@v2')}`) + assert.equal(gh.status, 200) + assert.ok(gh.body.form && Object.keys(gh.body.form).length) + const missing = await j('GET', `/api/workflows/item/form?w=${q('github.com/example-org/absent')}`) + assert.equal(missing.status, 404) + assert.match(missing.body.error, /No workflow\.yaml/) +}) + +test('runs pass the entry to pw workflows run as written', async () => { + calls.length = 0 + const { body } = await j('POST', `/api/workflows/item/run?w=${q('github.com/example-org/batch-flow@v2')}`, { inputs: { message: 'hi' } }) + assert.equal(body.ok, true) + const run = calls.find(c => c[1] === 'run') + assert.equal(run[2], 'github.com/example-org/batch-flow@v2') + const mp = await j('POST', `/api/workflows/item/run?w=${q('marketplace/jupyter')}`, { inputs: {}, dryRun: true }) + assert.equal(mp.body.ok, true) + assert.equal(calls.filter(c => c[1] === 'run').pop()[2], 'marketplace/jupyter') +}) + +test('a malformed entry is refused, and installing is only for account workflows', async () => { + assert.equal((await j('GET', `/api/workflows/item/form?w=${q('gitlab.com/a/b')}`)).status, 400) + assert.equal((await j('POST', `/api/workflows/item/install?w=${q('marketplace/jupyter')}`)).status, 409) +}) + +test('Settings accepts all three kinds and drops the rest', async () => { + const { settingsRoutes, effectiveSettings } = await import('../dist/settings.js') + const s = Fastify(); await s.register(settingsRoutes); await s.ready() + const res = await s.inject({ method: 'PUT', url: '/api/settings', payload: { workflowCollection: ['batch', 'marketplace/jupyter', 'https://github.com/o/r@main', 'bad entry', 'gitlab.com/x/y'] } }) + assert.equal(res.statusCode, 200) + assert.deepEqual(effectiveSettings().workflowCollection, ['batch', 'marketplace/jupyter', 'github.com/o/r@main']) +}) diff --git a/server/test/workflowsTab.test.mjs b/server/test/workflowsTab.test.mjs index 81a6d89..1512276 100644 --- a/server/test/workflowsTab.test.mjs +++ b/server/test/workflowsTab.test.mjs @@ -76,20 +76,20 @@ test('the catalog lists the deployment account and flags the curated ones', asyn }) test('the form converts the workflow inputs and carries saved configurations', async () => { - const { status, body } = await j('GET', '/api/workflows/batch-job/form') + const { status, body } = await j('GET', '/api/workflows/item/form?w=batch-job') assert.equal(status, 200) assert.ok(body.form && typeof body.form === 'object') assert.deepEqual(body.configurations, [{ name: 'small', inputs: { steps: 1 } }]) }) test('a workflow outside the collection is refused', async () => { - const { status } = await j('POST', '/api/workflows/other/run', { inputs: {}, dryRun: true }) + const { status } = await j('POST', '/api/workflows/item/run?w=other', { inputs: {}, dryRun: true }) assert.equal(status, 403) }) test('a picker object is submitted as its pw:// reference', async () => { calls.length = 0 - const { body } = await j('POST', '/api/workflows/batch-job/run', { + const { body } = await j('POST', '/api/workflows/item/run?w=batch-job', { inputs: { resource: { name: 'gpu', user: 'someone', schedulerType: 'slurm', _studioRef: 'pw://someone/gpu' }, steps: 3 }, }) assert.equal(body.ok, true) @@ -100,7 +100,7 @@ test('a picker object is submitted as its pw:// reference', async () => { }) test('a dry run validates without recording a run', async () => { - const { body } = await j('POST', '/api/workflows/batch-job/run', { inputs: { steps: 1 }, dryRun: true }) + const { body } = await j('POST', '/api/workflows/item/run?w=batch-job', { inputs: { steps: 1 }, dryRun: true }) assert.equal(body.ok, true) assert.equal(body.dryRun, true) }) diff --git a/web/src/components/WorkflowsSection.tsx b/web/src/components/WorkflowsSection.tsx index 24a9b69..0ec4046 100644 --- a/web/src/components/WorkflowsSection.tsx +++ b/web/src/components/WorkflowsSection.tsx @@ -7,11 +7,15 @@ import { useEffect, useMemo, useState } from 'react' * site's own workflows. Order is the order of the tiles. */ -interface CatalogRow { name: string; displayName: string; description: string; tags: string[]; curated: boolean } +interface CatalogRow { name: string; displayName: string; description: string; tags?: string[]; curated: boolean } export function WorkflowsSection() { const [catalog, setCatalog] = useState(null) const [source, setSource] = useState<'deployment' | 'viewer'>('deployment') + const [market, setMarket] = useState([]) + const [gh, setGh] = useState('') + const [ghNote, setGhNote] = useState('') + const [ghBusy, setGhBusy] = useState(false) const [picked, setPicked] = useState([]) const [filter, setFilter] = useState('') const [note, setNote] = useState('') @@ -25,14 +29,29 @@ export function WorkflowsSection() { if (!r.ok) throw new Error(d.error ?? `${r.status}`) setCatalog(d.workflows) setSource(d.source === 'viewer' ? 'viewer' : 'deployment') + setMarket(Array.isArray(d.marketplace) ? d.marketplace : []) }).catch(e => setNote(`Cannot list the platform's workflows: ${(e as Error).message}`)) }, []) - const byName = useMemo(() => new Map((catalog ?? []).map(w => [w.name, w])), [catalog]) - const shown = (catalog ?? []).filter(w => { + const byName = useMemo(() => new Map([...(catalog ?? []), ...market].map(w => [w.name, w])), [catalog, market]) + const matches = (w: CatalogRow) => { const q = filter.trim().toLowerCase() - return !q || w.name.toLowerCase().includes(q) || w.displayName.toLowerCase().includes(q) || w.tags.some(t => t.toLowerCase().includes(q)) - }) + return !q || w.name.toLowerCase().includes(q) || w.displayName.toLowerCase().includes(q) || (w.tags ?? []).some(t => t.toLowerCase().includes(q)) + } + const shown = (catalog ?? []).filter(matches) + const shownMarket = market.filter(matches) + const addGithub = async () => { + const entry = gh.trim().replace(/^https?:\/\//, '').replace(/\.git$/, '') + if (!entry) return + setGhBusy(true); setGhNote('') + try { + const r = await fetch(`/api/workflows/item/form?w=${encodeURIComponent(entry)}`) + const d = await r.json().catch(() => ({})) + if (!r.ok) throw new Error(d.error ?? `${r.status}`) + if (!picked.includes(entry)) { setPicked(p => [...p, entry]); setDirty(true) } + setGh(''); setGhNote(`Added ${entry}. Save to offer it.`) + } catch (e) { setGhNote(String((e as Error).message)) } finally { setGhBusy(false) } + } const toggle = (name: string) => { setDirty(true); setPicked(p => p.includes(name) ? p.filter(n => n !== name) : [...p, name]) } const move = (i: number, d: -1 | 1) => { const j = i + d @@ -55,8 +74,9 @@ export function WorkflowsSection() {

Workflows

The workflows this Studio offers as tiles on its Workflows tab, each run from its own form under the viewer's - account. Pick them from the workflows on the platform account this Studio is deployed under. A viewer who does - not have one yet can add a copy to their account from the tile. + account. Pick them from the account this Studio is deployed under, from the marketplace, or from a GitHub + repository. A viewer who lacks an account workflow can add a copy from its tile; marketplace and GitHub + workflows need no copy.

Offered here ({picked.length})
@@ -67,7 +87,7 @@ export function WorkflowsSection() {
  • {byName.get(n)?.displayName ?? n} {n} - {catalog && !byName.has(n) && not on the platform account} + {catalog && !n.includes('/') && !byName.has(n) && not on the platform account} @@ -82,7 +102,7 @@ export function WorkflowsSection() { {note && {note}} -
    On the platform{catalog ? ` (${catalog.length})` : ''}
    +
    On the account{catalog ? ` (${catalog.length})` : ''}
    {catalog && source === 'viewer' && (

    Listed from your own account, because the deployment's platform credential is not usable. Until it is renewed, other viewers can run a workflow picked here only if it is already in their own account.

    )} @@ -104,6 +124,38 @@ export function WorkflowsSection() { )} + +
    Marketplace{market.length ? ` (${market.length})` : ''}
    + {market.length === 0 && catalog &&

    No marketplace workflows are listed for this account.

    } + {shownMarket.length > 0 && ( +
    + + + + {shownMarket.map(w => ( + + + + + + ))} + +
    WorkflowDescription
    toggle(w.name)} aria-label={`Offer ${w.displayName}`} />
    {w.displayName}
    {w.name}    		{w.description}
    +
    + )} + +
    From GitHub
    +

    + A repository, or a directory or file in one, holding a workflow.yaml: github.com/owner/repo, + optionally with a path and @branch, tag, or commit. It runs as it is in the repository, without a copy + in anyone's account. The form is read from public repositories; a private one still runs. +

    +
    + setGh(e.target.value)} + onKeyDown={e => { if (e.key === 'Enter') void addGithub() }} /> + +
    + {ghNote &&

    {ghNote}

    } ) } diff --git a/web/src/styles.css b/web/src/styles.css index d226b46..67d3f62 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -1913,3 +1913,4 @@ code, pre, kbd, .text-body, .viewer-path, .sql-box, [data-streamdown='code-block .chats-manager { padding: 12px 14px 24px; } .chats-row-meta { display: none; } } +.wf-kind { font-size: 11px; color: var(--pw-muted); border: 1px solid var(--pw-border); border-radius: 999px; padding: 0 7px; } diff --git a/web/src/views/WorkflowsView.tsx b/web/src/views/WorkflowsView.tsx index 5614228..bf874b0 100644 --- a/web/src/views/WorkflowsView.tsx +++ b/web/src/views/WorkflowsView.tsx @@ -13,6 +13,7 @@ import { useWorkflowFields, forgetPlatformData } from '../components/WorkflowFie interface Tile { name: string; displayName: string; description: string; tags: string[] icon: string | null; configurations: string[]; installed: boolean; available: boolean + kind?: 'account' | 'marketplace' | 'github' } interface FormDoc { name: string; displayName: string; description: string @@ -27,6 +28,14 @@ async function json(res: Response): Promise { return d as T } +/** A tile's icon, or its first letter when there is none or it fails to + * load (a repository without a thumbnail, say). */ +function TileIcon({ src, letter }: { src: string | null; letter: string }) { + const [failed, setFailed] = useState(false) + if (!src || failed) return {letter} + return setFailed(true)} /> +} + export function WorkflowsView() { const [tiles, setTiles] = useState(null) const [configured, setConfigured] = useState(true) @@ -44,7 +53,7 @@ export function WorkflowsView() { if (open) { const t = tiles?.find(x => x.name === open) - return { setOpen(null); loadRuns() }} onLaunched={loadRuns} /> + return { setOpen(null); loadRuns() }} onLaunched={loadRuns} /> } return (
    @@ -55,8 +64,9 @@ export function WorkflowsView() { {tiles.map(t => ( @@ -87,7 +97,7 @@ export function WorkflowsView() { ) } -function WorkflowRunner({ name, title, onBack, onLaunched }: { name: string; title: string; onBack: () => void; onLaunched: () => void }) { +function WorkflowRunner({ name, kind, title, onBack, onLaunched }: { name: string; kind: string; title: string; onBack: () => void; onLaunched: () => void }) { const [doc, setDoc] = useState(null) const [error, setError] = useState('') const [preset, setPreset] = useState('') @@ -98,8 +108,8 @@ function WorkflowRunner({ name, title, onBack, onLaunched }: { name: string; tit const fields = useWorkflowFields() const load = () => { setError(''); setNeedsInstall(false) - fetch(`/api/workflows/${encodeURIComponent(name)}/form`).then(r => json(r)).then(setDoc) - .catch(e => { const m = String((e as Error).message); if (/not found|404/i.test(m)) setNeedsInstall(true); else setError(m) }) + fetch(`/api/workflows/item/form?w=${encodeURIComponent(name)}`).then(r => json(r)).then(setDoc) + .catch(e => { const m = String((e as Error).message); if (kind === 'account' && /not found|404/i.test(m)) setNeedsInstall(true); else setError(m) }) } useEffect(() => { forgetPlatformData(); load() }, [name]) @@ -114,7 +124,7 @@ function WorkflowRunner({ name, title, onBack, onLaunched }: { name: string; tit const values = formik.current?.values ?? {} setBusy(dryRun ? 'validate' : 'run'); setResult(null) try { - const r = await json<{ ok: boolean; message: string; slug?: string | null }>(await fetch(`/api/workflows/${encodeURIComponent(name)}/run`, { + const r = await json<{ ok: boolean; message: string; slug?: string | null }>(await fetch(`/api/workflows/item/run?w=${encodeURIComponent(name)}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ inputs: values, dryRun }), })) setResult({ ok: r.ok, text: r.ok && dryRun ? 'Validation passed. The platform accepts these inputs.' : r.message }) @@ -123,7 +133,7 @@ function WorkflowRunner({ name, title, onBack, onLaunched }: { name: string; tit } const install = async () => { setBusy('install') - try { await json(await fetch(`/api/workflows/${encodeURIComponent(name)}/install`, { method: 'POST' })); load() } + try { await json(await fetch(`/api/workflows/item/install?w=${encodeURIComponent(name)}`, { method: 'POST' })); load() } catch (e) { setError(String((e as Error).message)) } finally { setBusy('') } }