From db0e654196a5817e92fd575bc77dcff6d7fb4489 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 07:49:26 +0000 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20make=20share-ready=20=E2=80=94=20gre?= =?UTF-8?q?en=20CI,=20honest=20discovery,=20docs=20polish?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unblock pnpm 11 CI (xlsx integrity + core-js allowBuilds), strip vapourware OAuth/MCP claims from agent discovery, add /api/health, and point public docs at datasink.dev / chrisschouk/sink-cli. Co-authored-by: Chris Schofield --- CHANGELOG.md | 17 ++ CLAUDE.md | 3 +- CONTRIBUTING.md | 4 +- README.md | 17 +- package.json | 8 +- pnpm-lock.yaml | 2 +- pnpm-workspace.yaml | 2 + web/api/firecrawl-proxy.ts | 2 +- web/api/health.ts | 23 +++ web/middleware.ts | 40 ++--- .../.well-known/agent-skills/auth-md/SKILL.md | 11 -- .../agent-skills/datasink/SKILL.md | 48 ++++-- .../.well-known/agent-skills/index.json | 12 +- web/public/.well-known/ai-plugin.json | 6 +- web/public/.well-known/api-catalog | 6 +- web/public/.well-known/mcp/server-card.json | 20 --- .../.well-known/oauth-authorization-server | 53 ------- .../.well-known/oauth-protected-resource | 15 -- web/public/.well-known/openid-configuration | 26 --- web/public/.well-known/security.txt | 3 +- web/public/auth.md | 68 ++++---- web/public/docs/openapi.json | 16 +- web/public/llms-full.txt | 149 ++++++++---------- web/public/llms.txt | 36 +++-- web/src/webmcp.ts | 2 +- web/vercel.json | 38 +---- 26 files changed, 250 insertions(+), 377 deletions(-) create mode 100644 web/api/health.ts delete mode 100644 web/public/.well-known/agent-skills/auth-md/SKILL.md delete mode 100644 web/public/.well-known/mcp/server-card.json delete mode 100644 web/public/.well-known/oauth-authorization-server delete mode 100644 web/public/.well-known/oauth-protected-resource delete mode 100644 web/public/.well-known/openid-configuration diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f58d2d..48c8782 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- CI install failed under pnpm 11 supply-chain checks because the SheetJS CDN + `xlsx` tarball had no lockfile integrity hash. Integrity is now recorded. +- Agent discovery docs claimed OAuth, MCP HTTP, and registration endpoints that + do not exist. Discovery files now match the real surface: CLI, browser demo, + `/api/health`, and `/api/firecrawl-proxy`. +- README pointed at the Vercel preview hostname and an outdated org path; primary + demo link is datasink.dev and GitHub links use `chrisschouk/sink-cli`. + +### Added + +- `GET /api/health` for uptime checks and the OpenAPI catalog. +- `sink demo` documented in the README command table. + ## [0.4.0] - 2026-06-11 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 1157809..291b956 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,8 +36,7 @@ for library consumers. ## Publish flow (npm) -Published as `datasink` (no scope). Current version 0.4.0 is not yet published to npm -(the web demo was the focus; CLI behaviour is unchanged from 0.3.x). +Published as `datasink` (no scope). Current version on npm is **0.4.0** (published 2026-06-11). **2FA gotcha**: interactive `npm publish` hits the 2FA wall in non-interactive environments. Use a granular **automation token** (npm account → Access Tokens → Generate New Token → diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 30dbb98..e72c7c1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Thanks for your interest in contributing. This guide covers what you need to get ## Dev Setup ```bash -git clone https://github.com/totalaudiopromo/sink-cli.git +git clone https://github.com/chrisschouk/sink-cli.git cd sink-cli pnpm install pnpm build @@ -20,6 +20,8 @@ pnpm build | `pnpm test` | Run all tests (Vitest) | | `pnpm test:watch` | Tests in watch mode | | `pnpm typecheck` | Type-check without emitting | +| `pnpm lint` | ESLint on `src/` and `test/` | +| `pnpm format:check` | Prettier check | ## Running locally diff --git a/README.md b/README.md index 8c8d51b..9a38495 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ``` [![npm version](https://img.shields.io/npm/v/datasink.svg)](https://www.npmjs.com/package/datasink) -[![CI](https://github.com/totalaudiopromo/sink-cli/actions/workflows/ci.yml/badge.svg)](https://github.com/totalaudiopromo/sink-cli/actions/workflows/ci.yml) +[![CI](https://github.com/chrisschouk/sink-cli/actions/workflows/ci.yml/badge.svg)](https://github.com/chrisschouk/sink-cli/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Node](https://img.shields.io/badge/node-%3E%3D20-brightgreen.svg)](https://nodejs.org) @@ -32,13 +32,15 @@ ## Try it in the browser -**[sink-web-indol.vercel.app](https://sink-web-indol.vercel.app)** — drop a CSV -and watch the real engine run client-side. Your contacts never leave your -browser; only domain names are checked against DNS. Source in [`web/`](web/). +**[datasink.dev](https://datasink.dev)** — drop a CSV or XLSX and watch the real +engine run client-side. Scrub and rinse stay in your browser; only domain names +are checked against DNS. AI phases (soak/steep) use your own API keys. Source in +[`web/`](web/). ## Quick Start ```bash +npx datasink demo # sample data, no file needed npx datasink scrub contacts.csv # validate emails npx datasink rinse contacts.csv # deduplicate npx datasink wash contacts.csv # full pipeline @@ -56,6 +58,7 @@ sink scrub contacts.csv | Command | Description | | --------------------- | --------------------------------------------- | | `sink` | Interactive menu (no args) | +| `sink demo` | Full pipeline on built-in sample data | | `sink wash ` | Full pipeline: scrub + rinse + soak + steep | | `sink scrub ` | Validate & clean emails | | `sink rinse ` | Deduplicate contacts | @@ -66,6 +69,10 @@ sink scrub contacts.csv | `sink drain ` | Convert between formats | | `sink tui ` | Full TUI dashboard | +> Soak and steep need provider keys (`ANTHROPIC_API_KEY` or `OPENAI_API_KEY`; +> steep also needs `FIRECRAWL_API_KEY`). Without keys those phases are skipped +> with a clear warning — scrub, rinse, spot, inspect, and demo still work. + ## Why sink? - **Built for music PR.** Knows BBC Radio 1 from Radio X, catches `bbc.com` → `bbc.co.uk` typos, flags role-based emails like `press@`. Not a generic email validator -- it understands your industry. @@ -256,7 +263,7 @@ Tools I build for music PR, by [Chris Schofield](https://x.com/chrisschouk). Par | [SpotCheck](https://spotcheck.cc) | Spotify playlist validation | | [Newsjack](https://newsjack.cc) | Music industry newsjacking | | [Podflow](https://github.com/totalaudiopromo/podflow) | Podcast intelligence for music PR | -| [Sink](https://github.com/totalaudiopromo/sink-cli) | Contact data hygiene CLI | +| [Sink](https://github.com/chrisschouk/sink-cli) | Contact data hygiene CLI | Questions? Reach me on [X/@chrisschouk](https://x.com/chrisschouk) or [info@totalaudiopromo.com](mailto:info@totalaudiopromo.com). diff --git a/package.json b/package.json index 9f99888..080ac64 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "datasink", "version": "0.4.0", - "description": "sink — data hygiene for music PR. Scrub, rinse, soak, steep your contact lists. The `sink` CLI, published as datasink.", + "description": "sink \u2014 data hygiene for music PR. Scrub, rinse, soak, steep your contact lists. The `sink` CLI, published as datasink.", "license": "MIT", "type": "module", "bin": { @@ -76,13 +76,13 @@ "csv", "data-hygiene" ], - "homepage": "https://github.com/totalaudiopromo/sink-cli#readme", + "homepage": "https://datasink.dev", "bugs": { - "url": "https://github.com/totalaudiopromo/sink-cli/issues" + "url": "https://github.com/chrisschouk/sink-cli/issues" }, "repository": { "type": "git", - "url": "https://github.com/totalaudiopromo/sink-cli.git" + "url": "https://github.com/chrisschouk/sink-cli.git" }, "author": "Total Audio Promo", "files": [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 87db2a4..b5b409b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1804,7 +1804,7 @@ packages: optional: true xlsx@https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz: - resolution: {tarball: https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz} + resolution: {integrity: sha512-oLDq3jw7AcLqKWH2AhCpVTZl8mf6X2YReP+Neh0SJUzV/BdZYjth94tG5toiMB1PPrYtxOCfaoUCkvtuH+3AJA==, tarball: https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz} version: 0.20.3 engines: {node: '>=0.8'} hasBin: true diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 89a0c08..b5d5011 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,6 +5,8 @@ packages: allowBuilds: # esbuild's postinstall fetches its platform binary; vitest needs it. esbuild: true + # posthog-js pulls core-js; its install script is optional — explicitly deny. + core-js: false overrides: # ws < 8.20.1 has an uninitialized-memory-disclosure advisory (GHSA-58qx); diff --git a/web/api/firecrawl-proxy.ts b/web/api/firecrawl-proxy.ts index 267e87a..bcddae6 100644 --- a/web/api/firecrawl-proxy.ts +++ b/web/api/firecrawl-proxy.ts @@ -6,7 +6,7 @@ * once for the upstream call, and is never logged or stored. Everything else in * sink web runs in the browser. * - * Source is intentionally tiny and open — see github.com/totalaudiopromo/sink-cli. + * Source is intentionally tiny and open — see github.com/chrisschouk/sink-cli. */ export const config = { runtime: 'edge' } diff --git a/web/api/health.ts b/web/api/health.ts new file mode 100644 index 0000000..f309c57 --- /dev/null +++ b/web/api/health.ts @@ -0,0 +1,23 @@ +/** + * Tiny health endpoint for uptime checks and the OpenAPI catalog. + * No secrets, no side effects. + */ + +export const config = { runtime: 'edge' } + +export default function handler(): Response { + return new Response( + JSON.stringify({ + status: 'ok', + version: '0.4.0', + service: 'datasink', + }), + { + status: 200, + headers: { + 'Content-Type': 'application/json', + 'Cache-Control': 'public, max-age=60', + }, + }, + ) +} diff --git a/web/middleware.ts b/web/middleware.ts index a399a9a..91f7d3a 100644 --- a/web/middleware.ts +++ b/web/middleware.ts @@ -10,38 +10,38 @@ export default function middleware(request: Request): Response | void { sink scrubs, rinses, soaks, and steeps your contact lists. ## Overview -datasink is a data hygiene CLI and web application built for music PR and press lists. -It cleans formatting errors, deduplicates entries across fields, enriches contact metadata, and segments list relevance. -## Core Capabilities -- **Scrub**: Format validation, email syntax checking, domain syntax checks, and typo mapping. -- **Rinse**: Multi-field deduplication by email, name, and publication domain. -- **Soak**: Contact enrichment via LLM queries and outlet metadata extraction. -- **Steep**: AI-assisted list segmenting and campaign relevance scoring. +datasink is a data hygiene CLI and browser demo for music PR contact lists. +Scrub and rinse run locally in the browser. Soak and steep use bring-your-own-key AI providers. + +## Core capabilities + +- **Scrub**: format validation, typo mapping, disposable domains, role accounts, MX checks +- **Rinse**: multi-field deduplication +- **Soak**: LLM contact enrichment (your API key) +- **Steep**: outlet channel discovery via Firecrawl (your API key) + +## Getting started -## Getting Started -CLI installation: \`\`\`bash +npx datasink demo npx datasink scrub contacts.csv \`\`\` -## Agent Discovery & Endpoints -- Web App: https://datasink.dev -- API Catalog: https://datasink.dev/.well-known/api-catalog -- ChatGPT Plugin Manifest: https://datasink.dev/.well-known/ai-plugin.json -- Extended LLM Spec: https://datasink.dev/llms-full.txt -- Standard LLM Doc: https://datasink.dev/llms.txt -- MCP Server Card: https://datasink.dev/.well-known/mcp/server-card.json -- Agent Skills Index: https://datasink.dev/.well-known/agent-skills/index.json -- Agent Auth: https://datasink.dev/auth.md -- Security Policy: https://datasink.dev/.well-known/security.txt +## Links + +- Web demo: https://datasink.dev +- npm: https://www.npmjs.com/package/datasink +- Source: https://github.com/chrisschouk/sink-cli +- LLM docs: https://datasink.dev/llms.txt +- Auth notes: https://datasink.dev/auth.md +- Security: https://datasink.dev/.well-known/security.txt ` return new Response(markdownContent, { status: 200, headers: { 'content-type': 'text/markdown; charset=utf-8', - 'x-markdown-tokens': '210', 'cache-control': 'public, max-age=3600', }, }) diff --git a/web/public/.well-known/agent-skills/auth-md/SKILL.md b/web/public/.well-known/agent-skills/auth-md/SKILL.md deleted file mode 100644 index 54f54cb..0000000 --- a/web/public/.well-known/agent-skills/auth-md/SKILL.md +++ /dev/null @@ -1,11 +0,0 @@ -# Auth.md Agent Registration Skill - -Register and authenticate AI agents with datasink API endpoints. - -## Overview -This skill describes how an agent requests registration, asserts identity (ID-JAG or verified email), obtains bearer credentials, and invokes protected datasink services. - -## Metadata Endpoints -- `/auth.md`: Registration workflow instructions. -- `/.well-known/oauth-protected-resource`: Resource scopes and authorization server declarations. -- `/.well-known/oauth-authorization-server`: Token issuing parameters and registration endpoints. diff --git a/web/public/.well-known/agent-skills/datasink/SKILL.md b/web/public/.well-known/agent-skills/datasink/SKILL.md index 531324a..9b665ce 100644 --- a/web/public/.well-known/agent-skills/datasink/SKILL.md +++ b/web/public/.well-known/agent-skills/datasink/SKILL.md @@ -1,20 +1,34 @@ # datasink Contact Hygiene Skill -Clean, format, deduplicate, and enrich contact CSV lists for music PR campaigns. - -## Overview -datasink provides 4 distinct data hygiene phases: -1. **Scrub**: Validates email syntax, detects domain typos, normalizes names. -2. **Rinse**: Deduplicates entries across email addresses, full names, and domains. -3. **Soak**: Enriches outlet and contact data using web search and LLM extraction. -4. **Steep**: Segments lists and scores contact relevance for target music genres. - -## WebMCP Tools Available -In the browser, use `navigator.modelContext` tools: -- `scrub_contacts`: Run format and typo validation. -- `rinse_contacts`: Remove duplicate contacts. - -## CLI Usage -\`\`\`bash +Clean, deduplicate, and enrich music PR contact lists with the `sink` CLI (npm: `datasink`) or the browser demo at https://datasink.dev. + +## Phases + +1. **Scrub** — email format, typos, disposable domains, role accounts, MX +2. **Rinse** — exact-email / fuzzy-name / cross-field dedup +3. **Soak** — LLM enrichment (needs Anthropic or OpenAI key) +4. **Steep** — outlet channel discovery via Firecrawl + LLM (needs keys) + +## CLI + +```bash +npx datasink demo npx datasink scrub contacts.csv -\`\`\` +npx datasink wash contacts.csv +npx datasink spot name@outlet.co.uk +``` + +Keys (optional; soak/steep skipped without them): + +```bash +export ANTHROPIC_API_KEY=sk-ant-... +export FIRECRAWL_API_KEY=fc-... +``` + +## Browser + +https://datasink.dev — drop CSV/XLSX. Scrub/rinse are local. AI phases use bring-your-own-key. + +## Source + +https://github.com/chrisschouk/sink-cli diff --git a/web/public/.well-known/agent-skills/index.json b/web/public/.well-known/agent-skills/index.json index 7f6a99e..18ff24b 100644 --- a/web/public/.well-known/agent-skills/index.json +++ b/web/public/.well-known/agent-skills/index.json @@ -4,16 +4,8 @@ { "name": "datasink", "type": "skill-md", - "description": "Data hygiene workflows for music PR contact lists (scrub, rinse, soak, steep).", - "url": "https://datasink.dev/.well-known/agent-skills/datasink/SKILL.md", - "digest": "sha256:d24189d2248a8905ac17f46796d1bf0dc98ddf986c61123194a139014d21ae9c" - }, - { - "name": "auth-md", - "type": "skill-md", - "description": "Agent registration and authentication for datasink API services.", - "url": "https://datasink.dev/.well-known/agent-skills/auth-md/SKILL.md", - "digest": "sha256:80411a7c95148806038f258ae8c7b7ffbf49b51cf0d6740bce87d17fa4b7874b" + "description": "Data hygiene workflows for music PR contact lists (scrub, rinse, soak, steep) via CLI and browser demo.", + "url": "https://datasink.dev/.well-known/agent-skills/datasink/SKILL.md" } ] } diff --git a/web/public/.well-known/ai-plugin.json b/web/public/.well-known/ai-plugin.json index 68e49cc..fb53aa3 100644 --- a/web/public/.well-known/ai-plugin.json +++ b/web/public/.well-known/ai-plugin.json @@ -2,8 +2,8 @@ "schema_version": "v1", "name_for_human": "datasink", "name_for_model": "datasink", - "description_for_human": "Data hygiene suite for music PR contact lists. Scrub, rinse, soak, steep.", - "description_for_model": "Plugin for scrubbing, deduplicating, validating syntax, and enriching music PR contact lists and email addresses.", + "description_for_human": "Data hygiene for music PR contact lists. Scrub, rinse, soak, steep.", + "description_for_model": "Validate and deduplicate music PR contact lists. Prefer the npm CLI (npx datasink) for batch work. The web demo at datasink.dev runs scrub/rinse in-browser; soak/steep need user-supplied API keys. Server HTTP API is limited to /api/health and /api/firecrawl-proxy.", "auth": { "type": "none" }, @@ -12,6 +12,6 @@ "url": "https://datasink.dev/docs/openapi.json" }, "logo_url": "https://datasink.dev/favicon.svg", - "contact_email": "support@totalaudiopromo.com", + "contact_email": "info@totalaudiopromo.com", "legal_info_url": "https://datasink.dev" } diff --git a/web/public/.well-known/api-catalog b/web/public/.well-known/api-catalog index 49f2e33..c38f052 100644 --- a/web/public/.well-known/api-catalog +++ b/web/public/.well-known/api-catalog @@ -1,7 +1,7 @@ { "linkset": [ { - "anchor": "https://datasink.dev/api", + "anchor": "https://datasink.dev/", "service-desc": [ { "href": "https://datasink.dev/docs/openapi.json", @@ -10,8 +10,8 @@ ], "service-doc": [ { - "href": "https://datasink.dev/docs/api", - "type": "text/html" + "href": "https://datasink.dev/llms-full.txt", + "type": "text/plain" } ], "status": [ diff --git a/web/public/.well-known/mcp/server-card.json b/web/public/.well-known/mcp/server-card.json deleted file mode 100644 index b8dbb71..0000000 --- a/web/public/.well-known/mcp/server-card.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "serverInfo": { - "name": "datasink", - "version": "0.4.0", - "description": "Data hygiene suite for music PR contact lists: scrub, rinse, soak, steep." - }, - "endpoint": "/api/mcp", - "capabilities": { - "tools": { - "listChanged": true - }, - "resources": { - "subscribe": false, - "listChanged": true - }, - "prompts": { - "listChanged": true - } - } -} diff --git a/web/public/.well-known/oauth-authorization-server b/web/public/.well-known/oauth-authorization-server deleted file mode 100644 index d436088..0000000 --- a/web/public/.well-known/oauth-authorization-server +++ /dev/null @@ -1,53 +0,0 @@ -{ - "issuer": "https://datasink.dev", - "authorization_endpoint": "https://datasink.dev/oauth/authorize", - "token_endpoint": "https://datasink.dev/oauth/token", - "jwks_uri": "https://datasink.dev/.well-known/jwks.json", - "grant_types_supported": [ - "authorization_code", - "client_credentials", - "urn:ietf:params:oauth:grant-type:token-exchange" - ], - "response_types_supported": [ - "code", - "token" - ], - "token_endpoint_auth_methods_supported": [ - "client_secret_basic", - "client_secret_post", - "private_key_jwt" - ], - "scopes_supported": [ - "read", - "write", - "scrub", - "enrich" - ], - "agent_auth": { - "skill": "https://datasink.dev/.well-known/agent-skills/auth-md/SKILL.md", - "register_uri": "https://datasink.dev/agent/auth", - "identity_types_supported": [ - "anonymous", - "identity_assertion" - ], - "anonymous": { - "credential_types_supported": [ - "bearer" - ] - }, - "identity_assertion": { - "assertion_types_supported": [ - "urn:ietf:params:oauth:token-type:id-jag", - "verified_email" - ], - "credential_types_supported": [ - "bearer" - ] - }, - "claim_uri": "https://datasink.dev/agent/claim", - "revocation_uri": "https://datasink.dev/agent/revoke", - "events_supported": [ - "revocation" - ] - } -} diff --git a/web/public/.well-known/oauth-protected-resource b/web/public/.well-known/oauth-protected-resource deleted file mode 100644 index 33bb920..0000000 --- a/web/public/.well-known/oauth-protected-resource +++ /dev/null @@ -1,15 +0,0 @@ -{ - "resource": "https://datasink.dev", - "authorization_servers": [ - "https://datasink.dev" - ], - "scopes_supported": [ - "read", - "write", - "scrub", - "enrich" - ], - "bearer_methods_supported": [ - "header" - ] -} diff --git a/web/public/.well-known/openid-configuration b/web/public/.well-known/openid-configuration deleted file mode 100644 index 30ad609..0000000 --- a/web/public/.well-known/openid-configuration +++ /dev/null @@ -1,26 +0,0 @@ -{ - "issuer": "https://datasink.dev", - "authorization_endpoint": "https://datasink.dev/oauth/authorize", - "token_endpoint": "https://datasink.dev/oauth/token", - "jwks_uri": "https://datasink.dev/.well-known/jwks.json", - "grant_types_supported": [ - "authorization_code", - "client_credentials", - "urn:ietf:params:oauth:grant-type:token-exchange" - ], - "response_types_supported": [ - "code", - "token" - ], - "token_endpoint_auth_methods_supported": [ - "client_secret_basic", - "client_secret_post", - "private_key_jwt" - ], - "scopes_supported": [ - "read", - "write", - "scrub", - "enrich" - ] -} diff --git a/web/public/.well-known/security.txt b/web/public/.well-known/security.txt index 3421360..34d227f 100644 --- a/web/public/.well-known/security.txt +++ b/web/public/.well-known/security.txt @@ -1,5 +1,4 @@ -Contact: mailto:security@totalaudiopromo.com +Contact: mailto:info@totalaudiopromo.com Expires: 2027-12-31T23:59:59.000Z Preferred-Languages: en Canonical: https://datasink.dev/.well-known/security.txt -Policy: https://datasink.dev/security diff --git a/web/public/auth.md b/web/public/auth.md index 8619345..f068a8c 100644 --- a/web/public/auth.md +++ b/web/public/auth.md @@ -1,38 +1,30 @@ -# datasink.dev auth.md - -Agent registration and authentication specifications for the datasink platform. - -## Agent Audience -This service accepts registration requests from automated AI agents, browser extensions, and programmatic integrations seeking to execute contact hygiene operations (scrub, rinse, soak, steep). - -## Registration & Provisioning -Agents register dynamically by issuing a request to the registration endpoint: -- **Registration URI**: `https://datasink.dev/agent/auth` -- **Claim URI**: `https://datasink.dev/agent/claim` -- **Revocation URI**: `https://datasink.dev/agent/revoke` - -## Supported Identity Types & Flows - -### 1. Anonymous Access -- **Supported Identity**: `anonymous` -- **Credential Type**: `bearer` -- Allows restricted rate-limited access for basic scrubbing and catalog discovery. - -### 2. Identity Assertion -- **Supported Identity**: `identity_assertion` -- **Assertion Types**: - - `urn:ietf:params:oauth:token-type:id-jag` (Identity-JWT Assertion Grant) - - `verified_email` -- **Credential Type**: `bearer` - -## Authentication & Authorization Metadata -- **Protected Resource Metadata (PRM)**: [/.well-known/oauth-protected-resource](https://datasink.dev/.well-known/oauth-protected-resource) -- **OAuth Authorization Server**: [/.well-known/oauth-authorization-server](https://datasink.dev/.well-known/oauth-authorization-server) -- **Bearer Token Transmission**: Pass credentials in the standard HTTP Header: - `Authorization: Bearer ` - -## Scopes Supported -- `read`: Query public API catalogs and documentation. -- `scrub`: Perform email syntax, typo mapping, and format validation. -- `enrich`: Perform contact enrichment and web metadata queries. -- `write`: Modify and save processed contact datasets. +# datasink.dev — how auth works + +datasink does not run a hosted OAuth or agent registration server. + +## Browser demo ([datasink.dev](https://datasink.dev)) + +- Scrub and rinse run entirely in your browser. +- Soak and steep use **bring-your-own-key**: you paste an Anthropic or OpenAI key (and optionally a Firecrawl key for steep). Keys stay in the browser session and are never stored on our servers. +- The only server endpoint is `/api/firecrawl-proxy`, which forwards one scrape request with the Firecrawl key you supply for that call. + +## CLI (`npx datasink`) + +Set provider keys in your environment when you want AI phases: + +```bash +export ANTHROPIC_API_KEY=sk-ant-... +# or +export OPENAI_API_KEY=sk-... + +# steep also needs: +export FIRECRAWL_API_KEY=fc-... + +npx datasink wash contacts.csv +``` + +Without keys, scrub / rinse / inspect / spot / demo still work. Soak and steep are skipped with a clear warning. + +## What is not here + +There is no `/oauth/*`, no agent claim/revoke API, and no server-side bearer token store. Discovery docs that implied those endpoints were wrong and have been removed. diff --git a/web/public/docs/openapi.json b/web/public/docs/openapi.json index 90ccfbc..6d08452 100644 --- a/web/public/docs/openapi.json +++ b/web/public/docs/openapi.json @@ -2,25 +2,24 @@ "openapi": "3.0.3", "info": { "title": "datasink API", - "description": "Data hygiene API for music PR contact lists. Scrub syntax and typos, deduplicate contacts, enrich media metadata, and segment list relevance.", + "description": "Small HTTP surface for the datasink web demo. Scrub/rinse/soak run in the browser; only health and the Firecrawl proxy are server-side.", "version": "0.4.0", "contact": { "name": "Total Audio Promo", "url": "https://totalaudiopromo.com", - "email": "support@totalaudiopromo.com" + "email": "info@totalaudiopromo.com" } }, "servers": [ { "url": "https://datasink.dev", - "description": "Production server" + "description": "Production" } ], "paths": { "/api/health": { "get": { "summary": "Health check", - "description": "Returns operational status of datasink web services.", "responses": { "200": { "description": "Healthy", @@ -30,7 +29,8 @@ "type": "object", "properties": { "status": { "type": "string", "example": "ok" }, - "version": { "type": "string", "example": "0.4.0" } + "version": { "type": "string", "example": "0.4.0" }, + "service": { "type": "string", "example": "datasink" } } } } @@ -41,8 +41,8 @@ }, "/api/firecrawl-proxy": { "post": { - "summary": "Outlet page scrape", - "description": "Proxy for retrieving outlet markdown via Firecrawl for steep phase analysis.", + "summary": "Outlet page scrape proxy", + "description": "Forwards one Firecrawl scrape. Supply your own Firecrawl key in the body; it is not stored.", "requestBody": { "required": true, "content": { @@ -60,7 +60,7 @@ }, "responses": { "200": { - "description": "Scraped markdown output", + "description": "Scraped markdown", "content": { "application/json": { "schema": { diff --git a/web/public/llms-full.txt b/web/public/llms-full.txt index 8c33221..b77a429 100644 --- a/web/public/llms-full.txt +++ b/web/public/llms-full.txt @@ -1,116 +1,95 @@ -# datasink Extended Technical Specification & LLM Reference +# datasink Extended Technical Spec -> Full schema definitions, phase algorithms, cURL examples, and CLI parameter reference for AI agents, high-context LLMs, and automated integrations. +> Accurate reference for AI agents and humans. Only endpoints and behaviours that exist in production are documented here. ## Overview -`datasink` (published on npm as `datasink`, binary `sink`) cleans and enriches music PR contact lists through a 4-stage pipeline: `scrub` -> `rinse` -> `soak` -> `steep`. + +`datasink` (npm package name; CLI binary `sink`) cleans music PR contact lists through four phases: `scrub` → `rinse` → `soak` → `steep`. + +Repo: https://github.com/chrisschouk/sink-cli +Demo: https://datasink.dev +Install: `npx datasink scrub contacts.csv` --- -## Technical Pipeline Phases +## Pipeline phases -### Phase 1: Scrub (Format & Deliverability Validation) -- **Email Regex & Normalisation**: Lowercases domains, trims whitespace, standardizes separators. -- **Typo Correction Engine**: Applies 200+ domain typo mappings (e.g., `hotmial.com` -> `hotmail.com`, `gamil.com` -> `gmail.com`). -- **Disposable Domain Filter**: Checks against 3,000+ known disposable and burner email domains. -- **MX Record Validation**: Checks domain DNS for active mail exchanger records. -- **Role Account Tagging**: Identifies non-personal mailboxes (`info@`, `press@`, `contact@`, `editor@`). +### Scrub -### Phase 2: Rinse (Deduplication) -- **Exact Email Matching**: Merges duplicate records sharing the same normalized email address. -- **Fuzzy Name Matching**: Jaro-Winkler string similarity (threshold >= 0.92) across contact names within identical publication domains. -- **Cross-Field Reconciliation**: Combines raw field metadata into single primary records, preserving audit histories in `phases[]`. +- RFC-style email format checks, normalisation, UK domain typo correction +- Disposable domain detection, role-based mailbox tagging (`press@`, `info@`, …) +- MX verification (Node `dns` in CLI; Cloudflare DNS-over-HTTPS in browser — domains only) -### Phase 3: Soak (LLM Metadata Enrichment) -- **Provider Support**: Anthropic Claude (`claude-3-7-sonnet`, `claude-3-5-haiku`) or OpenAI (`gpt-4o`). -- **Enrichment Target**: Extracts missing contact names, job titles, Instagram handles, music genres covered, and outlet locations. +### Rinse -### Phase 4: Steep (Targeting & Relevance Scoring) -- **Outlet Web Scrape**: Uses Firecrawl (`/v1/scrape`) to retrieve fresh submission guidelines from publication websites. -- **Relevance Scoring**: Evaluates candidate contacts against campaign genre tags and target music styles (score 0-100). +- Exact email dedup (case-insensitive, keeps richer record) +- Fuzzy name match (Jaro-Winkler, default threshold 0.92) within the same domain +- Cross-field match by phone across different emails ---- +### Soak -## Machine Schemas & Output Data Model - -### SinkRecord Schema (JSON) -```json -{ - "id": "uuid-v4-string", - "raw": { - "name": "Jane Doe", - "email": "jane@musicblog.com", - "outlet": "The Music Blog" - }, - "scrub": { - "email": { - "original": "jane@musicblog.com", - "normalised": "jane@musicblog.com", - "valid": true, - "confidence": "high", - "corrected": false, - "disposable": false, - "roleBased": false, - "catchAll": false, - "checks": { - "regex": true, - "mx": true - } - } - }, - "rinse": { - "duplicate": false, - "mergedWith": null, - "matchType": null - }, - "timestamp": "2026-07-25T22:00:00.000Z" -} -``` +- LLM enrichment (platform type, genre, geo, submission tips) +- Providers: Anthropic or OpenAI via env keys / web BYOK panel +- Silently skipped when no key is present + +### Steep + +- Scrapes outlet public pages (via Firecrawl) and extracts channels, portals, presenters, pitch hooks +- Needs `FIRECRAWL_API_KEY` plus an LLM key; skipped when missing +- Web demo proxies Firecrawl through `/api/firecrawl-proxy` because of CORS --- -## cURL & API Integration Examples +## HTTP endpoints that exist -### 1. Health Check -```bash -curl -X GET https://datasink.dev/api/health \ - -H "Accept: application/json" -``` +### GET /api/health -### 2. Machine Discovery Catalog ```bash -curl -X GET https://datasink.dev/.well-known/api-catalog \ - -H "Accept: application/linkset+json" +curl -s https://datasink.dev/api/health +# {"status":"ok","version":"0.4.0","service":"datasink"} ``` -### 3. Agent Markdown Page Request -```bash -curl -X GET https://datasink.dev/ \ - -H "Accept: text/markdown" -``` +### POST /api/firecrawl-proxy + +Body JSON: `{ "url": "https://example.com", "firecrawlKey": "fc-..." }` +Returns scraped markdown. The key is used once and not stored. + +There is **no** hosted `/api/mcp`, OAuth server, or agent registration API. --- -## CLI Command Reference +## CLI quick reference ```bash -# Scrub contacts file +npx datasink demo # sample data, no file needed npx datasink scrub contacts.csv +npx datasink rinse contacts.csv +npx datasink wash contacts.csv # full pipeline +npx datasink spot name@outlet.co.uk +npx datasink inspect contacts.csv +``` -# Run full pipeline with custom threshold -npx datasink scrub contacts.csv --rinse --fuzzy-threshold 0.95 +Global flags include `-o/--output`, `--format`, `--dry-run`, `--provider`, `--verbose`, `--json`. -# Spot-check individual email address -npx datasink spot jane@musicblog.com -``` +--- + +## Browser / WebMCP + +On datasink.dev, if the browser exposes `navigator.modelContext`, the page registers a `scrub_contacts` helper. That is in-page WebMCP only — not a remote MCP HTTP server. --- -## Standards Compliance -- **RFC 8288**: Web Linking via `Link` response headers. -- **RFC 9727**: API Catalog specification (`/.well-known/api-catalog`). -- **RFC 9728**: OAuth Protected Resource Metadata (`/.well-known/oauth-protected-resource`). -- **RFC 9116**: Security vulnerability reporting (`/.well-known/security.txt`). -- **SEP-1649**: Model Context Protocol Server Card (`/.well-known/mcp/server-card.json`). -- **Agent Skills Discovery RFC v0.2.0**: Index at `/.well-known/agent-skills/index.json`. -- **Content Signals**: robots.txt directives (`ai-train=no, search=yes, ai-input=no`). +## Discovery files + +- `/.well-known/api-catalog` — RFC 9727 linkset for the real API surface +- `/.well-known/ai-plugin.json` — OpenAI plugin-style manifest (auth: none) +- `/.well-known/agent-skills/` — skill markdown for agents +- `/.well-known/security.txt` — vulnerability contact +- `/llms.txt`, `/llms-full.txt` — this documentation +- `/auth.md` — BYOK notes (no OAuth) + +--- + +## Licence + +MIT diff --git a/web/public/llms.txt b/web/public/llms.txt index 65ea21b..fa113e3 100644 --- a/web/public/llms.txt +++ b/web/public/llms.txt @@ -1,18 +1,26 @@ # datasink — Data Hygiene for Music PR -> `sink` is a CLI and browser data hygiene suite for music PR contact lists published on npm as `datasink`. It scrubs syntax errors, deduplicates contacts across fields, enriches metadata, and segments list relevance. +> `sink` is a CLI and browser data hygiene tool for music PR contact lists, published on npm as `datasink`. Scrub syntax and typos, rinse duplicates, soak with AI enrichment, steep outlet channels from public sites. -## Core Capabilities -- [Scrub](https://datasink.dev/#scrub): Format validation, MX record lookup, typo mapping (e.g., `gmal.com` -> `gmail.com`), disposable domain detection, and role account tagging. -- [Rinse](https://datasink.dev/#rinse): Multi-field deduplication by exact email, fuzzy contact name, and publication domain. -- [Soak](https://datasink.dev/#soak): LLM-driven contact enrichment for missing fields, social profiles, and publication details. -- [Steep](https://datasink.dev/#steep): Campaign relevance scoring and AI outlet scraping for targeted PR lists. +## What actually works today -## Machine Discovery & Endpoints -- [API Catalog](https://datasink.dev/.well-known/api-catalog): RFC 9727 linkset documentation. -- [Extended LLM Spec](https://datasink.dev/llms-full.txt): Complete technical schema details and examples. -- [ChatGPT Plugin Manifest](https://datasink.dev/.well-known/ai-plugin.json): OpenAI Custom Action manifest. -- [MCP Server Card](https://datasink.dev/.well-known/mcp/server-card.json): SEP-1649 MCP Server Card. -- [Agent Skills Index](https://datasink.dev/.well-known/agent-skills/index.json): Skills discovery index (v0.2.0). -- [Agent Auth](https://datasink.dev/auth.md): Auth.md registration instructions. -- [Security Policy](https://datasink.dev/.well-known/security.txt): Security disclosure policy (RFC 9116). +- **CLI** (`npx datasink` / `npm i -g datasink`): scrub, rinse, soak, steep, wash, spot, inspect, demo, drain, tui. +- **Browser demo** ([datasink.dev](https://datasink.dev)): drop a CSV or XLSX; scrub + rinse run locally. Soak/steep need your own API keys. +- **Library**: `import { runPipeline, loadConfig } from 'datasink'` and browser-safe `datasink/core`. + +## Phases + +- [Scrub](https://datasink.dev/): format validation, typo map, disposable domains, role accounts, MX checks. +- [Rinse](https://datasink.dev/): exact-email, fuzzy-name, cross-field dedup. +- [Soak](https://datasink.dev/): LLM enrichment (Anthropic or OpenAI) — BYOK. +- [Steep](https://datasink.dev/): outlet site scrape via Firecrawl + grounded extraction — BYOK. + +## Machine-readable docs + +- [Extended LLM Spec](https://datasink.dev/llms-full.txt) +- [OpenAPI (real endpoints only)](https://datasink.dev/docs/openapi.json) +- [API Catalog](https://datasink.dev/.well-known/api-catalog) +- [Agent Skills Index](https://datasink.dev/.well-known/agent-skills/index.json) +- [Auth notes](https://datasink.dev/auth.md) +- [Security Policy](https://datasink.dev/.well-known/security.txt) +- Source: https://github.com/chrisschouk/sink-cli diff --git a/web/src/webmcp.ts b/web/src/webmcp.ts index b4fe9bd..4d5211e 100644 --- a/web/src/webmcp.ts +++ b/web/src/webmcp.ts @@ -76,9 +76,9 @@ export function initWebMcp(): void { return { catalogUrl: 'https://datasink.dev/.well-known/api-catalog', openApiUrl: 'https://datasink.dev/docs/openapi.json', - mcpServerCard: 'https://datasink.dev/.well-known/mcp/server-card.json', agentSkillsIndex: 'https://datasink.dev/.well-known/agent-skills/index.json', authMd: 'https://datasink.dev/auth.md', + llmsTxt: 'https://datasink.dev/llms.txt', } }, }) diff --git a/web/vercel.json b/web/vercel.json index 75ac76d..d2a2dc3 100644 --- a/web/vercel.json +++ b/web/vercel.json @@ -9,7 +9,7 @@ "headers": [ { "key": "Link", - "value": "; rel=\"api-catalog\", ; rel=\"service-doc\", ; rel=\"mcp-server-card\", ; rel=\"oauth-protected-resource\", ; rel=\"agent-skills\", ; rel=\"ai-plugin\", ; rel=\"help\", ; rel=\"llms-full\", ; rel=\"security-policy\"" + "value": "; rel=\"api-catalog\", ; rel=\"service-doc\", ; rel=\"agent-skills\", ; rel=\"ai-plugin\", ; rel=\"help\", ; rel=\"llms-full\", ; rel=\"security-policy\"" } ] }, @@ -22,42 +22,6 @@ } ] }, - { - "source": "/.well-known/openid-configuration", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ] - }, - { - "source": "/.well-known/oauth-authorization-server", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ] - }, - { - "source": "/.well-known/oauth-protected-resource", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ] - }, - { - "source": "/.well-known/mcp/server-card.json", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ] - }, { "source": "/.well-known/agent-skills/index.json", "headers": [ From 69d79fb7ce4367e0f269ae639a216a17a45a970c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 08:04:20 +0000 Subject: [PATCH 2/3] fix: root Vercel config + README accuracy for share readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git deploys were building the monorepo root and looking for public/, so preview/production stayed broken. Add root vercel.json → web/dist, promote api/ + middleware to root, and align README provider docs with the CLI. Co-authored-by: Chris Schofield --- CHANGELOG.md | 8 ++++ CLAUDE.md | 10 ++++- CONTRIBUTING.md | 4 ++ README.md | 12 +++--- api/firecrawl-proxy.ts | 71 +++++++++++++++++++++++++++++++++ api/health.ts | 23 +++++++++++ middleware.ts | 49 +++++++++++++++++++++++ vercel.json | 89 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 259 insertions(+), 7 deletions(-) create mode 100644 api/firecrawl-proxy.ts create mode 100644 api/health.ts create mode 100644 middleware.ts create mode 100644 vercel.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 48c8782..ddddfe7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Vercel git deploys were building the repo root and looking for `public/`, + ignoring `web/vercel.json`. Added a root `vercel.json` that builds the web + app to `web/dist` and serves edge functions from root `api/`. - CI install failed under pnpm 11 supply-chain checks because the SheetJS CDN `xlsx` tarball had no lockfile integrity hash. Integrity is now recorded. +- `pnpm-workspace.yaml` left `core-js` as an unresolved `allowBuilds` placeholder + (`set this to true or false`), which also fails pnpm 11 installs. Explicitly + denied. - Agent discovery docs claimed OAuth, MCP HTTP, and registration endpoints that do not exist. Discovery files now match the real surface: CLI, browser demo, `/api/health`, and `/api/firecrawl-proxy`. - README pointed at the Vercel preview hostname and an outdated org path; primary demo link is datasink.dev and GitHub links use `chrisschouk/sink-cli`. +- README provider flags and tagline brought in line with the CLI (`haiku|sonnet|…`, + four-phase strapline). ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 291b956..1539076 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,8 +45,14 @@ Check the token type before any publish attempt. ## Web demo -`web/` is a Vite + React app. Deploy: `cd web && vercel build --prod && vercel deploy --prebuilt --prod` -(auto-deploy requires Root Directory = `web` in the Vercel `sink-web` project settings — see NEXT_SESSION.md). +`web/` is a Vite + React app. Deployed as the Vercel `sink-web` project (domain datasink.dev). + +Git-linked deploys use the **repo-root** [`vercel.json`](vercel.json): build datasink + +`sink-web`, publish `web/dist`, serve edge functions from root [`api/`](api/). Keep +`web/api/` and `web/middleware.ts` in sync with the root copies (same handlers) so a +Root Directory = `web` setup still works. + +Manual prebuilt path: `cd web && vercel build --prod && vercel deploy --prebuilt --prod`. ## House standards diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e72c7c1..2268eeb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,14 +69,18 @@ src/ scrub/ Email validation, parsing, typo correction rinse/ Deduplication strategies soak/ AI enrichment providers + steep/ Outlet channel discovery (Firecrawl + LLM) output/ CSV/JSON/JSONL formatters ui/ Terminal UI (format helpers, TUI, interactive) utils/ MX cache, helpers +web/ Browser demo (Vite + React) → datasink.dev +api/ Vercel edge functions (health, Firecrawl proxy) test/ fixtures/ Sample CSV files scrub/ Scrub phase tests rinse/ Rinse phase tests soak/ Provider tests + steep/ Steep phase tests pipeline.test.ts Integration tests ``` diff --git a/README.md b/README.md index 9a38495..91359ad 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Node](https://img.shields.io/badge/node-%3E%3D20-brightgreen.svg)](https://nodejs.org) -**Data hygiene for music PR.** Scrub, rinse, and soak your contact lists. +**Data hygiene for music PR.** Scrub, rinse, soak, and steep your contact lists. > The product is **sink**; the binary is `sink`. It's published on npm as > [`datasink`](https://www.npmjs.com/package/datasink) because the `sink` name @@ -127,7 +127,8 @@ One scrape powers every contact at that outlet. The CLI caches scrapes in memory for the duration of a run; a persistent 30-day cache is available to programmatic consumers that supply their own `CacheAdapter` (see below). -Requires `FIRECRAWL_API_KEY` and an LLM provider key. Phase is silently skipped if creds are missing. +Requires `FIRECRAWL_API_KEY` and an LLM provider key. Skipped with a warning if +creds are missing. ## Global Flags @@ -140,7 +141,8 @@ Requires `FIRECRAWL_API_KEY` and an LLM provider key. Phase is silently skipped -q, --quiet Suppress all output except errors --json JSON stdout (for piping) --no-colour Disable colours ---provider Enrichment provider (anthropic|openai) +--provider Enrichment model shortcut (haiku|sonnet|opus|codex|gpt-4o-mini) + Also accepts anthropic|openai (default model for that vendor) ``` ## Exit Codes @@ -159,14 +161,14 @@ Requires `FIRECRAWL_API_KEY` and an LLM provider key. Phase is silently skipped ```bash export ANTHROPIC_API_KEY=sk-ant-... -sink soak contacts.csv --provider anthropic +sink soak contacts.csv --provider haiku # or sonnet / opus ``` ### OpenAI ```bash export OPENAI_API_KEY=sk-... -sink soak contacts.csv --provider openai +sink soak contacts.csv --provider gpt-4o-mini # or codex ``` ## Input Format diff --git a/api/firecrawl-proxy.ts b/api/firecrawl-proxy.ts new file mode 100644 index 0000000..bcddae6 --- /dev/null +++ b/api/firecrawl-proxy.ts @@ -0,0 +1,71 @@ +/** + * Thin Firecrawl proxy — the ONLY server-side code in sink web. + * + * Firecrawl's REST API blocks browser CORS, so a single outlet scrape is + * forwarded here. The user's Firecrawl key arrives in the request body, is used + * once for the upstream call, and is never logged or stored. Everything else in + * sink web runs in the browser. + * + * Source is intentionally tiny and open — see github.com/chrisschouk/sink-cli. + */ + +export const config = { runtime: 'edge' } + +const FIRECRAWL_URL = 'https://api.firecrawl.dev/v1/scrape' +const TIMEOUT_MS = 15_000 + +function json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }, + }) +} + +export default async function handler(req: Request): Promise { + if (req.method !== 'POST') return json({ error: 'Method not allowed' }, 405) + + let payload: { url?: string; firecrawlKey?: string } + try { + payload = (await req.json()) as { url?: string; firecrawlKey?: string } + } catch { + return json({ error: 'Invalid JSON body' }, 400) + } + + const { url, firecrawlKey } = payload + if (!url || !/^https:\/\//i.test(url)) return json({ error: 'A https url is required' }, 400) + if (!firecrawlKey || !firecrawlKey.startsWith('fc-')) { + return json({ error: 'A Firecrawl key (fc-…) is required' }, 400) + } + + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), TIMEOUT_MS) + try { + const upstream = await fetch(FIRECRAWL_URL, { + method: 'POST', + headers: { + Authorization: `Bearer ${firecrawlKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ url, formats: ['markdown'], onlyMainContent: true }), + signal: controller.signal, + }) + + if (!upstream.ok) { + // Surface auth failures so the UI can prompt for a valid key; treat other + // upstream errors as an empty (skippable) scrape. + if (upstream.status === 401 || upstream.status === 403) { + return json({ error: 'Firecrawl rejected the key' }, 401) + } + return json({ markdown: '' }, 200) + } + + const data = (await upstream.json()) as { + data?: { markdown?: string; content?: string } + } + return json({ markdown: data?.data?.markdown ?? data?.data?.content ?? '' }) + } catch { + return json({ markdown: '' }, 200) + } finally { + clearTimeout(timer) + } +} diff --git a/api/health.ts b/api/health.ts new file mode 100644 index 0000000..f309c57 --- /dev/null +++ b/api/health.ts @@ -0,0 +1,23 @@ +/** + * Tiny health endpoint for uptime checks and the OpenAPI catalog. + * No secrets, no side effects. + */ + +export const config = { runtime: 'edge' } + +export default function handler(): Response { + return new Response( + JSON.stringify({ + status: 'ok', + version: '0.4.0', + service: 'datasink', + }), + { + status: 200, + headers: { + 'Content-Type': 'application/json', + 'Cache-Control': 'public, max-age=60', + }, + }, + ) +} diff --git a/middleware.ts b/middleware.ts new file mode 100644 index 0000000..91f7d3a --- /dev/null +++ b/middleware.ts @@ -0,0 +1,49 @@ +export const config = { + matcher: ['/', '/index.html'], +} + +export default function middleware(request: Request): Response | void { + const accept = request.headers.get('accept') || '' + if (accept.includes('text/markdown')) { + const markdownContent = `# datasink.dev — Data Hygiene for Music PR + +sink scrubs, rinses, soaks, and steeps your contact lists. + +## Overview + +datasink is a data hygiene CLI and browser demo for music PR contact lists. +Scrub and rinse run locally in the browser. Soak and steep use bring-your-own-key AI providers. + +## Core capabilities + +- **Scrub**: format validation, typo mapping, disposable domains, role accounts, MX checks +- **Rinse**: multi-field deduplication +- **Soak**: LLM contact enrichment (your API key) +- **Steep**: outlet channel discovery via Firecrawl (your API key) + +## Getting started + +\`\`\`bash +npx datasink demo +npx datasink scrub contacts.csv +\`\`\` + +## Links + +- Web demo: https://datasink.dev +- npm: https://www.npmjs.com/package/datasink +- Source: https://github.com/chrisschouk/sink-cli +- LLM docs: https://datasink.dev/llms.txt +- Auth notes: https://datasink.dev/auth.md +- Security: https://datasink.dev/.well-known/security.txt +` + + return new Response(markdownContent, { + status: 200, + headers: { + 'content-type': 'text/markdown; charset=utf-8', + 'cache-control': 'public, max-age=3600', + }, + }) + } +} diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000..c6c88a2 --- /dev/null +++ b/vercel.json @@ -0,0 +1,89 @@ +{ + "buildCommand": "pnpm build && pnpm --filter sink-web build", + "outputDirectory": "web/dist", + "installCommand": "pnpm install --frozen-lockfile", + "framework": null, + "headers": [ + { + "source": "/(.*)", + "headers": [ + { + "key": "Link", + "value": "; rel=\"api-catalog\", ; rel=\"service-doc\", ; rel=\"agent-skills\", ; rel=\"ai-plugin\", ; rel=\"help\", ; rel=\"llms-full\", ; rel=\"security-policy\"" + } + ] + }, + { + "source": "/.well-known/api-catalog", + "headers": [ + { + "key": "Content-Type", + "value": "application/linkset+json" + } + ] + }, + { + "source": "/.well-known/agent-skills/index.json", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + } + ] + }, + { + "source": "/.well-known/ai-plugin.json", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + } + ] + }, + { + "source": "/auth.md", + "headers": [ + { + "key": "Content-Type", + "value": "text/markdown; charset=utf-8" + } + ] + }, + { + "source": "/llms.txt", + "headers": [ + { + "key": "Content-Type", + "value": "text/plain; charset=utf-8" + } + ] + }, + { + "source": "/llms-full.txt", + "headers": [ + { + "key": "Content-Type", + "value": "text/plain; charset=utf-8" + } + ] + }, + { + "source": "/.well-known/security.txt", + "headers": [ + { + "key": "Content-Type", + "value": "text/plain; charset=utf-8" + } + ] + }, + { + "source": "/docs/openapi.json", + "headers": [ + { + "key": "Content-Type", + "value": "application/openapi+json" + } + ] + } + ] +} From 0710dac043e222ecbf56f58994ee4ce2235c5644 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 08:57:49 +0000 Subject: [PATCH 3/3] feat: model-agnostic Anthropic/OpenAI selection Add --model for any vendor model ID, apply the same LLM choice to soak and steep, share alias resolution, and open the web BYOK panel to either vendor with a free-form model field. Co-authored-by: Chris Schofield --- CHANGELOG.md | 14 ++- README.md | 18 ++- pnpm-lock.yaml | 159 +++++++++++++------------ sink.config.example.ts | 17 +-- src/cli.ts | 113 +++++++++++------- src/index.ts | 6 + src/phases/soak/models.ts | 84 +++++++++++++ src/phases/soak/providers/anthropic.ts | 12 +- src/phases/soak/providers/openai.ts | 11 +- src/phases/steep/index.ts | 5 +- src/ui/interactive.ts | 52 ++++++++ test/soak/models.test.ts | 76 ++++++++++++ web/package.json | 5 +- web/src/components/ApiKeyPanel.tsx | 122 +++++++++++++++---- web/src/hooks/useEngine.ts | 14 ++- web/src/soak-browser.ts | 81 +++++++++---- web/src/steep-browser.ts | 59 ++++++--- web/src/styles/results.css | 23 ++++ web/src/types.ts | 5 +- 19 files changed, 648 insertions(+), 228 deletions(-) create mode 100644 src/phases/soak/models.ts create mode 100644 test/soak/models.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ddddfe7..5db4e21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Model-agnostic LLM selection.** `--model ` accepts any Anthropic or + OpenAI model string; `--provider` takes vendors (`anthropic|openai`) or + shortcuts. The same choice applies to soak and steep. Web BYOK panel offers + either vendor plus a free-form model field. +- `GET /api/health` for uptime checks and the OpenAPI catalog. +- `sink demo` documented in the README command table. + ### Fixed - Vercel git deploys were building the repo root and looking for `public/`, @@ -25,11 +34,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - README provider flags and tagline brought in line with the CLI (`haiku|sonnet|…`, four-phase strapline). -### Added - -- `GET /api/health` for uptime checks and the OpenAPI catalog. -- `sink demo` documented in the README command table. - ## [0.4.0] - 2026-06-11 ### Added diff --git a/README.md b/README.md index 91359ad..8c5d92d 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,9 @@ Enriches contacts with AI: - Submission guidelines - Pitch tips -Supports **Anthropic** (Claude Haiku) and **OpenAI** (GPT-4o-mini). +Supports **Anthropic** and **OpenAI** with any model ID those vendors accept. +CLI shortcuts (`haiku`, `sonnet`, `opus`, `gpt-4o-mini`, `codex`) are convenience +defaults only — use `--provider anthropic|openai --model ` for anything else. ### Steep @@ -141,8 +143,8 @@ creds are missing. -q, --quiet Suppress all output except errors --json JSON stdout (for piping) --no-colour Disable colours ---provider Enrichment model shortcut (haiku|sonnet|opus|codex|gpt-4o-mini) - Also accepts anthropic|openai (default model for that vendor) +--provider LLM vendor or shortcut (anthropic|openai|haiku|sonnet|opus|codex|gpt-4o-mini) +--model Any Anthropic/OpenAI model ID (overrides shortcut default) ``` ## Exit Codes @@ -157,18 +159,24 @@ creds are missing. ## Provider Setup +Sink is model-agnostic across Anthropic and OpenAI. Shortcuts pick a convenient +default; `--model` accepts any current model ID from that vendor. The same +choice applies to both soak and steep. + ### Anthropic ```bash export ANTHROPIC_API_KEY=sk-ant-... -sink soak contacts.csv --provider haiku # or sonnet / opus +sink soak contacts.csv --provider haiku +sink soak contacts.csv --provider anthropic --model claude-sonnet-4-5-20250514 ``` ### OpenAI ```bash export OPENAI_API_KEY=sk-... -sink soak contacts.csv --provider gpt-4o-mini # or codex +sink soak contacts.csv --provider gpt-4o-mini +sink soak contacts.csv --provider openai --model gpt-4.1-mini ``` ## Input Format diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b5b409b..293ae9b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -46,7 +46,7 @@ importers: devDependencies: '@eslint/js': specifier: ^10.0.1 - version: 10.0.1(eslint@10.0.3) + version: 10.0.1(eslint@10.0.3(supports-color@7.2.0)) '@types/node': specifier: ^22.0.0 version: 22.19.15 @@ -55,10 +55,10 @@ importers: version: 19.2.10 '@vitest/coverage-v8': specifier: ^3.2.6 - version: 3.2.6(vitest@3.2.6(@types/node@22.19.15)) + version: 3.2.6(supports-color@7.2.0)(vitest@3.2.6(@types/node@22.19.15)(supports-color@7.2.0)) eslint: specifier: ^10.0.3 - version: 10.0.3 + version: 10.0.3(supports-color@7.2.0) prettier: specifier: ^3.8.1 version: 3.8.1 @@ -67,10 +67,10 @@ importers: version: 5.9.3 typescript-eslint: specifier: ^8.56.1 - version: 8.56.1(eslint@10.0.3)(typescript@5.9.3) + version: 8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3) vitest: specifier: ^3.2.6 - version: 3.2.6(@types/node@22.19.15) + version: 3.2.6(@types/node@22.19.15)(supports-color@7.2.0) web: dependencies: @@ -80,6 +80,9 @@ importers: datasink: specifier: workspace:* version: link:.. + openai: + specifier: ^4.104.0 + version: 4.104.0(ws@8.21.0) posthog-js: specifier: ^1.407.2 version: 1.407.2 @@ -101,7 +104,7 @@ importers: version: 19.2.3(@types/react@19.2.10) '@vitejs/plugin-react': specifier: ^5.0.0 - version: 5.2.0(vite@7.3.1(@types/node@22.19.15)) + version: 5.2.0(supports-color@7.2.0)(vite@7.3.1(@types/node@22.19.15)) typescript: specifier: ^5.9.3 version: 5.9.3 @@ -1851,20 +1854,20 @@ snapshots: '@babel/compat-data@7.29.7': {} - '@babel/core@7.29.7': + '@babel/core@7.29.7(supports-color@7.2.0)': dependencies: '@babel/code-frame': 7.29.7 '@babel/generator': 7.29.7 '@babel/helper-compilation-targets': 7.29.7 - '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7) + '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helpers': 7.29.7 '@babel/parser': 7.29.7 '@babel/template': 7.29.7 - '@babel/traverse': 7.29.7 + '@babel/traverse': 7.29.7(supports-color@7.2.0) '@babel/types': 7.29.7 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -1889,19 +1892,19 @@ snapshots: '@babel/helper-globals@7.29.7': {} - '@babel/helper-module-imports@7.29.7': + '@babel/helper-module-imports@7.29.7(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.7 + '@babel/traverse': 7.29.7(supports-color@7.2.0) '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.29.7(@babel/core@7.29.7)': + '@babel/helper-module-transforms@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.7 - '@babel/helper-module-imports': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) + '@babel/helper-module-imports': 7.29.7(supports-color@7.2.0) '@babel/helper-validator-identifier': 7.29.7 - '@babel/traverse': 7.29.7 + '@babel/traverse': 7.29.7(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -1922,14 +1925,14 @@ snapshots: dependencies: '@babel/types': 7.29.7 - '@babel/plugin-transform-react-jsx-self@7.29.7(@babel/core@7.29.7)': + '@babel/plugin-transform-react-jsx-self@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': dependencies: - '@babel/core': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 - '@babel/plugin-transform-react-jsx-source@7.29.7(@babel/core@7.29.7)': + '@babel/plugin-transform-react-jsx-source@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': dependencies: - '@babel/core': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 '@babel/template@7.29.7': @@ -1938,7 +1941,7 @@ snapshots: '@babel/parser': 7.29.7 '@babel/types': 7.29.7 - '@babel/traverse@7.29.7': + '@babel/traverse@7.29.7(supports-color@7.2.0)': dependencies: '@babel/code-frame': 7.29.7 '@babel/generator': 7.29.7 @@ -1946,7 +1949,7 @@ snapshots: '@babel/parser': 7.29.7 '@babel/template': 7.29.7 '@babel/types': 7.29.7 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -2044,17 +2047,17 @@ snapshots: '@esbuild/win32-x64@0.27.3': optional: true - '@eslint-community/eslint-utils@4.9.1(eslint@10.0.3)': + '@eslint-community/eslint-utils@4.9.1(eslint@10.0.3(supports-color@7.2.0))': dependencies: - eslint: 10.0.3 + eslint: 10.0.3(supports-color@7.2.0) eslint-visitor-keys: 3.4.3 '@eslint-community/regexpp@4.12.2': {} - '@eslint/config-array@0.23.3': + '@eslint/config-array@0.23.3(supports-color@7.2.0)': dependencies: '@eslint/object-schema': 3.0.3 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) minimatch: 10.2.4 transitivePeerDependencies: - supports-color @@ -2067,9 +2070,9 @@ snapshots: dependencies: '@types/json-schema': 7.0.15 - '@eslint/js@10.0.1(eslint@10.0.3)': + '@eslint/js@10.0.1(eslint@10.0.3(supports-color@7.2.0))': optionalDependencies: - eslint: 10.0.3 + eslint: 10.0.3(supports-color@7.2.0) '@eslint/object-schema@3.0.3': {} @@ -2268,15 +2271,15 @@ snapshots: '@types/trusted-types@2.0.7': optional: true - '@typescript-eslint/eslint-plugin@8.56.1(@typescript-eslint/parser@8.56.1(eslint@10.0.3)(typescript@5.9.3))(eslint@10.0.3)(typescript@5.9.3)': + '@typescript-eslint/eslint-plugin@8.56.1(@typescript-eslint/parser@8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3))(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.56.1(eslint@10.0.3)(typescript@5.9.3) + '@typescript-eslint/parser': 8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3) '@typescript-eslint/scope-manager': 8.56.1 - '@typescript-eslint/type-utils': 8.56.1(eslint@10.0.3)(typescript@5.9.3) - '@typescript-eslint/utils': 8.56.1(eslint@10.0.3)(typescript@5.9.3) + '@typescript-eslint/type-utils': 8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3) + '@typescript-eslint/utils': 8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3) '@typescript-eslint/visitor-keys': 8.56.1 - eslint: 10.0.3 + eslint: 10.0.3(supports-color@7.2.0) ignore: 7.0.5 natural-compare: 1.4.0 ts-api-utils: 2.4.0(typescript@5.9.3) @@ -2284,23 +2287,23 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.56.1(eslint@10.0.3)(typescript@5.9.3)': + '@typescript-eslint/parser@8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3)': dependencies: '@typescript-eslint/scope-manager': 8.56.1 '@typescript-eslint/types': 8.56.1 - '@typescript-eslint/typescript-estree': 8.56.1(typescript@5.9.3) + '@typescript-eslint/typescript-estree': 8.56.1(supports-color@7.2.0)(typescript@5.9.3) '@typescript-eslint/visitor-keys': 8.56.1 - debug: 4.4.3 - eslint: 10.0.3 + debug: 4.4.3(supports-color@7.2.0) + eslint: 10.0.3(supports-color@7.2.0) typescript: 5.9.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/project-service@8.56.1(typescript@5.9.3)': + '@typescript-eslint/project-service@8.56.1(supports-color@7.2.0)(typescript@5.9.3)': dependencies: '@typescript-eslint/tsconfig-utils': 8.56.1(typescript@5.9.3) '@typescript-eslint/types': 8.56.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -2314,13 +2317,13 @@ snapshots: dependencies: typescript: 5.9.3 - '@typescript-eslint/type-utils@8.56.1(eslint@10.0.3)(typescript@5.9.3)': + '@typescript-eslint/type-utils@8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3)': dependencies: '@typescript-eslint/types': 8.56.1 - '@typescript-eslint/typescript-estree': 8.56.1(typescript@5.9.3) - '@typescript-eslint/utils': 8.56.1(eslint@10.0.3)(typescript@5.9.3) - debug: 4.4.3 - eslint: 10.0.3 + '@typescript-eslint/typescript-estree': 8.56.1(supports-color@7.2.0)(typescript@5.9.3) + '@typescript-eslint/utils': 8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3) + debug: 4.4.3(supports-color@7.2.0) + eslint: 10.0.3(supports-color@7.2.0) ts-api-utils: 2.4.0(typescript@5.9.3) typescript: 5.9.3 transitivePeerDependencies: @@ -2328,13 +2331,13 @@ snapshots: '@typescript-eslint/types@8.56.1': {} - '@typescript-eslint/typescript-estree@8.56.1(typescript@5.9.3)': + '@typescript-eslint/typescript-estree@8.56.1(supports-color@7.2.0)(typescript@5.9.3)': dependencies: - '@typescript-eslint/project-service': 8.56.1(typescript@5.9.3) + '@typescript-eslint/project-service': 8.56.1(supports-color@7.2.0)(typescript@5.9.3) '@typescript-eslint/tsconfig-utils': 8.56.1(typescript@5.9.3) '@typescript-eslint/types': 8.56.1 '@typescript-eslint/visitor-keys': 8.56.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) minimatch: 10.2.4 semver: 7.7.4 tinyglobby: 0.2.15 @@ -2343,13 +2346,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.56.1(eslint@10.0.3)(typescript@5.9.3)': + '@typescript-eslint/utils@8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3)': dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@10.0.3) + '@eslint-community/eslint-utils': 4.9.1(eslint@10.0.3(supports-color@7.2.0)) '@typescript-eslint/scope-manager': 8.56.1 '@typescript-eslint/types': 8.56.1 - '@typescript-eslint/typescript-estree': 8.56.1(typescript@5.9.3) - eslint: 10.0.3 + '@typescript-eslint/typescript-estree': 8.56.1(supports-color@7.2.0)(typescript@5.9.3) + eslint: 10.0.3(supports-color@7.2.0) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -2359,11 +2362,11 @@ snapshots: '@typescript-eslint/types': 8.56.1 eslint-visitor-keys: 5.0.1 - '@vitejs/plugin-react@5.2.0(vite@7.3.1(@types/node@22.19.15))': + '@vitejs/plugin-react@5.2.0(supports-color@7.2.0)(vite@7.3.1(@types/node@22.19.15))': dependencies: - '@babel/core': 7.29.7 - '@babel/plugin-transform-react-jsx-self': 7.29.7(@babel/core@7.29.7) - '@babel/plugin-transform-react-jsx-source': 7.29.7(@babel/core@7.29.7) + '@babel/core': 7.29.7(supports-color@7.2.0) + '@babel/plugin-transform-react-jsx-self': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) + '@babel/plugin-transform-react-jsx-source': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) '@rolldown/pluginutils': 1.0.0-rc.3 '@types/babel__core': 7.20.5 react-refresh: 0.18.0 @@ -2371,22 +2374,22 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitest/coverage-v8@3.2.6(vitest@3.2.6(@types/node@22.19.15))': + '@vitest/coverage-v8@3.2.6(supports-color@7.2.0)(vitest@3.2.6(@types/node@22.19.15)(supports-color@7.2.0))': dependencies: '@ampproject/remapping': 2.3.0 '@bcoe/v8-coverage': 1.0.2 ast-v8-to-istanbul: 0.3.12 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) istanbul-lib-coverage: 3.2.2 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 5.0.6 + istanbul-lib-source-maps: 5.0.6(supports-color@7.2.0) istanbul-reports: 3.2.0 magic-string: 0.30.21 magicast: 0.3.5 std-env: 3.10.0 test-exclude: 7.0.2 tinyrainbow: 2.0.0 - vitest: 3.2.6(@types/node@22.19.15) + vitest: 3.2.6(@types/node@22.19.15)(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -2569,9 +2572,11 @@ snapshots: csstype@3.2.3: {} - debug@4.4.3: + debug@4.4.3(supports-color@7.2.0): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 7.2.0 deep-eql@5.0.2: {} @@ -2666,11 +2671,11 @@ snapshots: eslint-visitor-keys@5.0.1: {} - eslint@10.0.3: + eslint@10.0.3(supports-color@7.2.0): dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@10.0.3) + '@eslint-community/eslint-utils': 4.9.1(eslint@10.0.3(supports-color@7.2.0)) '@eslint-community/regexpp': 4.12.2 - '@eslint/config-array': 0.23.3 + '@eslint/config-array': 0.23.3(supports-color@7.2.0) '@eslint/config-helpers': 0.5.3 '@eslint/core': 1.1.1 '@eslint/plugin-kit': 0.6.1 @@ -2680,7 +2685,7 @@ snapshots: '@types/estree': 1.0.8 ajv: 6.14.0 cross-spawn: 7.0.6 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) escape-string-regexp: 4.0.0 eslint-scope: 9.1.2 eslint-visitor-keys: 5.0.1 @@ -2913,10 +2918,10 @@ snapshots: make-dir: 4.0.0 supports-color: 7.2.0 - istanbul-lib-source-maps@5.0.6: + istanbul-lib-source-maps@5.0.6(supports-color@7.2.0): dependencies: '@jridgewell/trace-mapping': 0.3.31 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) istanbul-lib-coverage: 3.2.2 transitivePeerDependencies: - supports-color @@ -3298,13 +3303,13 @@ snapshots: dependencies: tagged-tag: 1.0.0 - typescript-eslint@8.56.1(eslint@10.0.3)(typescript@5.9.3): + typescript-eslint@8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3): dependencies: - '@typescript-eslint/eslint-plugin': 8.56.1(@typescript-eslint/parser@8.56.1(eslint@10.0.3)(typescript@5.9.3))(eslint@10.0.3)(typescript@5.9.3) - '@typescript-eslint/parser': 8.56.1(eslint@10.0.3)(typescript@5.9.3) - '@typescript-eslint/typescript-estree': 8.56.1(typescript@5.9.3) - '@typescript-eslint/utils': 8.56.1(eslint@10.0.3)(typescript@5.9.3) - eslint: 10.0.3 + '@typescript-eslint/eslint-plugin': 8.56.1(@typescript-eslint/parser@8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3))(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3) + '@typescript-eslint/parser': 8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3) + '@typescript-eslint/typescript-estree': 8.56.1(supports-color@7.2.0)(typescript@5.9.3) + '@typescript-eslint/utils': 8.56.1(eslint@10.0.3(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3) + eslint: 10.0.3(supports-color@7.2.0) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -3325,10 +3330,10 @@ snapshots: dependencies: punycode: 2.3.1 - vite-node@3.2.4(@types/node@22.19.15): + vite-node@3.2.4(@types/node@22.19.15)(supports-color@7.2.0): dependencies: cac: 6.7.14 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) es-module-lexer: 1.7.0 pathe: 2.0.3 vite: 7.3.1(@types/node@22.19.15) @@ -3358,7 +3363,7 @@ snapshots: '@types/node': 22.19.15 fsevents: 2.3.3 - vitest@3.2.6(@types/node@22.19.15): + vitest@3.2.6(@types/node@22.19.15)(supports-color@7.2.0): dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.6 @@ -3369,7 +3374,7 @@ snapshots: '@vitest/spy': 3.2.6 '@vitest/utils': 3.2.6 chai: 5.3.3 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) expect-type: 1.3.0 magic-string: 0.30.21 pathe: 2.0.3 @@ -3381,7 +3386,7 @@ snapshots: tinypool: 1.1.1 tinyrainbow: 2.0.0 vite: 7.3.1(@types/node@22.19.15) - vite-node: 3.2.4(@types/node@22.19.15) + vite-node: 3.2.4(@types/node@22.19.15)(supports-color@7.2.0) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 22.19.15 diff --git a/sink.config.example.ts b/sink.config.example.ts index 0e12fca..efee75b 100644 --- a/sink.config.example.ts +++ b/sink.config.example.ts @@ -24,26 +24,29 @@ const config: Partial = { }, soak: { - // Which AI provider to use for enrichment - // Can also use shortcuts via CLI: --provider haiku|sonnet|opus|codex|gpt-4o-mini + // Vendor: 'anthropic' | 'openai'. CLI shortcuts: --provider haiku|sonnet|opus|codex|gpt-4o-mini + // Any model ID: --provider anthropic --model claude-… / --provider openai --model gpt-… provider: 'anthropic', - // Anthropic config - // Models: 'claude-haiku-4-5-20251001' (fast/cheap), 'claude-sonnet-4-5-20250514' (balanced), 'claude-opus-4-0-20250514' (most capable) - // Or use aliases: 'haiku', 'sonnet', 'opus' + // Anthropic — model string is free-form (aliases: haiku, sonnet, opus) anthropic: { model: 'claude-haiku-4-5-20251001', apiKey: process.env.ANTHROPIC_API_KEY, }, - // OpenAI config - // Models: 'gpt-4o-mini', 'codex-mini-latest' (or alias: 'codex') + // OpenAI — model string is free-form (aliases: gpt-4o-mini, codex) // openai: { // model: 'gpt-4o-mini', // apiKey: process.env.OPENAI_API_KEY, // }, }, + steep: { + // Extractor vendor mirrors soak unless overridden. Same free-form model IDs. + // extractor: 'anthropic', + // anthropic: { model: 'claude-haiku-4-5-20251001' }, + }, + output: { format: 'csv', locale: 'en-GB', diff --git a/src/cli.ts b/src/cli.ts index 041cc2d..2293d0f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -26,6 +26,7 @@ import { LOGO_LINES, } from './ui/format.js' import { VERSION } from './version.js' +import { resolveLlmChoice } from './phases/soak/models.js' import type { SinkRecord, SinkConfig, Phase } from './types.js' // Load .env if present (works on Node 20+, no dependencies) @@ -57,23 +58,38 @@ const EXIT = { } as const // --------------------------------------------------------------------------- -// Provider shortcuts — map friendly names to provider + model +// LLM vendor + model (model-agnostic: any Anthropic/OpenAI ID via --model) // --------------------------------------------------------------------------- -const PROVIDER_SHORTCUTS: Record = { - haiku: { provider: 'anthropic', model: 'claude-haiku-4-5-20251001' }, - sonnet: { provider: 'anthropic', model: 'claude-sonnet-4-5-20250514' }, - opus: { provider: 'anthropic', model: 'claude-opus-4-0-20250514' }, - codex: { provider: 'openai', model: 'codex-mini-latest' }, - 'gpt-4o-mini': { provider: 'openai', model: 'gpt-4o-mini' }, +function applyLlmOverrides( + config: SinkConfig, + choice: { provider?: string; model?: string }, +): void { + const { provider, model } = choice + if (!provider) return + + config.soak.provider = provider + config.steep.extractor = provider + + if (!model) return + + const soakBlock = (config.soak[provider] as Record | undefined) ?? {} + config.soak[provider] = { ...soakBlock, model } + + const steepBlock = (config.steep[provider] as Record | undefined) ?? {} + config.steep[provider] = { ...steepBlock, model } } -function resolveProvider(name?: string): { provider?: string; model?: string } { - if (!name) return {} - const shortcut = PROVIDER_SHORTCUTS[name] - if (shortcut) return shortcut - // Pass through as-is (e.g. "anthropic", "openai") - return { provider: name } +function resolveCliLlm(opts: { provider?: string; model?: string }): { + provider?: string + model?: string +} { + const result = resolveLlmChoice({ provider: opts.provider, model: opts.model }) + if (result.error) { + console.error(`\n Error: ${result.error}\n`) + process.exit(EXIT.CONFIG_ERROR) + } + return { provider: result.provider, model: result.model } } // --------------------------------------------------------------------------- @@ -113,6 +129,7 @@ async function runPhases( noColour?: boolean smtp?: boolean provider?: string + model?: string demo?: boolean url?: string skipIntro?: boolean @@ -131,17 +148,19 @@ async function runPhases( ) } - const { provider: providerName, model: providerModel } = resolveProvider(opts.provider) + const llm = resolveCliLlm({ provider: opts.provider, model: opts.model }) const config = await loadConfig({ configPath: opts.config, overrides: { scrub: { smtp: opts.smtp }, soak: { - ...(providerName ? { provider: providerName } : undefined), - ...(providerName && providerModel - ? { [providerName]: { model: providerModel } } - : undefined), + ...(llm.provider ? { provider: llm.provider } : undefined), + ...(llm.provider && llm.model ? { [llm.provider]: { model: llm.model } } : undefined), + }, + steep: { + ...(llm.provider ? { extractor: llm.provider } : undefined), + ...(llm.provider && llm.model ? { [llm.provider]: { model: llm.model } } : undefined), }, output: { format: (opts.format as SinkConfig['output']['format']) ?? 'csv', @@ -293,7 +312,14 @@ async function runInspect( async function runTui( rawPath: string | undefined, - opts: { smtp?: boolean; provider?: string; config?: string; demo?: boolean; url?: string }, + opts: { + smtp?: boolean + provider?: string + model?: string + config?: string + demo?: boolean + url?: string + }, ): Promise { // TUI currently requires a file path for its React component // For demo/url/stdin, write to a temp file @@ -312,16 +338,7 @@ async function runTui( const config = await loadConfig({ configPath: opts.config }) if (opts.smtp) config.scrub.smtp = true - const { provider: providerName, model: providerModel } = resolveProvider(opts.provider) - if (providerName) config.soak.provider = providerName - if (providerName && providerModel) { - ;(config.soak as Record)[providerName] = { - ...((config.soak as Record)[providerName] as - | Record - | undefined), - model: providerModel, - } - } + applyLlmOverrides(config, resolveCliLlm({ provider: opts.provider, model: opts.model })) const { render } = await import('ink') const { App } = await import('./ui/tui/app.js') @@ -394,11 +411,14 @@ Examples: ${chalk.dim('$')} sink scrub --url https://... Fetch & validate from URL ${chalk.dim('$')} pbpaste | sink scrub - Pipe from clipboard ${chalk.dim('$')} sink wash contacts.csv --provider sonnet Enrich with Claude Sonnet + ${chalk.dim('$')} sink soak contacts.csv --provider openai --model gpt-4.1-mini ${chalk.dim('$')} sink spot sarah@bbc.co.uk Check a single email ${chalk.dim('$')} sink inspect contacts.csv Data quality score Providers: - haiku, sonnet, opus (Anthropic) | gpt-4o-mini, codex (OpenAI) + Vendors: anthropic | openai + Shortcuts: haiku, sonnet, opus | gpt-4o-mini, codex + Any model ID: --provider anthropic|openai --model `, ) @@ -413,7 +433,14 @@ const globalOpts = (cmd: typeof program) => .option('--json', 'JSON stdout (for piping)') .option('--no-colour', 'disable colours') .option('--smtp', '(deprecated, no-op) SMTP verification removed in 0.3.0') - .option('--provider ', 'enrichment provider (haiku|sonnet|opus|codex|gpt-4o-mini)') + .option( + '--provider ', + 'LLM vendor or shortcut (anthropic|openai|haiku|sonnet|opus|codex|gpt-4o-mini)', + ) + .option( + '--model ', + 'any Anthropic/OpenAI model ID (overrides shortcut default; use with --provider)', + ) .option('--demo', 'use built-in sample data (no file needed)') .option('--url ', 'fetch CSV from a URL') @@ -453,7 +480,11 @@ globalOpts( program .command('demo') .description('Run the full pipeline on sample data (no file needed)') - .option('--provider ', 'enrichment provider (haiku|sonnet|opus|codex|gpt-4o-mini)') + .option( + '--provider ', + 'LLM vendor or shortcut (anthropic|openai|haiku|sonnet|opus|codex|gpt-4o-mini)', + ) + .option('--model ', 'any Anthropic/OpenAI model ID') .option('--smtp', '(deprecated, no-op) SMTP verification removed in 0.3.0') .option('--verbose', 'detailed output') .option('--no-colour', 'disable colours') @@ -510,6 +541,7 @@ program.action(async () => { const interactiveOpts = { smtp: result.options?.smtp as boolean, provider: result.options?.provider as string, + model: result.options?.model as string | undefined, demo: result.options?.demo as boolean, verbose: result.options?.verbose as boolean, } @@ -519,18 +551,13 @@ program.action(async () => { const records = parseInputText(result.text) const config = await loadConfig() - const { provider: providerName, model: providerModel } = resolveProvider( - interactiveOpts.provider, + applyLlmOverrides( + config, + resolveCliLlm({ + provider: interactiveOpts.provider, + model: interactiveOpts.model, + }), ) - if (providerName) config.soak.provider = providerName - if (providerName && providerModel) { - ;(config.soak as Record)[providerName] = { - ...((config.soak as Record)[providerName] as - | Record - | undefined), - model: providerModel, - } - } const phases: Phase[] = [] switch (result.command) { diff --git a/src/index.ts b/src/index.ts index da102ee..4b02366 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,12 @@ export { scrub } from './phases/scrub/index.js' export { rinse } from './phases/rinse/index.js' export { soak } from './phases/soak/index.js' export { steep } from './phases/steep/index.js' +export { + resolveLlmChoice, + expandModelId, + MODEL_SHORTCUTS, + KNOWN_PROVIDERS, +} from './phases/soak/models.js' // Steep cache adapter (consumers can implement against the CacheAdapter interface) export { InMemoryCache } from './phases/steep/cache/in-memory.js' diff --git a/src/phases/soak/models.ts b/src/phases/soak/models.ts new file mode 100644 index 0000000..f6ec43c --- /dev/null +++ b/src/phases/soak/models.ts @@ -0,0 +1,84 @@ +/** + * Shared LLM vendor/model resolution for soak + steep. + * + * Shortcuts (haiku, sonnet, …) are convenience aliases only. Any Anthropic or + * OpenAI model ID is accepted via --model / config — providers pass the string + * straight through to the vendor SDK. + */ + +export const KNOWN_PROVIDERS = ['anthropic', 'openai'] as const +export type KnownProvider = (typeof KNOWN_PROVIDERS)[number] + +/** Friendly shortcut → vendor + concrete model ID. */ +export const MODEL_SHORTCUTS: Record = { + haiku: { provider: 'anthropic', model: 'claude-haiku-4-5-20251001' }, + sonnet: { provider: 'anthropic', model: 'claude-sonnet-4-5-20250514' }, + opus: { provider: 'anthropic', model: 'claude-opus-4-0-20250514' }, + codex: { provider: 'openai', model: 'codex-mini-latest' }, + 'gpt-4o-mini': { provider: 'openai', model: 'gpt-4o-mini' }, +} + +/** Expand a shortcut alias to a concrete model ID; pass through unknown strings. */ +export function expandModelId(model: string): string { + return MODEL_SHORTCUTS[model]?.model ?? model +} + +export interface LlmChoice { + provider?: KnownProvider + model?: string +} + +export interface LlmChoiceResult extends LlmChoice { + error?: string +} + +/** + * Resolve CLI/config LLM selection. + * + * - `--provider haiku|sonnet|opus|codex|gpt-4o-mini` → vendor + default model + * - `--provider anthropic|openai` → vendor only (provider default model) + * - `--model ` → any model string (aliases expanded); needs a vendor unless + * the model itself is a known shortcut + * - Combining `--provider anthropic --model claude-…` (or any ID) works + */ +export function resolveLlmChoice(opts: { provider?: string; model?: string }): LlmChoiceResult { + const rawProvider = opts.provider?.trim() + const rawModel = opts.model?.trim() + + if (rawProvider && MODEL_SHORTCUTS[rawProvider]) { + const shortcut = MODEL_SHORTCUTS[rawProvider] + return { + provider: shortcut.provider, + model: rawModel ? expandModelId(rawModel) : shortcut.model, + } + } + + if (rawProvider && (KNOWN_PROVIDERS as readonly string[]).includes(rawProvider)) { + return { + provider: rawProvider as KnownProvider, + model: rawModel ? expandModelId(rawModel) : undefined, + } + } + + if (!rawProvider && rawModel) { + const shortcut = MODEL_SHORTCUTS[rawModel] + if (shortcut) { + return { provider: shortcut.provider, model: shortcut.model } + } + return { + error: `Pass --provider anthropic|openai with --model ${rawModel} (cannot infer vendor from a free-form model ID).`, + } + } + + if (rawProvider) { + return { + error: `Unknown provider '${rawProvider}'. Use anthropic|openai, or a shortcut (haiku|sonnet|opus|codex|gpt-4o-mini).`, + } + } + + return {} +} + +export function defaultModelFor(provider: KnownProvider): string { + return provider === 'openai' ? 'gpt-4o-mini' : 'claude-haiku-4-5-20251001' +} diff --git a/src/phases/soak/providers/anthropic.ts b/src/phases/soak/providers/anthropic.ts index 191f616..49da615 100644 --- a/src/phases/soak/providers/anthropic.ts +++ b/src/phases/soak/providers/anthropic.ts @@ -1,18 +1,13 @@ import type { SoakProvider, SoakResult, SinkRecord } from '../../../types.js' import { SoakConfigError } from '../provider.js' import { buildPrompt, calculateConfidence } from '../prompt.js' - -const MODEL_ALIASES: Record = { - haiku: 'claude-haiku-4-5-20251001', - sonnet: 'claude-sonnet-4-5-20250514', - opus: 'claude-opus-4-0-20250514', -} +import { defaultModelFor, expandModelId } from '../models.js' export class AnthropicProvider implements SoakProvider { name = 'anthropic' // eslint-disable-next-line @typescript-eslint/no-explicit-any -- lazy-loaded SDK private client: any = null - private model = 'claude-haiku-4-5-20251001' + private model = defaultModelFor('anthropic') async init(config: Record): Promise { const apiKey = (config.apiKey as string) || process.env.ANTHROPIC_API_KEY @@ -21,8 +16,7 @@ export class AnthropicProvider implements SoakProvider { const { default: Anthropic } = await import('@anthropic-ai/sdk') this.client = new Anthropic({ apiKey }) if (config.model) { - const alias = MODEL_ALIASES[config.model as string] - this.model = alias ?? (config.model as string) + this.model = expandModelId(config.model as string) } } diff --git a/src/phases/soak/providers/openai.ts b/src/phases/soak/providers/openai.ts index c2c1a17..5171164 100644 --- a/src/phases/soak/providers/openai.ts +++ b/src/phases/soak/providers/openai.ts @@ -1,17 +1,13 @@ import type { SoakProvider, SoakResult, SinkRecord } from '../../../types.js' import { SoakConfigError } from '../provider.js' import { buildPrompt, calculateConfidence } from '../prompt.js' - -const MODEL_ALIASES: Record = { - codex: 'codex-mini-latest', - 'gpt-4o-mini': 'gpt-4o-mini', -} +import { defaultModelFor, expandModelId } from '../models.js' export class OpenAIProvider implements SoakProvider { name = 'openai' // eslint-disable-next-line @typescript-eslint/no-explicit-any -- lazy-loaded SDK private client: any = null - private model = 'gpt-4o-mini' + private model = defaultModelFor('openai') async init(config: Record): Promise { const apiKey = (config.apiKey as string) || process.env.OPENAI_API_KEY @@ -20,8 +16,7 @@ export class OpenAIProvider implements SoakProvider { const { default: OpenAI } = await import('openai') this.client = new OpenAI({ apiKey }) if (config.model) { - const alias = MODEL_ALIASES[config.model as string] - this.model = alias ?? (config.model as string) + this.model = expandModelId(config.model as string) } } diff --git a/src/phases/steep/index.ts b/src/phases/steep/index.ts index b970b14..32f4a55 100644 --- a/src/phases/steep/index.ts +++ b/src/phases/steep/index.ts @@ -9,6 +9,7 @@ import { getScraper } from './registry.js' import { getProvider as getSoakProvider } from '../soak/registry.js' import { buildSteepPrompt, calculateOutletConfidence } from './prompt.js' import { outletToDomain, SteepConfigError } from './provider.js' +import { SoakConfigError } from '../soak/provider.js' import { InMemoryCache } from './cache/in-memory.js' const DEFAULT_CACHE_TTL = 30 * 24 * 60 * 60 * 1000 // 30 days @@ -130,8 +131,8 @@ export async function steep( await scraper.init(scraperConfig) await extractor.init(extractorConfig) } catch (err) { - if (err instanceof SteepConfigError) { - // Skip the phase silently if creds are missing -- mirrors soak's behaviour + if (err instanceof SteepConfigError || err instanceof SoakConfigError) { + // Skip the phase if creds are missing -- mirrors soak's behaviour return records } throw err diff --git a/src/ui/interactive.ts b/src/ui/interactive.ts index 4399873..1b624f7 100644 --- a/src/ui/interactive.ts +++ b/src/ui/interactive.ts @@ -249,6 +249,11 @@ export async function runInteractive(): Promise<{ label: 'OpenAI Codex', hint: hasOpenAIKey ? undefined : chalk.red('OPENAI_API_KEY not set'), }, + { + value: 'custom', + label: 'Custom model ID', + hint: 'any Anthropic or OpenAI model string', + }, { value: 'skip', label: 'Skip enrichment' }, ] @@ -260,6 +265,53 @@ export async function runInteractive(): Promise<{ if (!p.isCancel(provider)) { if (provider === 'skip') { opts.provider = undefined + } else if (provider === 'custom') { + const vendor = await p.select({ + message: 'Vendor', + options: [ + { + value: 'anthropic', + label: 'Anthropic', + hint: hasAnthropicKey ? undefined : chalk.red('ANTHROPIC_API_KEY not set'), + }, + { + value: 'openai', + label: 'OpenAI', + hint: hasOpenAIKey ? undefined : chalk.red('OPENAI_API_KEY not set'), + }, + ], + }) + if (p.isCancel(vendor)) { + opts.provider = undefined + } else { + opts.provider = vendor + const modelId = await p.text({ + message: 'Model ID', + placeholder: vendor === 'anthropic' ? 'claude-sonnet-4-5-20250514' : 'gpt-4.1-mini', + validate: (value) => { + if (!value?.trim()) return 'A model ID is required.' + }, + }) + if (!p.isCancel(modelId) && modelId?.trim()) { + opts.model = modelId.trim() + } + + const needsAnthropic = vendor === 'anthropic' && !hasAnthropicKey + const needsOpenAI = vendor === 'openai' && !hasOpenAIKey + if (needsAnthropic || needsOpenAI) { + const envVar = needsAnthropic ? 'ANTHROPIC_API_KEY' : 'OPENAI_API_KEY' + const apiKey = await p.text({ + message: `${envVar} not set. Enter it now (for this session only):`, + placeholder: 'sk-...', + validate: (value) => { + if (!value?.trim()) return `${envVar} is required.` + }, + }) + if (!p.isCancel(apiKey) && apiKey) { + process.env[envVar] = apiKey.trim() + } + } + } } else { opts.provider = provider diff --git a/test/soak/models.test.ts b/test/soak/models.test.ts new file mode 100644 index 0000000..cd2ce24 --- /dev/null +++ b/test/soak/models.test.ts @@ -0,0 +1,76 @@ +import { describe, it, expect } from 'vitest' +import { + expandModelId, + resolveLlmChoice, + defaultModelFor, + MODEL_SHORTCUTS, +} from '../../src/phases/soak/models.js' + +describe('LLM model resolution', () => { + it('expands shortcuts to concrete model IDs', () => { + expect(expandModelId('haiku')).toBe(MODEL_SHORTCUTS.haiku.model) + expect(expandModelId('sonnet')).toBe(MODEL_SHORTCUTS.sonnet.model) + expect(expandModelId('gpt-4o-mini')).toBe('gpt-4o-mini') + }) + + it('passes through arbitrary model IDs unchanged', () => { + expect(expandModelId('claude-opus-4-20250514')).toBe('claude-opus-4-20250514') + expect(expandModelId('gpt-4.1-mini')).toBe('gpt-4.1-mini') + expect(expandModelId('o4-mini')).toBe('o4-mini') + }) + + it('resolves provider shortcuts', () => { + expect(resolveLlmChoice({ provider: 'sonnet' })).toEqual({ + provider: 'anthropic', + model: MODEL_SHORTCUTS.sonnet.model, + }) + expect(resolveLlmChoice({ provider: 'codex' })).toEqual({ + provider: 'openai', + model: MODEL_SHORTCUTS.codex.model, + }) + }) + + it('allows vendor + free-form model', () => { + expect(resolveLlmChoice({ provider: 'anthropic', model: 'claude-sonnet-4-20250514' })).toEqual({ + provider: 'anthropic', + model: 'claude-sonnet-4-20250514', + }) + expect(resolveLlmChoice({ provider: 'openai', model: 'gpt-4.1' })).toEqual({ + provider: 'openai', + model: 'gpt-4.1', + }) + }) + + it('allows vendor alone (provider default model)', () => { + expect(resolveLlmChoice({ provider: 'anthropic' })).toEqual({ + provider: 'anthropic', + model: undefined, + }) + expect(defaultModelFor('anthropic')).toBe('claude-haiku-4-5-20251001') + expect(defaultModelFor('openai')).toBe('gpt-4o-mini') + }) + + it('infers vendor from model-only shortcut', () => { + expect(resolveLlmChoice({ model: 'haiku' })).toEqual({ + provider: 'anthropic', + model: MODEL_SHORTCUTS.haiku.model, + }) + }) + + it('errors when free-form model has no vendor', () => { + const result = resolveLlmChoice({ model: 'gpt-4.1-mini' }) + expect(result.error).toMatch(/--provider/) + }) + + it('errors on unknown provider names', () => { + const result = resolveLlmChoice({ provider: 'gemini' }) + expect(result.error).toMatch(/Unknown provider/) + }) + + it('lets --model override a shortcut default', () => { + expect(resolveLlmChoice({ provider: 'haiku', model: 'claude-opus-4-0-20250514' })).toEqual({ + provider: 'anthropic', + model: 'claude-opus-4-0-20250514', + }) + }) +}) diff --git a/web/package.json b/web/package.json index ed8c848..2c957b7 100644 --- a/web/package.json +++ b/web/package.json @@ -11,10 +11,11 @@ "dependencies": { "@anthropic-ai/sdk": "^0.39.0", "datasink": "workspace:*", + "openai": "^4.104.0", + "posthog-js": "^1.407.2", "react": "^19.2.0", "react-dom": "^19.2.0", - "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", - "posthog-js": "^1.407.2" + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz" }, "devDependencies": { "@types/react": "^19.2.0", diff --git a/web/src/components/ApiKeyPanel.tsx b/web/src/components/ApiKeyPanel.tsx index be43190..6d137e6 100644 --- a/web/src/components/ApiKeyPanel.tsx +++ b/web/src/components/ApiKeyPanel.tsx @@ -1,9 +1,16 @@ import { useState } from 'react' import type { ApiKeys } from '../types' -const STORE_ANTHROPIC = 'sink:anthropic' +const STORE_PROVIDER = 'sink:provider' +const STORE_API_KEY = 'sink:apiKey' +const STORE_MODEL = 'sink:model' const STORE_FIRECRAWL = 'sink:firecrawl' +const DEFAULT_MODELS = { + anthropic: 'claude-haiku-4-5-20251001', + openai: 'gpt-4o-mini', +} as const + function readStored(key: string): string { try { return sessionStorage.getItem(key) ?? '' @@ -13,8 +20,8 @@ function readStored(key: string): string { } /** - * Terminal-styled key entry, shown after rinse. Anthropic key unlocks soak - * (AI enrichment); adding a Firecrawl key also unlocks steep (outlet research). + * Terminal-styled key entry, shown after rinse. Bring your own Anthropic or + * OpenAI key (any model ID) for soak; add Firecrawl to also unlock steep. * Keys live in memory only unless the user opts into sessionStorage. */ export function ApiKeyPanel({ @@ -24,42 +31,65 @@ export function ApiKeyPanel({ onRun: (keys: ApiKeys) => void onSkip: () => void }) { - const storedAnthropic = readStored(STORE_ANTHROPIC) - const storedFirecrawl = readStored(STORE_FIRECRAWL) - const [anthropic, setAnthropic] = useState(storedAnthropic) - const [firecrawl, setFirecrawl] = useState(storedFirecrawl) - const [remember, setRemember] = useState(Boolean(storedAnthropic)) + const storedProvider = (readStored(STORE_PROVIDER) as 'anthropic' | 'openai') || 'anthropic' + const [provider, setProvider] = useState<'anthropic' | 'openai'>( + storedProvider === 'openai' ? 'openai' : 'anthropic', + ) + const [apiKey, setApiKey] = useState(readStored(STORE_API_KEY)) + const [model, setModel] = useState(readStored(STORE_MODEL) || DEFAULT_MODELS[storedProvider === 'openai' ? 'openai' : 'anthropic']) + const [firecrawl, setFirecrawl] = useState(readStored(STORE_FIRECRAWL)) + const [remember, setRemember] = useState(Boolean(readStored(STORE_API_KEY))) const [err, setErr] = useState(null) + const switchProvider = (next: 'anthropic' | 'openai') => { + setProvider(next) + // Reset model to vendor default when switching unless user already typed a custom one + // that still looks like it belongs — keep it simple: always swap default. + setModel(DEFAULT_MODELS[next]) + } + const submit = () => { - const a = anthropic.trim() + const key = apiKey.trim() + const modelId = model.trim() const f = firecrawl.trim() - if (!a) { - setErr('An Anthropic key is required to run AI enrichment.') + if (!key) { + setErr('An API key is required to run AI enrichment.') return } - if (!a.startsWith('sk-ant-')) { + if (provider === 'anthropic' && !key.startsWith('sk-ant-')) { setErr('That doesn’t look like an Anthropic key (expected sk-ant-…).') return } + if (provider === 'openai' && !key.startsWith('sk-')) { + setErr('That doesn’t look like an OpenAI key (expected sk-…).') + return + } + if (!modelId) { + setErr('A model ID is required (any current Anthropic or OpenAI model string).') + return + } if (f && !f.startsWith('fc-')) { setErr('That doesn’t look like a Firecrawl key (expected fc-…).') return } try { if (remember) { - sessionStorage.setItem(STORE_ANTHROPIC, a) + sessionStorage.setItem(STORE_PROVIDER, provider) + sessionStorage.setItem(STORE_API_KEY, key) + sessionStorage.setItem(STORE_MODEL, modelId) if (f) sessionStorage.setItem(STORE_FIRECRAWL, f) else sessionStorage.removeItem(STORE_FIRECRAWL) } else { - sessionStorage.removeItem(STORE_ANTHROPIC) + sessionStorage.removeItem(STORE_PROVIDER) + sessionStorage.removeItem(STORE_API_KEY) + sessionStorage.removeItem(STORE_MODEL) sessionStorage.removeItem(STORE_FIRECRAWL) } } catch { // sessionStorage may be unavailable (private mode) — keys still work in-memory. } setErr(null) - onRun({ anthropic: a, firecrawl: f || undefined }) + onRun({ provider, apiKey: key, model: modelId, firecrawl: f || undefined }) } return ( @@ -70,20 +100,61 @@ export function ApiKeyPanel({

Scrub & rinse are done — locally. Soak enriches each contact (genres, platform, pitch tips); steep researches each - outlet (submission portals, socials, recent coverage). Bring your own keys. + outlet (submission portals, socials, recent coverage). Bring your own keys — any Anthropic + or OpenAI model.

+
+ Vendor + + +
+ + +