From ac7ceeee0666623f80fb4e7f23adebc766d5149c Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 22 Sep 2026 14:08:11 +0000 Subject: [PATCH 1/2] Stop the directory serving broken logos and ignoring ?limit= Three fixes to the directory, all found while verifying the rasterly swap. A page can declare to suppress the favicon request. That parses as a valid URL and z.string().url() accepts it, so it was stored as logo_url and rendered as a broken image; example.com submits this way. Favicon and og:image hrefs now go through toHttpUrl(), which resolves them against the page and keeps them only if they end up http(s). The /favicon.ico last resort is HEAD-checked instead of asserted, so a site without one yields an empty logo rather than a link that 404s. GET /api/directory hard-coded a page size of 20 and silently dropped ?limit=. Every other list route in the app reads limit through parsePaginationParam, so this one now does too: default 20 (unchanged), max 50, and per_page in the response reports what was actually applied. scripts/backfill-screenshots.ts captures the listings that predate screenshots, through rasterly, reusing the storage path shape the route writes. It honours --dry-run and --limit, prints the quota left after each render and stops when the API says the budget is gone. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/backfill-screenshots.ts | 143 ++++++++++++++++++++++ src/app/api/directory/fetch-meta/route.ts | 39 +++--- src/app/api/directory/route.test.ts | 35 ++++++ src/app/api/directory/route.ts | 9 +- src/lib/meta-url.test.ts | 47 +++++++ src/lib/meta-url.ts | 19 +++ 6 files changed, 277 insertions(+), 15 deletions(-) create mode 100644 scripts/backfill-screenshots.ts create mode 100644 src/lib/meta-url.test.ts create mode 100644 src/lib/meta-url.ts diff --git a/scripts/backfill-screenshots.ts b/scripts/backfill-screenshots.ts new file mode 100644 index 00000000..21df10d1 --- /dev/null +++ b/scripts/backfill-screenshots.ts @@ -0,0 +1,143 @@ +#!/usr/bin/env tsx +/** + * Backfill homepage screenshots for directory listings that have none. + * + * Usage: + * tsx scripts/backfill-screenshots.ts # capture everything missing + * tsx scripts/backfill-screenshots.ts --dry-run # list what would be captured + * tsx scripts/backfill-screenshots.ts --limit 5 # cap how many are rendered + * + * Requires: NEXT_PUBLIC_SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, RASTERLY_API_KEY in .env + * + * rasterly's free tier is 100 renders/month, so the script reports the quota + * remaining after each capture and stops if the API says the budget is gone. + */ + +import { createClient } from "@supabase/supabase-js"; +import crypto from "crypto"; +import { config } from "dotenv"; + +config(); // load .env + +const SUPABASE_URL = process.env.NEXT_PUBLIC_SUPABASE_URL!; +const SUPABASE_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY!; +const RASTERLY_KEY = process.env.RASTERLY_API_KEY!; + +if (!SUPABASE_URL || !SUPABASE_KEY || !RASTERLY_KEY) { + console.error( + "Missing NEXT_PUBLIC_SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, or RASTERLY_API_KEY" + ); + process.exit(1); +} + +const supabase = createClient(SUPABASE_URL, SUPABASE_KEY); + +const args = process.argv.slice(2); +const dryRun = args.includes("--dry-run"); +const limitArg = args.indexOf("--limit"); +const limit = + limitArg !== -1 && args[limitArg + 1] ? Number(args[limitArg + 1]) : Infinity; + +type Listing = { id: string; title: string; url: string }; + +async function render(url: string): Promise<{ buf: Buffer; quota: string | null }> { + const endpoint = `https://api.rasterly.dev/v1/screenshot?url=${encodeURIComponent( + url + )}&format=png&width=1280&height=800`; + + const res = await fetch(endpoint, { + headers: { "X-Api-Key": RASTERLY_KEY }, + signal: AbortSignal.timeout(30000), + }); + + if (!res.ok) throw new Error(`rasterly HTTP ${res.status}`); + if (!(res.headers.get("content-type") || "").startsWith("image/")) { + throw new Error("rasterly returned a non-image response"); + } + + const buf = Buffer.from(await res.arrayBuffer()); + if (buf.length === 0) throw new Error("rasterly returned an empty image"); + + return { buf, quota: res.headers.get("x-quota-remaining") }; +} + +async function upload(url: string, buf: Buffer): Promise { + // Same path shape the fetch-meta route writes, so both sources interleave. + const urlHash = crypto.createHash("md5").update(url).digest("hex"); + const filePath = `${urlHash}/${Date.now()}.png`; + + const { error } = await supabase.storage + .from("directory-screenshots") + .upload(filePath, buf, { contentType: "image/png", upsert: true }); + + if (error) throw new Error(`upload failed: ${error.message}`); + + const { + data: { publicUrl }, + } = supabase.storage.from("directory-screenshots").getPublicUrl(filePath); + + return publicUrl; +} + +async function main() { + const { data, error } = await supabase + .from("project_listings") + .select("id, title, url") + .eq("status", "active") + .is("screenshot_url", null) + .order("created_at", { ascending: false }); + + if (error) { + console.error("Failed to read listings:", error.message); + process.exit(1); + } + + const listings = (data || []) as Listing[]; + console.log(`${listings.length} active listing(s) without a screenshot`); + + if (dryRun) { + for (const l of listings) console.log(` would capture: ${l.title} — ${l.url}`); + return; + } + + let done = 0; + let failed = 0; + + for (const listing of listings) { + if (done >= limit) { + console.log(`Reached --limit ${limit}, stopping.`); + break; + } + + try { + const { buf, quota } = await render(listing.url); + const publicUrl = await upload(listing.url, buf); + + const { error: updateError } = await supabase + .from("project_listings") + .update({ screenshot_url: publicUrl }) + .eq("id", listing.id); + + if (updateError) throw new Error(`update failed: ${updateError.message}`); + + done++; + console.log(`✓ ${listing.title} — ${buf.length} bytes, quota left: ${quota ?? "?"}`); + + if (quota !== null && Number(quota) <= 0) { + console.log("rasterly quota exhausted, stopping."); + break; + } + } catch (err) { + failed++; + const msg = err instanceof Error ? err.message : String(err); + console.error(`✗ ${listing.title} — ${msg}`); + } + } + + console.log(`\nCaptured ${done}, failed ${failed}.`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/src/app/api/directory/fetch-meta/route.ts b/src/app/api/directory/fetch-meta/route.ts index 20964a18..da85941d 100644 --- a/src/app/api/directory/fetch-meta/route.ts +++ b/src/app/api/directory/fetch-meta/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { createServiceClient } from "@/lib/supabase/service"; +import { toHttpUrl } from "@/lib/meta-url"; import OpenAI from "openai"; import crypto from "crypto"; @@ -101,26 +102,20 @@ export async function POST(request: NextRequest) { } if (!logo_url) { - const faviconHref = extractFavicon(html); - if (faviconHref) { - try { - logo_url = new URL(faviconHref, url).href; - } catch { - logo_url = faviconHref; - } - } + // A page may declare to suppress the + // favicon request. That parses as a valid URL, so it has to be rejected + // by scheme or it reaches the directory and renders as a broken image. + logo_url = toHttpUrl(extractFavicon(html), url); } if (!logo_url && ogImage) { - try { - logo_url = new URL(ogImage, url).href; - } catch { - logo_url = ogImage; - } + logo_url = toHttpUrl(ogImage, url); } if (!logo_url) { - logo_url = `${parsedUrl.origin}/favicon.ico`; + // Only fall back to /favicon.ico if the site actually serves one. + const icoFallback = `${parsedUrl.origin}/favicon.ico`; + if (await urlExists(icoFallback)) logo_url = icoFallback; } // --- Banner detection --- @@ -344,6 +339,22 @@ function extractTitle(html: string): string { return match ? decodeEntities(match[1].trim()) : ""; } +/** + * HEAD a URL to see whether it is actually served. + */ +async function urlExists(candidate: string): Promise { + try { + const res = await fetch(candidate, { + method: "HEAD", + signal: AbortSignal.timeout(3000), + redirect: "follow", + }); + return res.ok; + } catch { + return false; + } +} + function extractFavicon(html: string): string { const match = html.match( /]*rel=["'](?:icon|shortcut icon|apple-touch-icon)["'][^>]*href=["']([^"']*)["'][^>]*\/?>/i diff --git a/src/app/api/directory/route.test.ts b/src/app/api/directory/route.test.ts index efab593c..3eac00bb 100644 --- a/src/app/api/directory/route.test.ts +++ b/src/app/api/directory/route.test.ts @@ -63,4 +63,39 @@ describe("GET /api/directory", () => { expect(chain.range).toHaveBeenCalledWith(0, 19); expect(body.page).toBe(1); }); + + it("honours limit instead of always returning 20 rows", async () => { + const chain = chainResult({ data: [], error: null, count: 0 }); + mockFrom.mockReturnValue(chain); + + const res = await GET(makeRequest({ limit: "5" })); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(chain.range).toHaveBeenCalledWith(0, 4); + expect(body.per_page).toBe(5); + }); + + it("offsets by the requested limit when paging", async () => { + const chain = chainResult({ data: [], error: null, count: 0 }); + mockFrom.mockReturnValue(chain); + + await GET(makeRequest({ limit: "5", page: "3" })); + + expect(chain.range).toHaveBeenCalledWith(10, 14); + }); + + it("clamps limit to the allowed range and defaults when absent", async () => { + const chain = chainResult({ data: [], error: null, count: 0 }); + mockFrom.mockReturnValue(chain); + + await GET(makeRequest({ limit: "9999" })); + expect(chain.range).toHaveBeenCalledWith(0, 49); + + await GET(makeRequest({ limit: "0" })); + expect(chain.range).toHaveBeenCalledWith(0, 0); + + await GET(makeRequest()); + expect(chain.range).toHaveBeenCalledWith(0, 19); + }); }); diff --git a/src/app/api/directory/route.ts b/src/app/api/directory/route.ts index 56d738e2..c048fd91 100644 --- a/src/app/api/directory/route.ts +++ b/src/app/api/directory/route.ts @@ -20,6 +20,8 @@ import { parsePaginationParam } from "@/lib/api-pagination"; const LNBITS_INVOICE_KEY = process.env.LNBITS_INVOICE_KEY || ""; const MAX_DIRECTORY_PAGE = 10_000; +const DEFAULT_DIRECTORY_LIMIT = 20; +const MAX_DIRECTORY_LIMIT = 50; const createListingSchema = z.object({ title: z.string().min(1).max(100), @@ -45,7 +47,12 @@ export async function GET(request: NextRequest) { 1, MAX_DIRECTORY_PAGE ); - const limit = 20; + const limit = parsePaginationParam( + url.searchParams.get("limit"), + DEFAULT_DIRECTORY_LIMIT, + 1, + MAX_DIRECTORY_LIMIT + ); const offset = (page - 1) * limit; const supabase = await createClient(); diff --git a/src/lib/meta-url.test.ts b/src/lib/meta-url.test.ts new file mode 100644 index 00000000..3543a2ff --- /dev/null +++ b/src/lib/meta-url.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { toHttpUrl } from "./meta-url"; + +describe("toHttpUrl", () => { + it("rejects the data: favicon that suppresses the icon request", () => { + // — example.com ships this, and it used to + // reach the directory as logo_url and render as a broken image. + expect(toHttpUrl("data:,", "https://example.com")).toBe(""); + expect( + toHttpUrl("data:image/png;base64,iVBORw0KGgo=", "https://example.com") + ).toBe(""); + }); + + it("rejects other non-http schemes", () => { + expect(toHttpUrl("about:blank", "https://example.com")).toBe(""); + expect(toHttpUrl("javascript:void(0)", "https://example.com")).toBe(""); + }); + + it("returns nothing for empty or whitespace hrefs", () => { + expect(toHttpUrl("", "https://example.com")).toBe(""); + expect(toHttpUrl(" ", "https://example.com")).toBe(""); + expect(toHttpUrl(null, "https://example.com")).toBe(""); + expect(toHttpUrl(undefined, "https://example.com")).toBe(""); + }); + + it("resolves relative hrefs against the page URL", () => { + expect(toHttpUrl("/favicon.ico", "https://example.com/a/b")).toBe( + "https://example.com/favicon.ico" + ); + expect(toHttpUrl("icon.png", "https://example.com/a/b")).toBe( + "https://example.com/a/icon.png" + ); + }); + + it("keeps absolute http(s) URLs, including protocol-relative ones", () => { + expect(toHttpUrl("https://cdn.example.com/logo.svg", "https://example.com")).toBe( + "https://cdn.example.com/logo.svg" + ); + expect(toHttpUrl("//cdn.example.com/logo.svg", "https://example.com")).toBe( + "https://cdn.example.com/logo.svg" + ); + }); + + it("returns nothing when the href cannot be parsed", () => { + expect(toHttpUrl("http://[", "https://example.com")).toBe(""); + }); +}); diff --git a/src/lib/meta-url.ts b/src/lib/meta-url.ts new file mode 100644 index 00000000..c1ab17d4 --- /dev/null +++ b/src/lib/meta-url.ts @@ -0,0 +1,19 @@ +/** + * Resolve a candidate image href (favicon, og:image) against the page URL, + * keeping it only if it ends up as http(s). + * + * A page may declare `` to suppress the favicon + * request. That parses as a perfectly valid URL, so without a scheme check it + * reaches the directory as a logo and renders as a broken image. + */ +export function toHttpUrl(href: string | null | undefined, base: string): string { + if (!href || !href.trim()) return ""; + + try { + const resolved = new URL(href.trim(), base); + if (resolved.protocol !== "http:" && resolved.protocol !== "https:") return ""; + return resolved.href; + } catch { + return ""; + } +} From 64a16dd469ad660f5d2c7ebd890edd8572f18f69 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 22 Sep 2026 14:11:42 +0000 Subject: [PATCH 2/2] Await the deferred load in the UserReviews load-more test "shows loading state when loading more" queues a response that resolves 100ms later, asserts the Loading... label synchronously and then ends. The deferred promise resolves after the test is over, so loadMore's setReviews runs against a torn-down jsdom and vitest reports an unhandled "window is not defined" that fails the whole run. It only surfaced when adding a test file elsewhere shifted the scheduling, so the suite was one file away from going red at any time. Waiting for the label to clear keeps the state update inside the test's lifetime. Co-Authored-By: Claude Opus 5 (1M context) --- src/components/reviews/UserReviews.test.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/components/reviews/UserReviews.test.tsx b/src/components/reviews/UserReviews.test.tsx index 33b1d958..30f8f4b7 100644 --- a/src/components/reviews/UserReviews.test.tsx +++ b/src/components/reviews/UserReviews.test.tsx @@ -262,6 +262,13 @@ describe("UserReviews", () => { fireEvent.click(screen.getByText("Load more reviews")); expect(screen.getByText("Loading...")).toBeInTheDocument(); + + // The deferred response resolves 100ms from now. Let it land before the + // test ends: otherwise loadMore's setReviews runs after jsdom teardown and + // vitest reports an unhandled "window is not defined" for the whole run. + await waitFor(() => { + expect(screen.queryByText("Loading...")).not.toBeInTheDocument(); + }); }); it("appends new reviews to existing ones", async () => {