diff --git a/scripts/backfill-screenshots.ts b/scripts/backfill-screenshots.ts new file mode 100644 index 00000000..21df10d1 --- /dev/null +++ b/scripts/backfill-screenshots.ts @@ -0,0 +1,143 @@ +#!/usr/bin/env tsx +/** + * Backfill homepage screenshots for directory listings that have none. + * + * Usage: + * tsx scripts/backfill-screenshots.ts # capture everything missing + * tsx scripts/backfill-screenshots.ts --dry-run # list what would be captured + * tsx scripts/backfill-screenshots.ts --limit 5 # cap how many are rendered + * + * Requires: NEXT_PUBLIC_SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, RASTERLY_API_KEY in .env + * + * rasterly's free tier is 100 renders/month, so the script reports the quota + * remaining after each capture and stops if the API says the budget is gone. + */ + +import { createClient } from "@supabase/supabase-js"; +import crypto from "crypto"; +import { config } from "dotenv"; + +config(); // load .env + +const SUPABASE_URL = process.env.NEXT_PUBLIC_SUPABASE_URL!; +const SUPABASE_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY!; +const RASTERLY_KEY = process.env.RASTERLY_API_KEY!; + +if (!SUPABASE_URL || !SUPABASE_KEY || !RASTERLY_KEY) { + console.error( + "Missing NEXT_PUBLIC_SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, or RASTERLY_API_KEY" + ); + process.exit(1); +} + +const supabase = createClient(SUPABASE_URL, SUPABASE_KEY); + +const args = process.argv.slice(2); +const dryRun = args.includes("--dry-run"); +const limitArg = args.indexOf("--limit"); +const limit = + limitArg !== -1 && args[limitArg + 1] ? Number(args[limitArg + 1]) : Infinity; + +type Listing = { id: string; title: string; url: string }; + +async function render(url: string): Promise<{ buf: Buffer; quota: string | null }> { + const endpoint = `https://api.rasterly.dev/v1/screenshot?url=${encodeURIComponent( + url + )}&format=png&width=1280&height=800`; + + const res = await fetch(endpoint, { + headers: { "X-Api-Key": RASTERLY_KEY }, + signal: AbortSignal.timeout(30000), + }); + + if (!res.ok) throw new Error(`rasterly HTTP ${res.status}`); + if (!(res.headers.get("content-type") || "").startsWith("image/")) { + throw new Error("rasterly returned a non-image response"); + } + + const buf = Buffer.from(await res.arrayBuffer()); + if (buf.length === 0) throw new Error("rasterly returned an empty image"); + + return { buf, quota: res.headers.get("x-quota-remaining") }; +} + +async function upload(url: string, buf: Buffer): Promise { + // Same path shape the fetch-meta route writes, so both sources interleave. + const urlHash = crypto.createHash("md5").update(url).digest("hex"); + const filePath = `${urlHash}/${Date.now()}.png`; + + const { error } = await supabase.storage + .from("directory-screenshots") + .upload(filePath, buf, { contentType: "image/png", upsert: true }); + + if (error) throw new Error(`upload failed: ${error.message}`); + + const { + data: { publicUrl }, + } = supabase.storage.from("directory-screenshots").getPublicUrl(filePath); + + return publicUrl; +} + +async function main() { + const { data, error } = await supabase + .from("project_listings") + .select("id, title, url") + .eq("status", "active") + .is("screenshot_url", null) + .order("created_at", { ascending: false }); + + if (error) { + console.error("Failed to read listings:", error.message); + process.exit(1); + } + + const listings = (data || []) as Listing[]; + console.log(`${listings.length} active listing(s) without a screenshot`); + + if (dryRun) { + for (const l of listings) console.log(` would capture: ${l.title} — ${l.url}`); + return; + } + + let done = 0; + let failed = 0; + + for (const listing of listings) { + if (done >= limit) { + console.log(`Reached --limit ${limit}, stopping.`); + break; + } + + try { + const { buf, quota } = await render(listing.url); + const publicUrl = await upload(listing.url, buf); + + const { error: updateError } = await supabase + .from("project_listings") + .update({ screenshot_url: publicUrl }) + .eq("id", listing.id); + + if (updateError) throw new Error(`update failed: ${updateError.message}`); + + done++; + console.log(`✓ ${listing.title} — ${buf.length} bytes, quota left: ${quota ?? "?"}`); + + if (quota !== null && Number(quota) <= 0) { + console.log("rasterly quota exhausted, stopping."); + break; + } + } catch (err) { + failed++; + const msg = err instanceof Error ? err.message : String(err); + console.error(`✗ ${listing.title} — ${msg}`); + } + } + + console.log(`\nCaptured ${done}, failed ${failed}.`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/src/app/api/directory/fetch-meta/route.ts b/src/app/api/directory/fetch-meta/route.ts index 20964a18..da85941d 100644 --- a/src/app/api/directory/fetch-meta/route.ts +++ b/src/app/api/directory/fetch-meta/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { createServiceClient } from "@/lib/supabase/service"; +import { toHttpUrl } from "@/lib/meta-url"; import OpenAI from "openai"; import crypto from "crypto"; @@ -101,26 +102,20 @@ export async function POST(request: NextRequest) { } if (!logo_url) { - const faviconHref = extractFavicon(html); - if (faviconHref) { - try { - logo_url = new URL(faviconHref, url).href; - } catch { - logo_url = faviconHref; - } - } + // A page may declare to suppress the + // favicon request. That parses as a valid URL, so it has to be rejected + // by scheme or it reaches the directory and renders as a broken image. + logo_url = toHttpUrl(extractFavicon(html), url); } if (!logo_url && ogImage) { - try { - logo_url = new URL(ogImage, url).href; - } catch { - logo_url = ogImage; - } + logo_url = toHttpUrl(ogImage, url); } if (!logo_url) { - logo_url = `${parsedUrl.origin}/favicon.ico`; + // Only fall back to /favicon.ico if the site actually serves one. + const icoFallback = `${parsedUrl.origin}/favicon.ico`; + if (await urlExists(icoFallback)) logo_url = icoFallback; } // --- Banner detection --- @@ -344,6 +339,22 @@ function extractTitle(html: string): string { return match ? decodeEntities(match[1].trim()) : ""; } +/** + * HEAD a URL to see whether it is actually served. + */ +async function urlExists(candidate: string): Promise { + try { + const res = await fetch(candidate, { + method: "HEAD", + signal: AbortSignal.timeout(3000), + redirect: "follow", + }); + return res.ok; + } catch { + return false; + } +} + function extractFavicon(html: string): string { const match = html.match( /]*rel=["'](?:icon|shortcut icon|apple-touch-icon)["'][^>]*href=["']([^"']*)["'][^>]*\/?>/i diff --git a/src/app/api/directory/route.test.ts b/src/app/api/directory/route.test.ts index efab593c..3eac00bb 100644 --- a/src/app/api/directory/route.test.ts +++ b/src/app/api/directory/route.test.ts @@ -63,4 +63,39 @@ describe("GET /api/directory", () => { expect(chain.range).toHaveBeenCalledWith(0, 19); expect(body.page).toBe(1); }); + + it("honours limit instead of always returning 20 rows", async () => { + const chain = chainResult({ data: [], error: null, count: 0 }); + mockFrom.mockReturnValue(chain); + + const res = await GET(makeRequest({ limit: "5" })); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(chain.range).toHaveBeenCalledWith(0, 4); + expect(body.per_page).toBe(5); + }); + + it("offsets by the requested limit when paging", async () => { + const chain = chainResult({ data: [], error: null, count: 0 }); + mockFrom.mockReturnValue(chain); + + await GET(makeRequest({ limit: "5", page: "3" })); + + expect(chain.range).toHaveBeenCalledWith(10, 14); + }); + + it("clamps limit to the allowed range and defaults when absent", async () => { + const chain = chainResult({ data: [], error: null, count: 0 }); + mockFrom.mockReturnValue(chain); + + await GET(makeRequest({ limit: "9999" })); + expect(chain.range).toHaveBeenCalledWith(0, 49); + + await GET(makeRequest({ limit: "0" })); + expect(chain.range).toHaveBeenCalledWith(0, 0); + + await GET(makeRequest()); + expect(chain.range).toHaveBeenCalledWith(0, 19); + }); }); diff --git a/src/app/api/directory/route.ts b/src/app/api/directory/route.ts index 56d738e2..c048fd91 100644 --- a/src/app/api/directory/route.ts +++ b/src/app/api/directory/route.ts @@ -20,6 +20,8 @@ import { parsePaginationParam } from "@/lib/api-pagination"; const LNBITS_INVOICE_KEY = process.env.LNBITS_INVOICE_KEY || ""; const MAX_DIRECTORY_PAGE = 10_000; +const DEFAULT_DIRECTORY_LIMIT = 20; +const MAX_DIRECTORY_LIMIT = 50; const createListingSchema = z.object({ title: z.string().min(1).max(100), @@ -45,7 +47,12 @@ export async function GET(request: NextRequest) { 1, MAX_DIRECTORY_PAGE ); - const limit = 20; + const limit = parsePaginationParam( + url.searchParams.get("limit"), + DEFAULT_DIRECTORY_LIMIT, + 1, + MAX_DIRECTORY_LIMIT + ); const offset = (page - 1) * limit; const supabase = await createClient(); diff --git a/src/components/reviews/UserReviews.test.tsx b/src/components/reviews/UserReviews.test.tsx index 33b1d958..30f8f4b7 100644 --- a/src/components/reviews/UserReviews.test.tsx +++ b/src/components/reviews/UserReviews.test.tsx @@ -262,6 +262,13 @@ describe("UserReviews", () => { fireEvent.click(screen.getByText("Load more reviews")); expect(screen.getByText("Loading...")).toBeInTheDocument(); + + // The deferred response resolves 100ms from now. Let it land before the + // test ends: otherwise loadMore's setReviews runs after jsdom teardown and + // vitest reports an unhandled "window is not defined" for the whole run. + await waitFor(() => { + expect(screen.queryByText("Loading...")).not.toBeInTheDocument(); + }); }); it("appends new reviews to existing ones", async () => { diff --git a/src/lib/meta-url.test.ts b/src/lib/meta-url.test.ts new file mode 100644 index 00000000..3543a2ff --- /dev/null +++ b/src/lib/meta-url.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { toHttpUrl } from "./meta-url"; + +describe("toHttpUrl", () => { + it("rejects the data: favicon that suppresses the icon request", () => { + // — example.com ships this, and it used to + // reach the directory as logo_url and render as a broken image. + expect(toHttpUrl("data:,", "https://example.com")).toBe(""); + expect( + toHttpUrl("data:image/png;base64,iVBORw0KGgo=", "https://example.com") + ).toBe(""); + }); + + it("rejects other non-http schemes", () => { + expect(toHttpUrl("about:blank", "https://example.com")).toBe(""); + expect(toHttpUrl("javascript:void(0)", "https://example.com")).toBe(""); + }); + + it("returns nothing for empty or whitespace hrefs", () => { + expect(toHttpUrl("", "https://example.com")).toBe(""); + expect(toHttpUrl(" ", "https://example.com")).toBe(""); + expect(toHttpUrl(null, "https://example.com")).toBe(""); + expect(toHttpUrl(undefined, "https://example.com")).toBe(""); + }); + + it("resolves relative hrefs against the page URL", () => { + expect(toHttpUrl("/favicon.ico", "https://example.com/a/b")).toBe( + "https://example.com/favicon.ico" + ); + expect(toHttpUrl("icon.png", "https://example.com/a/b")).toBe( + "https://example.com/a/icon.png" + ); + }); + + it("keeps absolute http(s) URLs, including protocol-relative ones", () => { + expect(toHttpUrl("https://cdn.example.com/logo.svg", "https://example.com")).toBe( + "https://cdn.example.com/logo.svg" + ); + expect(toHttpUrl("//cdn.example.com/logo.svg", "https://example.com")).toBe( + "https://cdn.example.com/logo.svg" + ); + }); + + it("returns nothing when the href cannot be parsed", () => { + expect(toHttpUrl("http://[", "https://example.com")).toBe(""); + }); +}); diff --git a/src/lib/meta-url.ts b/src/lib/meta-url.ts new file mode 100644 index 00000000..c1ab17d4 --- /dev/null +++ b/src/lib/meta-url.ts @@ -0,0 +1,19 @@ +/** + * Resolve a candidate image href (favicon, og:image) against the page URL, + * keeping it only if it ends up as http(s). + * + * A page may declare `` to suppress the favicon + * request. That parses as a perfectly valid URL, so without a scheme check it + * reaches the directory as a logo and renders as a broken image. + */ +export function toHttpUrl(href: string | null | undefined, base: string): string { + if (!href || !href.trim()) return ""; + + try { + const resolved = new URL(href.trim(), base); + if (resolved.protocol !== "http:" && resolved.protocol !== "https:") return ""; + return resolved.href; + } catch { + return ""; + } +}