Fix directory broken logos, ignored ?limit=, and missing screenshots

scripts/backfill-screenshots.ts

@@ -0,0 +1,143 @@
+#!/usr/bin/env tsx
+/**
+ * Backfill homepage screenshots for directory listings that have none.
+ *
+ * Usage:
+ *   tsx scripts/backfill-screenshots.ts            # capture everything missing
+ *   tsx scripts/backfill-screenshots.ts --dry-run  # list what would be captured
+ *   tsx scripts/backfill-screenshots.ts --limit 5  # cap how many are rendered
+ *
+ * Requires: NEXT_PUBLIC_SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, RASTERLY_API_KEY in .env
+ *
+ * rasterly's free tier is 100 renders/month, so the script reports the quota
+ * remaining after each capture and stops if the API says the budget is gone.
+ */
+
+import { createClient } from "@supabase/supabase-js";
+import crypto from "crypto";
+import { config } from "dotenv";
+
+config(); // load .env
+
+const SUPABASE_URL = process.env.NEXT_PUBLIC_SUPABASE_URL!;
+const SUPABASE_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY!;
+const RASTERLY_KEY = process.env.RASTERLY_API_KEY!;
+
+if (!SUPABASE_URL || !SUPABASE_KEY || !RASTERLY_KEY) {
+  console.error(
+    "Missing NEXT_PUBLIC_SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, or RASTERLY_API_KEY"
+  );
+  process.exit(1);
+}
+
+const supabase = createClient(SUPABASE_URL, SUPABASE_KEY);
+
+const args = process.argv.slice(2);
+const dryRun = args.includes("--dry-run");
+const limitArg = args.indexOf("--limit");
+const limit =
+  limitArg !== -1 && args[limitArg + 1] ? Number(args[limitArg + 1]) : Infinity;
+
+type Listing = { id: string; title: string; url: string };
+
+async function render(url: string): Promise<{ buf: Buffer; quota: string | null }> {
+  const endpoint = `https://api.rasterly.dev/v1/screenshot?url=${encodeURIComponent(
+    url
+  )}&format=png&width=1280&height=800`;
+
+  const res = await fetch(endpoint, {
+    headers: { "X-Api-Key": RASTERLY_KEY },
+    signal: AbortSignal.timeout(30000),
+  });
+
+  if (!res.ok) throw new Error(`rasterly HTTP ${res.status}`);
+  if (!(res.headers.get("content-type") || "").startsWith("image/")) {
+    throw new Error("rasterly returned a non-image response");
+  }
+
+  const buf = Buffer.from(await res.arrayBuffer());
+  if (buf.length === 0) throw new Error("rasterly returned an empty image");
+
+  return { buf, quota: res.headers.get("x-quota-remaining") };
+}
+
+async function upload(url: string, buf: Buffer): Promise<string> {
+  // Same path shape the fetch-meta route writes, so both sources interleave.
+  const urlHash = crypto.createHash("md5").update(url).digest("hex");
+  const filePath = `${urlHash}/${Date.now()}.png`;
+
+  const { error } = await supabase.storage
+    .from("directory-screenshots")
+    .upload(filePath, buf, { contentType: "image/png", upsert: true });
+
+  if (error) throw new Error(`upload failed: ${error.message}`);
+
+  const {
+    data: { publicUrl },
+  } = supabase.storage.from("directory-screenshots").getPublicUrl(filePath);
+
+  return publicUrl;
+}
+
+async function main() {
+  const { data, error } = await supabase
+    .from("project_listings")
+    .select("id, title, url")
+    .eq("status", "active")
+    .is("screenshot_url", null)
+    .order("created_at", { ascending: false });
+
+  if (error) {
+    console.error("Failed to read listings:", error.message);
+    process.exit(1);
+  }
+
+  const listings = (data || []) as Listing[];
+  console.log(`${listings.length} active listing(s) without a screenshot`);
+
+  if (dryRun) {
+    for (const l of listings) console.log(`  would capture: ${l.title} — ${l.url}`);
+    return;
+  }
+
+  let done = 0;
+  let failed = 0;
+
+  for (const listing of listings) {
+    if (done >= limit) {
+      console.log(`Reached --limit ${limit}, stopping.`);
+      break;
+    }
+
+    try {
+      const { buf, quota } = await render(listing.url);
+      const publicUrl = await upload(listing.url, buf);
+
+      const { error: updateError } = await supabase
+        .from("project_listings")
+        .update({ screenshot_url: publicUrl })
+        .eq("id", listing.id);
+
+      if (updateError) throw new Error(`update failed: ${updateError.message}`);
+
+      done++;
+      console.log(`✓ ${listing.title} — ${buf.length} bytes, quota left: ${quota ?? "?"}`);
+
+      if (quota !== null && Number(quota) <= 0) {
+        console.log("rasterly quota exhausted, stopping.");
+        break;
+      }
+    } catch (err) {
+      failed++;
+      const msg = err instanceof Error ? err.message : String(err);
+      console.error(`✗ ${listing.title} — ${msg}`);
+    }
+  }
+
+  console.log(`\nCaptured ${done}, failed ${failed}.`);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

src/app/api/directory/fetch-meta/route.ts

@@ -1,5 +1,6 @@
 import { NextRequest, NextResponse } from "next/server";
 import { createServiceClient } from "@/lib/supabase/service";
+import { toHttpUrl } from "@/lib/meta-url";
 import OpenAI from "openai";
 import crypto from "crypto";
 
@@ -101,26 +102,20 @@ export async function POST(request: NextRequest) {
     }
 
     if (!logo_url) {
-      const faviconHref = extractFavicon(html);
-      if (faviconHref) {
-        try {
-          logo_url = new URL(faviconHref, url).href;
-        } catch {
-          logo_url = faviconHref;
-        }
-      }
+      // A page may declare <link rel="icon" href="data:,"> to suppress the
+      // favicon request. That parses as a valid URL, so it has to be rejected
+      // by scheme or it reaches the directory and renders as a broken image.
+      logo_url = toHttpUrl(extractFavicon(html), url);
     }
 
     if (!logo_url && ogImage) {
-      try {
-        logo_url = new URL(ogImage, url).href;
-      } catch {
-        logo_url = ogImage;
-      }
+      logo_url = toHttpUrl(ogImage, url);
     }
 
     if (!logo_url) {
-      logo_url = `${parsedUrl.origin}/favicon.ico`;
+      // Only fall back to /favicon.ico if the site actually serves one.
+      const icoFallback = `${parsedUrl.origin}/favicon.ico`;
+      if (await urlExists(icoFallback)) logo_url = icoFallback;
     }
 
     // --- Banner detection ---
@@ -344,6 +339,22 @@ function extractTitle(html: string): string {
   return match ? decodeEntities(match[1].trim()) : "";
 }
 
+/**
+ * HEAD a URL to see whether it is actually served.
+ */
+async function urlExists(candidate: string): Promise<boolean> {
+  try {
+    const res = await fetch(candidate, {
+      method: "HEAD",
+      signal: AbortSignal.timeout(3000),
+      redirect: "follow",
+    });
+    return res.ok;
+  } catch {
+    return false;
+  }
+}
+
 function extractFavicon(html: string): string {
   const match = html.match(
     /<link[^>]*rel=["'](?:icon|shortcut icon|apple-touch-icon)["'][^>]*href=["']([^"']*)["'][^>]*\/?>/i

src/app/api/directory/route.test.ts

@@ -63,4 +63,39 @@ describe("GET /api/directory", () => {
     expect(chain.range).toHaveBeenCalledWith(0, 19);
     expect(body.page).toBe(1);
   });
+
+  it("honours limit instead of always returning 20 rows", async () => {
+    const chain = chainResult({ data: [], error: null, count: 0 });
+    mockFrom.mockReturnValue(chain);
+
+    const res = await GET(makeRequest({ limit: "5" }));
+    const body = await res.json();
+
+    expect(res.status).toBe(200);
+    expect(chain.range).toHaveBeenCalledWith(0, 4);
+    expect(body.per_page).toBe(5);
+  });
+
+  it("offsets by the requested limit when paging", async () => {
+    const chain = chainResult({ data: [], error: null, count: 0 });
+    mockFrom.mockReturnValue(chain);
+
+    await GET(makeRequest({ limit: "5", page: "3" }));
+
+    expect(chain.range).toHaveBeenCalledWith(10, 14);
+  });
+
+  it("clamps limit to the allowed range and defaults when absent", async () => {
+    const chain = chainResult({ data: [], error: null, count: 0 });
+    mockFrom.mockReturnValue(chain);
+
+    await GET(makeRequest({ limit: "9999" }));
+    expect(chain.range).toHaveBeenCalledWith(0, 49);
+
+    await GET(makeRequest({ limit: "0" }));
+    expect(chain.range).toHaveBeenCalledWith(0, 0);
+
+    await GET(makeRequest());
+    expect(chain.range).toHaveBeenCalledWith(0, 19);
+  });
 });

src/app/api/directory/route.ts

@@ -20,6 +20,8 @@ import { parsePaginationParam } from "@/lib/api-pagination";
 
 const LNBITS_INVOICE_KEY = process.env.LNBITS_INVOICE_KEY || "";
 const MAX_DIRECTORY_PAGE = 10_000;
+const DEFAULT_DIRECTORY_LIMIT = 20;
+const MAX_DIRECTORY_LIMIT = 50;
 
 const createListingSchema = z.object({
   title: z.string().min(1).max(100),
@@ -45,7 +47,12 @@ export async function GET(request: NextRequest) {
       1,
       MAX_DIRECTORY_PAGE
     );
-    const limit = 20;
+    const limit = parsePaginationParam(
+      url.searchParams.get("limit"),
+      DEFAULT_DIRECTORY_LIMIT,
+      1,
+      MAX_DIRECTORY_LIMIT
+    );
     const offset = (page - 1) * limit;
 
     const supabase = await createClient();

src/components/reviews/UserReviews.test.tsx

@@ -262,6 +262,13 @@ describe("UserReviews", () => {
     fireEvent.click(screen.getByText("Load more reviews"));
 
     expect(screen.getByText("Loading...")).toBeInTheDocument();
+
+    // The deferred response resolves 100ms from now. Let it land before the
+    // test ends: otherwise loadMore's setReviews runs after jsdom teardown and
+    // vitest reports an unhandled "window is not defined" for the whole run.
+    await waitFor(() => {
+      expect(screen.queryByText("Loading...")).not.toBeInTheDocument();
+    });
   });
 
   it("appends new reviews to existing ones", async () => {

src/lib/meta-url.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { toHttpUrl } from "./meta-url";
+
+describe("toHttpUrl", () => {
+  it("rejects the data: favicon that suppresses the icon request", () => {
+    // <link rel="icon" href="data:,"> — example.com ships this, and it used to
+    // reach the directory as logo_url and render as a broken image.
+    expect(toHttpUrl("data:,", "https://example.com")).toBe("");
+    expect(
+      toHttpUrl("data:image/png;base64,iVBORw0KGgo=", "https://example.com")
+    ).toBe("");
+  });
+
+  it("rejects other non-http schemes", () => {
+    expect(toHttpUrl("about:blank", "https://example.com")).toBe("");
+    expect(toHttpUrl("javascript:void(0)", "https://example.com")).toBe("");
+  });
+
+  it("returns nothing for empty or whitespace hrefs", () => {
+    expect(toHttpUrl("", "https://example.com")).toBe("");
+    expect(toHttpUrl("   ", "https://example.com")).toBe("");
+    expect(toHttpUrl(null, "https://example.com")).toBe("");
+    expect(toHttpUrl(undefined, "https://example.com")).toBe("");
+  });
+
+  it("resolves relative hrefs against the page URL", () => {
+    expect(toHttpUrl("/favicon.ico", "https://example.com/a/b")).toBe(
+      "https://example.com/favicon.ico"
+    );
+    expect(toHttpUrl("icon.png", "https://example.com/a/b")).toBe(
+      "https://example.com/a/icon.png"
+    );
+  });
+
+  it("keeps absolute http(s) URLs, including protocol-relative ones", () => {
+    expect(toHttpUrl("https://cdn.example.com/logo.svg", "https://example.com")).toBe(
+      "https://cdn.example.com/logo.svg"
+    );
+    expect(toHttpUrl("//cdn.example.com/logo.svg", "https://example.com")).toBe(
+      "https://cdn.example.com/logo.svg"
+    );
+  });
+
+  it("returns nothing when the href cannot be parsed", () => {
+    expect(toHttpUrl("http://[", "https://example.com")).toBe("");
+  });
+});

src/lib/meta-url.ts

@@ -0,0 +1,19 @@
+/**
+ * Resolve a candidate image href (favicon, og:image) against the page URL,
+ * keeping it only if it ends up as http(s).
+ *
+ * A page may declare `<link rel="icon" href="data:,">` to suppress the favicon
+ * request. That parses as a perfectly valid URL, so without a scheme check it
+ * reaches the directory as a logo and renders as a broken image.
+ */
+export function toHttpUrl(href: string | null | undefined, base: string): string {
+  if (!href || !href.trim()) return "";
+
+  try {
+    const resolved = new URL(href.trim(), base);
+    if (resolved.protocol !== "http:" && resolved.protocol !== "https:") return "";
+    return resolved.href;
+  } catch {
+    return "";
+  }
+}