diff --git a/docs/DEVELOPMENT_LOG.md b/docs/DEVELOPMENT_LOG.md index 7085500..8f6f9ad 100644 --- a/docs/DEVELOPMENT_LOG.md +++ b/docs/DEVELOPMENT_LOG.md @@ -4,6 +4,47 @@ This log is written for both human developers and future coding agents. Keep ent short, dated, and implementation-oriented so the next contributor can understand what changed, why it changed, how it was checked, and what remains open. +## 2026-06-17 / Bing Metadata Image Crawlability + +### Goals + +- Improve Bing/search-engine result presentation signals without changing + visible page design, product behavior, or customer-facing copy. +- Ensure Open Graph, Twitter, and JSON-LD logo/image URLs are fetchable by + crawlers that do not send a page Referer. + +### Changes + +- Added `mediaMetadataUrl()` for search metadata images. It keeps visible page + imagery on direct OSS/CDN URLs, while routing metadata and structured-data + images through the site-owned `/api/media` endpoint. +- Enabled `/api/media` in production with safe bucket-relative path validation, + a stable `https://mist-arch.com/` OSS Referer, and public cache headers. +- Updated Open Graph, Twitter, Organization JSON-LD, and project JSON-LD image + URLs to use crawler-fetchable site URLs. +- Added explicit `index, follow` page robots metadata. +- Updated `robots.txt` to allow `/api/media` while keeping the broader `/api/` + path disallowed. +- Added Playwright coverage for metadata image URLs, `/api/media` image fetch, + JSON-LD logo URL, and robots rules. +- Allowed local Playwright runs to use `PLAYWRIGHT_PORT` so tests can run + without stopping unrelated local services on port `3000`. +- Updated `docs/SEO.md` and `docs/IMAGE_PIPELINE.md` with the metadata-image + proxy behavior. + +### Verification + +- `npm run typecheck`: passed. +- `npm run lint`: passed. +- `npm run build`: passed. +- `PLAYWRIGHT_PORT=3005 npm run test:e2e`: passed, 9 Chromium tests. +- `git diff --check`: passed. + +### Deployment + +- No merge, preview deploy, production deploy, Bing Webmaster action, or + production credential/account operation was performed. + ## 2026-06-06 / Bing URL Inspection Follow-up ### Goals diff --git a/docs/IMAGE_PIPELINE.md b/docs/IMAGE_PIPELINE.md index 6e6b70d..6baa1f3 100644 --- a/docs/IMAGE_PIPELINE.md +++ b/docs/IMAGE_PIPELINE.md @@ -250,6 +250,12 @@ production builds still emit direct OSS/CDN URLs. Set `NEXT_PUBLIC_DISABLE_MEDIA_PROXY=1` to bypass the local proxy when testing bucket Referer behavior directly. +Search metadata is the exception. `mediaMetadataUrl()` returns a site-owned +`/api/media?...` URL for Open Graph, Twitter, and JSON-LD images so crawlers +that fetch images without a page Referer can retrieve OSS assets. This does +not change visible page imagery; browser-facing `` and `` +sources should continue to use `mediaUrl()` / `pictureSet()`. + --- ## 7. Adding a new image diff --git a/docs/SEO.md b/docs/SEO.md index 570e519..ac1db8b 100644 --- a/docs/SEO.md +++ b/docs/SEO.md @@ -113,6 +113,7 @@ That helper generates: - localized alternates - Open Graph metadata - Twitter card metadata +- search-crawler-safe metadata image URLs Description guidance: @@ -130,6 +131,18 @@ zh: 岚·建筑设计是由程博、李博创立的建筑事务所,专注建 en: MIST Architects is a Shenzhen architecture studio working across cultural spaces, interiors, adaptive reuse, and public architecture. ``` +### Metadata images + +Visible page imagery still resolves through `mediaUrl()` and is served directly +from OSS or the future CDN. Search metadata is different: Open Graph, Twitter, +and JSON-LD image/logo URLs should use the site-owned `/api/media` proxy via +`mediaMetadataUrl()` so crawlers that fetch without a page Referer can retrieve +the assets. + +Reason: the OSS bucket has Referer protection enabled. Direct no-Referer GETs +to OSS return `403`, which can prevent Bing and other search engines from +fetching the logo or social preview images even when the page HTML is correct. + ## 5. Canonical and hreflang Localized canonical URLs are generated with: @@ -218,11 +231,15 @@ When adding new images: ```text Allow: / +Allow: /api/media Disallow: /api/ Sitemap: https://mist-arch.com/sitemap.xml Host: https://mist-arch.com ``` +The `/api/media` allow rule is intentional. Metadata images use this endpoint, +while the broader `/api/` path remains blocked from indexing. + `src/app/sitemap.ts` generates localized entries for: - home diff --git a/playwright.config.ts b/playwright.config.ts index efd8c7b..dfcc691 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,6 +1,8 @@ import { defineConfig, devices } from "@playwright/test"; const isCi = Boolean(process.env.CI); +const port = process.env.PLAYWRIGHT_PORT ?? "3000"; +const baseURL = `http://127.0.0.1:${port}`; export default defineConfig({ testDir: "./tests/e2e", @@ -8,12 +10,12 @@ export default defineConfig({ workers: 1, reporter: [["list"]], use: { - baseURL: "http://127.0.0.1:3000", + baseURL, trace: "on-first-retry" }, webServer: { - command: isCi ? "npm run start:standalone" : "npm run dev -- --hostname 127.0.0.1 --port 3000", - url: "http://127.0.0.1:3000/zh", + command: isCi ? "npm run start:standalone" : `npm run dev -- --hostname 127.0.0.1 --port ${port}`, + url: `${baseURL}/zh`, reuseExistingServer: !isCi, timeout: 120_000 }, diff --git a/src/app/[locale]/layout.tsx b/src/app/[locale]/layout.tsx index 46114a2..71c3079 100644 --- a/src/app/[locale]/layout.tsx +++ b/src/app/[locale]/layout.tsx @@ -32,6 +32,10 @@ export const metadata: Metadata = { creator: "MIST Architects", publisher: "MIST Architects", category: "architecture", + robots: { + index: true, + follow: true + }, formatDetection: { telephone: false, email: false, diff --git a/src/app/api/media/route.ts b/src/app/api/media/route.ts index 216d6af..cb65578 100644 --- a/src/app/api/media/route.ts +++ b/src/app/api/media/route.ts @@ -1,7 +1,7 @@ import { NextRequest } from "next/server"; const MEDIA_BASE = "https://mist-architects-media.oss-cn-shenzhen.aliyuncs.com"; -const DEV_REFERER = "http://127.0.0.1:3000/"; +const OSS_REFERER = "https://mist-arch.com/"; function encodePath(path: string): string { return path @@ -23,10 +23,6 @@ function isSafeMediaPath(path: string): boolean { } export async function GET(request: NextRequest) { - if (process.env.NODE_ENV !== "development") { - return new Response("Not found", { status: 404 }); - } - const path = request.nextUrl.searchParams.get("path") || ""; const ossProcess = request.nextUrl.searchParams.get("process"); @@ -41,7 +37,7 @@ export async function GET(request: NextRequest) { const upstream = await fetch(source, { headers: { - Referer: DEV_REFERER + Referer: OSS_REFERER } }); @@ -60,7 +56,7 @@ export async function GET(request: NextRequest) { const contentLength = upstream.headers.get("content-length"); if (contentType) headers.set("content-type", contentType); if (contentLength) headers.set("content-length", contentLength); - headers.set("cache-control", "public, max-age=86400"); + headers.set("cache-control", "public, max-age=604800, stale-while-revalidate=86400"); return new Response(upstream.body, { status: upstream.status, diff --git a/src/app/robots.ts b/src/app/robots.ts index 362fbce..82fd5b9 100644 --- a/src/app/robots.ts +++ b/src/app/robots.ts @@ -6,7 +6,7 @@ export default function robots(): MetadataRoute.Robots { return { rules: { userAgent: "*", - allow: "/", + allow: ["/", "/api/media"], disallow: ["/api/"] }, sitemap: `${siteOrigin}/sitemap.xml`, diff --git a/src/lib/media.ts b/src/lib/media.ts index 6b17f87..40a6cd0 100644 --- a/src/lib/media.ts +++ b/src/lib/media.ts @@ -264,6 +264,53 @@ export function mediaUrl(path: string, opts: ProcessOptions = {}): string { return process ? `${url}?x-oss-process=${process}` : url; } +function mediaProxyPath(path: string, process: string | null) { + const params = new URLSearchParams({ path }); + if (process) { + params.set("process", process); + } + return `/api/media?${params.toString()}`; +} + +function proxiedOssUrl(url: string) { + try { + const parsed = new URL(url); + const mediaBase = new URL(MEDIA_BASE); + + if (parsed.origin !== mediaBase.origin) { + return null; + } + + const path = decodeURIComponent(parsed.pathname.replace(/^\/+/, "")); + return mediaProxyPath(path, parsed.searchParams.get("x-oss-process")); + } catch { + return null; + } +} + +/** + * Resolve a bucket-relative path or existing OSS URL to the site's media proxy. + * + * Use this for metadata/structured-data image URLs that search engines fetch + * without a page Referer. Visible page imagery should keep using `mediaUrl()` + * so browser delivery continues to go directly through OSS/CDN. + */ +export function mediaMetadataUrl(path: string, opts: ProcessOptions = {}): string { + if (!path || path.startsWith("data:")) { + return path; + } + + if (path.startsWith("/api/media?")) { + return path; + } + + if (/^https?:\/\//i.test(path)) { + return proxiedOssUrl(path) ?? path; + } + + return mediaProxyPath(path, buildOssProcess(opts)); +} + /** * Build a `srcset` string for a single (path, format, quality) * combination across a width ladder. diff --git a/src/lib/seo.ts b/src/lib/seo.ts index f01a7b8..30f7f03 100644 --- a/src/lib/seo.ts +++ b/src/lib/seo.ts @@ -2,7 +2,7 @@ import type { Metadata } from "next"; import { about, type Project } from "@/content/site"; import { defaultLocale, locales, otherLocale, type Locale, withLocale } from "@/lib/i18n"; -import { mediaUrl } from "@/lib/media"; +import { mediaMetadataUrl, mediaUrl } from "@/lib/media"; const DEFAULT_SITE_ORIGIN = "https://mist-arch.com"; @@ -118,7 +118,7 @@ export function buildPageMetadata({ }: PageMetadataInput): Metadata { const canonical = localizedSiteUrl(locale, path); const fullTitle = formatTitle(title); - const imageUrl = absoluteUrl(image); + const imageUrl = absoluteUrl(mediaMetadataUrl(image)); const pageKeywords = Array.from(new Set([...siteKeywords[locale], ...keywords])); return { @@ -167,8 +167,8 @@ export function organizationJsonLd(locale: Locale) { wechatSeoContact.publicAccount[locale] ], url: localizedSiteUrl(locale), - logo: absoluteUrl(mediaUrl("LOGO/logo.png", { quality: "raw" })), - image: absoluteUrl(mediaUrl(about.heroImage, { width: 1920, quality: "std" })), + logo: absoluteUrl(mediaMetadataUrl("LOGO/logo.png", { quality: "raw" })), + image: absoluteUrl(mediaMetadataUrl(about.heroImage, { width: 1920, quality: "std" })), description: siteDescription[locale], email: "info@mist-arch.com", telephone: "+86 186 1303 3310", @@ -323,7 +323,7 @@ export function projectJsonLd(locale: Locale, project: Project) { headline: project.title[locale], description: project.dek[locale], url: localizedSiteUrl(locale, `/projects/${project.slug}`), - image: absoluteUrl(project.heroImage), + image: absoluteUrl(mediaMetadataUrl(project.heroImage)), creator: { "@id": `${siteOrigin}/#organization` }, diff --git a/tests/e2e/home.spec.ts b/tests/e2e/home.spec.ts index 1a6fea8..f940af7 100644 --- a/tests/e2e/home.spec.ts +++ b/tests/e2e/home.spec.ts @@ -39,6 +39,34 @@ test.describe("localized homepage", () => { "alt", /MIST Architects project photograph/ ); + + const ogImage = page.locator('meta[property="og:image"]'); + await expect(ogImage).toHaveAttribute("content", /^https:\/\/mist-arch\.com\/api\/media\?/); + + const ogImageUrl = new URL((await ogImage.getAttribute("content")) ?? ""); + expect(ogImageUrl.searchParams.get("path")).toContain("home/horizontal/"); + expect(ogImageUrl.searchParams.get("process")).toContain("image/resize"); + + const twitterImage = page.locator('meta[name="twitter:image"]'); + await expect(twitterImage).toHaveAttribute("content", ogImageUrl.toString()); + + await expect(page.locator('meta[name="robots"]')).toHaveAttribute("content", "index, follow"); + + const localImageResponse = await page.request.get(`${ogImageUrl.pathname}${ogImageUrl.search}`); + expect(localImageResponse.ok()).toBe(true); + expect(localImageResponse.headers()["content-type"]).toContain("image/"); + }); +}); + +test.describe("search crawler access", () => { + test("robots allows metadata media while keeping the API blocked", async ({ page }) => { + const response = await page.request.get("/robots.txt"); + expect(response.ok()).toBe(true); + + const robots = await response.text(); + expect(robots).toContain("Allow: /api/media"); + expect(robots).toContain("Disallow: /api/"); + expect(robots).toContain("Sitemap: https://mist-arch.com/sitemap.xml"); }); }); @@ -65,6 +93,7 @@ test.describe("about contact interactions", () => { expect(structuredData).toContain("ContactPage"); expect(structuredData).toContain("MIST Architects WeChat public account MIST-ARCH QR code"); expect(structuredData).toContain("/20260531-191007.jpeg"); + expect(structuredData).toContain("https://mist-arch.com/api/media?path=LOGO%2Flogo.png"); await page.getByRole("button", { name: "MIST-ARCH (岚建筑设计)" }).click();