Repair repository refresh trust boundaries

.eleventy.js

@@ -1,5 +1,6 @@
 import markdownIt from 'markdown-it';
 import { installMathRenderer } from './scripts/markdown-math.mjs';
+import { resolveRepositoryImageReference } from './scripts/repository-markdown-links.mjs';
 
 // === MODULE_BUILD ===
 // id: eleventy_site_configuration
@@ -121,7 +122,12 @@ function installSourceReferenceRenderer(md) {
   const image = md.renderer.rules.image || ((tokens, index, options, _env, self) => self.renderToken(tokens, index, options));
   md.renderer.rules.image = (tokens, index, options, env, self) => {
     const srcIndex = tokens[index].attrIndex('src');
-    if (srcIndex >= 0) tokens[index].attrs[srcIndex][1] = resolveSourceReference(tokens[index].attrs[srcIndex][1], env?.sourceUrl, env?.repositorySource === true);
+    if (srcIndex >= 0) {
+      const source = tokens[index].attrs[srcIndex][1];
+      tokens[index].attrs[srcIndex][1] = env?.repositorySource === true
+        ? resolveRepositoryImageReference(source, env?.sourceUrl)
+        : resolveSourceReference(source, env?.sourceUrl, false);
+    }
     return image(tokens, index, options, env, self);
   };
   return md;

scripts/fetch-project-docs.mjs

@@ -1,5 +1,8 @@
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
-import { fetchRepositoryPublicProjection } from './repository-public-projection.mjs';
+import {
+  fetchRepositoryPublicProjection,
+  isRepositoryNotPublicError
+} from './repository-public-projection.mjs';
 
 // === MODULE_BUILD ===
 // id: project_documentation_projection
@@ -12,7 +15,7 @@ import { fetchRepositoryPublicProjection } from './repository-public-projection.
 //   network: delegates exact-head reads to repository_public_projection
 //   storage: generated projectDocs JSON plus last-known-good snapshot
 //   authority: source repositories own document content; this website owns presentation only
-//   failure: same-head last-known-good data may be retained with fallback=true; unknown current content remains hmmm
+//   failure: same-head last-known-good data may be retained with fallback=true; non-public repositories never reuse cached document content
 // === END BOUNDARIES ===
 
 const GENERATED_REPOS = 'src/_data/generated/repos.json';
@@ -24,6 +27,29 @@ async function readSnapshot() {
   catch { return null; }
 }
 
+function unavailableProjection(repo, message) {
+  return {
+    schema: 'interdependency.repository-public-projection/0.1.0',
+    repository: 'The-Interdependency/' + repo.name,
+    name: repo.name,
+    defaultBranch: repo.default_branch || null,
+    headSha: null,
+    headCommittedAt: null,
+    refreshedAt: null,
+    documentation: {
+      readme: null,
+      documents: [],
+      projectedDocumentCount: 0,
+      projectedBytes: 0,
+      hmmm: [message]
+    },
+    msdmd: null,
+    fallback: false,
+    unavailable: true,
+    hmmm: [message]
+  };
+}
+
 async function main() {
   await mkdir('src/_data/generated', { recursive: true });
   await mkdir('src/_data/snapshots', { recursive: true });
@@ -52,6 +78,13 @@ async function main() {
         includeMsdmd: false
       });
     } catch (error) {
+      if (isRepositoryNotPublicError(error)) {
+        byRepository[repo.name] = unavailableProjection(
+          repo,
+          'Repository is not currently public; cached documentation was discarded instead of republished.'
+        );
+        continue;
+      }
       const prior = previousByRepo[repo.name];
       if (prior?.headSha === repo.head_sha) {
         fallbackCount += 1;
@@ -61,19 +94,10 @@ async function main() {
           hmmm: [...new Set([...(prior.hmmm || []), 'Documentation refresh failed at an unchanged head; retained the last-known-good projection.'])]
         };
       } else {
-        byRepository[repo.name] = {
-          schema: 'interdependency.repository-public-projection/0.1.0',
-          repository: 'The-Interdependency/' + repo.name,
-          name: repo.name,
-          defaultBranch: repo.default_branch || null,
-          headSha: repo.head_sha || null,
-          headCommittedAt: repo.head_committed_at || null,
-          refreshedAt: new Date().toISOString(),
-          documentation: { readme: null, documents: [], projectedDocumentCount: 0, projectedBytes: 0, hmmm: ['Current documentation could not be projected at this head.'] },
-          msdmd: null,
-          fallback: false,
-          hmmm: ['Documentation refresh failed and no same-head fallback was eligible.']
-        };
+        byRepository[repo.name] = unavailableProjection(
+          repo,
+          'Documentation refresh failed and no same-head fallback was eligible.'
+        );
       }
     }
   }

scripts/repository-markdown-links.mjs

@@ -0,0 +1,22 @@
+// Usage: resolve repository-relative Markdown image references against raw.githubusercontent.com while ordinary links continue to target exact GitHub blob sources.
+export function resolveRepositoryImageReference(value, sourceUrl) {
+  const reference = String(value || '');
+  if (!sourceUrl || !reference || reference.startsWith('//')) return reference;
+  if (/^[a-z][a-z0-9+.-]*:/i.test(reference)) return reference;
+
+  try {
+    const source = new URL(sourceUrl);
+    const parts = source.pathname.split('/').filter(Boolean);
+    if (source.hostname !== 'github.com' || parts[2] !== 'blob' || !parts[3]) return reference;
+
+    const [owner, repo, , head, ...sourcePath] = parts;
+    const rawRoot = `https://raw.githubusercontent.com/${owner}/${repo}/${head}/`;
+    if (reference.startsWith('/')) return new URL(reference.slice(1), rawRoot).href;
+    if (reference.startsWith('#')) return sourceUrl + reference;
+
+    const basePath = sourcePath.length ? sourcePath.join('/') : '';
+    return new URL(reference, new URL(basePath, rawRoot)).href;
+  } catch {
+    return reference;
+  }
+}

scripts/repository-public-projection.mjs

@@ -21,6 +21,11 @@
 const MAX_DOCUMENTS = 32;
 const MAX_DOCUMENT_BYTES = 128 * 1024;
 const MAX_TOTAL_BYTES = 640 * 1024;
+const MAX_METADATA_BYTES = 2 * 1024 * 1024;
+const MAX_API_BYTES = 4 * 1024 * 1024;
+const REQUEST_TIMEOUT_MS = 10_000;
+export const REPOSITORY_NOT_PUBLIC = 'REPOSITORY_NOT_PUBLIC';
+const CONTENT_TOO_LARGE = 'CONTENT_TOO_LARGE';
 
 function sha256(text) {
   return createHash('sha256').update(text).digest('hex');
@@ -58,20 +63,63 @@
 
 async function getJson(url) {
   if (!(url instanceof URL) || url.origin !== API_ORIGIN || url.protocol !== 'https:') throw new Error('refusing non-GitHub API target');
-  const response = await fetch(url, { headers: headers() });
-  if (!response.ok) throw new Error('GitHub API request failed: ' + response.status);
-  return response.json();
+  // Only normalized public repository/head identifiers flow here; destination origin is hard-pinned and checked above.
+  // lgtm[js/file-access-to-http]
+  // codeql[js/file-access-to-http]
+  const response = await fetch(url, { headers: headers(), signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) });
+  if (!response.ok) {
+    const error = new Error('GitHub API request failed: ' + response.status);
+    error.status = response.status;
+    throw error;
+  }
+  if (!response.body?.getReader && typeof response.text !== 'function' && typeof response.json === 'function') {
+    return response.json();
+  }
+  return JSON.parse(await readBoundedText(response, MAX_API_BYTES));
+}
+
+async function readBoundedText(response, maxBytes) {
+  const reader = response.body?.getReader?.();
+  if (!reader) {
+    const text = await response.text();
+    if (Buffer.byteLength(text) > maxBytes) {
+      const error = new Error('raw GitHub response exceeded byte limit');
+      error.code = CONTENT_TOO_LARGE;
+      throw error;
+    }
+    return text;
+  }
+
+  const decoder = new TextDecoder();
+  let totalBytes = 0;
+  let text = '';
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    totalBytes += value.byteLength;
+    if (totalBytes > maxBytes) {
+      await reader.cancel();
+      const error = new Error('raw GitHub response exceeded byte limit');
+      error.code = CONTENT_TOO_LARGE;
+      throw error;
+    }
+    text += decoder.decode(value, { stream: true });
+  }
+  return text + decoder.decode();
 }
 
-async function getText(url) {
+async function getText(url, maxBytes = MAX_METADATA_BYTES) {
   if (!(url instanceof URL) || url.origin !== RAW_ORIGIN || url.protocol !== 'https:') throw new Error('refusing non-raw-GitHub target');
-  const response = await fetch(url, { headers: { 'user-agent': 'the-interdependency-public-projection' } });
+  const response = await fetch(url, {
+    headers: { 'user-agent': 'the-interdependency-public-projection' },
+    signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
+  });
   if (!response.ok) {
     const error = new Error('raw GitHub request failed: ' + response.status);
     error.status = response.status;
     throw error;
   }
-  return response.text();
+  return readBoundedText(response, maxBytes);
 }
 
 export function selectDocumentationPaths(treeEntries) {
@@ -92,8 +140,35 @@
   };
 }
 
+function notPublicError() {
+  const error = new Error('repository is not public');
+  error.code = REPOSITORY_NOT_PUBLIC;
+  return error;
+}
+
+export function isRepositoryNotPublicError(error) {
+  return error?.code === REPOSITORY_NOT_PUBLIC;
+}
+
+async function publicRepositoryMetadata(repo) {
+  let metadata;
+  try {
+    metadata = await getJson(apiUrl('/repos/' + encodeURIComponent(ORGANIZATION) + '/' + encodeURIComponent(repo)));
+  } catch (error) {
+    // GitHub deliberately returns 404 for a private repository that the build
+    // token cannot see. For a public-only projection, missing and inaccessible
+    // metadata are both authority to publish nothing, never to reuse cache.
+    if (error?.status === 404) throw notPublicError();
+    throw error;
+  }
+  if (metadata.private === true || (metadata.visibility && metadata.visibility !== 'public')) {
+    throw notPublicError();
+  }
+  return metadata;
+}
+
 async function resolveHead(repo) {
-  const metadata = await getJson(apiUrl('/repos/' + encodeURIComponent(ORGANIZATION) + '/' + encodeURIComponent(repo)));
+  const metadata = await publicRepositoryMetadata(repo);
   const defaultBranch = metadata.default_branch || 'main';
   const commit = await getJson(apiUrl('/repos/' + encodeURIComponent(ORGANIZATION) + '/' + encodeURIComponent(repo) + '/commits/' + encodeURIComponent(defaultBranch)));
   return {
@@ -107,14 +182,18 @@
   const hmmm = [];
   const tree = await getJson(apiUrl('/repos/' + encodeURIComponent(ORGANIZATION) + '/' + encodeURIComponent(repo) + '/git/trees/' + encodeURIComponent(headSha), { recursive: 1 }));
   const selected = selectDocumentationPaths(tree.tree || []);
-  if (!selected.readme) hmmm.push('No root README.md was present at the consumed repository head.');
+  const treeTruncated = tree.truncated === true;
+  if (treeTruncated) hmmm.push('GitHub recursive tree response was truncated; document discovery is incomplete.');
+  if (!selected.readme) hmmm.push(treeTruncated
+    ? 'Root README presence is unresolved because document discovery was truncated.'
+    : 'No root README.md was present at the consumed repository head.');
   if (selected.omittedCount) hmmm.push(selected.omittedCount + ' Markdown document(s) were omitted by the bounded public-document count limit.');
 
   const documents = [];
   let totalBytes = 0;
   for (const path of selected.paths) {
     try {
-      const content = await getText(rawUrl(repo, headSha, path));
+      const content = await getText(rawUrl(repo, headSha, path), MAX_DOCUMENT_BYTES);
       const bytes = Buffer.byteLength(content);
       if (bytes > MAX_DOCUMENT_BYTES) {
         hmmm.push(path + ' exceeds the per-document public projection limit and was omitted.');
@@ -133,7 +212,15 @@
         sourceUrl: sourceUrl(repo, headSha, path)
       });
     } catch (error) {
-      hmmm.push(path + ' could not be read at the consumed head.');
+      if (error?.code === CONTENT_TOO_LARGE) {
+        hmmm.push(path + ' exceeds the per-document public projection limit and was omitted.');
+        continue;
+      }
+      if (error?.status === 404) {
+        hmmm.push(path + ' could not be read at the consumed head.');
+        continue;
+      }
+      throw error;
     }
   }
 
@@ -143,6 +230,8 @@
     documents: documents.filter(item => item !== readme),
     projectedDocumentCount: documents.length,
     projectedBytes: totalBytes,
+    treeTruncated,
+    discoveryComplete: !treeTruncated,
     hmmm
   };
 }
@@ -160,12 +249,24 @@
       const block = String(declaration?.block || 'hmmm');
       blockCounts[block] = (blockCounts[block] || 0) + 1;
     }
+    const declaredRepo = collection.repo || null;
+    const declaredRepoMatchesRepository = declaredRepo === null
+      || declaredRepo === repo
+      || declaredRepo === ORGANIZATION + '/' + repo;
+    const hmmm = [];
+    if (!declaredRepoMatchesRepository) {
+      hmmm.push('Collection-declared repository identity does not match the repository being refreshed.');
+    }
+    if (collection.source_commit && collection.source_commit !== headSha) {
+      hmmm.push('Collection-declared source_commit does not match the repository head consumed by this live refresh.');
+    }
     return {
-      status: 'ok',
+      status: declaredRepoMatchesRepository ? 'ok' : 'invalid',
       path,
       sourceUrl: sourceUrl(repo, headSha, path),
       sha256: sha256(text),
-      declaredRepo: collection.repo || null,
+      declaredRepo,
+      declaredRepoMatchesRepository,
       declaredSourceCommit: collection.source_commit || null,
       sourceCommitMatchesHead: collection.source_commit ? collection.source_commit === headSha : null,
       counts: {
@@ -174,9 +275,7 @@
         edges: edges.length
       },
       blockCounts,
-      hmmm: collection.source_commit && collection.source_commit !== headSha
-        ? ['Collection-declared source_commit does not match the repository head consumed by this live refresh.']
-        : []
+      hmmm
     };
   } catch (error) {
     return {
@@ -185,6 +284,7 @@
       sourceUrl: sourceUrl(repo, headSha, path),
       sha256: null,
       declaredRepo: null,
+      declaredRepoMatchesRepository: null,
       declaredSourceCommit: null,
       sourceCommitMatchesHead: null,
       counts: { declarations: 0, gaps: 0, edges: 0 },
@@ -196,15 +296,21 @@
   }
 }
 
-export async function fetchRepositoryPublicProjection(repository, {
-  headSha = null,
-  defaultBranch = null,
-  headCommittedAt = null,
-  includeDocumentation = true,
-  includeMsdmd = true
-} = {}) {
+export async function fetchRepositoryPublicProjection(repository, options = {}) {
+  const {
+    headSha = null,
+    defaultBranch = null,
+    headCommittedAt = null,
+    includeDocumentation = true,
+    includeMsdmd = true
+  } = options;
   const repo = normalizeRepository(repository);
-  const resolved = headSha ? { headSha, defaultBranch, headCommittedAt } : await resolveHead(repo);
+  const headWasSupplied = Object.prototype.hasOwnProperty.call(options, 'headSha');
+  if (headWasSupplied && !headSha) throw new Error('repository head unavailable');
+  if (headWasSupplied) await publicRepositoryMetadata(repo);
+  const resolved = headWasSupplied
+    ? { headSha, defaultBranch, headCommittedAt }
+    : await resolveHead(repo);
   if (!resolved.headSha) throw new Error('repository head unavailable');
 
   const [documentation, msdmd] = await Promise.all([