Add daily traffic collection with weekly review cadence

.github/workflows/build-catalog.yml

@@ -66,11 +66,8 @@ jobs:
         working-directory: tools/catalog-build
       - run: npm run build
         working-directory: tools/catalog-build
-      - run: npm run build:stats
+      - run: npm run build:stats -- --render-only
         working-directory: tools/catalog-build
-        env:
-          CLARITY_API_TOKEN: ${{ secrets.CLARITY_API_TOKEN }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Commit generated catalog
         env:
           # Prefer a PAT when configured; the built-in token can be blocked from

tools/catalog-build/build-stats.js

@@ -1,15 +1,18 @@
 import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { replaceClarityDays, unchangedStatsPayload } from './stats-state.js';
 
 const toolDirectory = dirname(fileURLToPath(import.meta.url));
 const repositoryRoot = join(toolDirectory, '..', '..');
 const checkOnly = process.argv.includes('--check');
 const requireClarity = process.argv.includes('--require-clarity');
+const renderOnly = process.argv.includes('--render-only');
 const trafficDirectory = join(repositoryRoot, 'traffic-data');
 const clarityStatePath = join(trafficDirectory, 'clarity-views.json');
 const catalogPath = join(repositoryRoot, 'catalog.json');
 const discussionsPath = join(repositoryRoot, 'resource-discussions.json');
+const statsPath = join(repositoryRoot, 'resource-stats.json');
 const githubToken = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN;
 const clarityToken = process.env.CLARITY_API_TOKEN;
 const dayMilliseconds = 24 * 60 * 60 * 1000;
@@ -199,17 +202,17 @@ function subtractClarityTotals(larger, smaller) {
   return result;
 }
 
-function mapToObject(map) {
-  return Object.fromEntries([...map].sort(([left], [right]) => left.localeCompare(right)));
-}
-
 function setActionOutput(name, value) {
   if (process.env.GITHUB_OUTPUT) {
     appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`);
   }
 }
 
 async function updateClarityState(state, knownSlugs) {
+  if (renderOnly) {
+    console.log('Render-only mode; using persisted Clarity views without collecting.');
+    return { available: false, changed: false };
+  }
   if (!clarityToken) {
     console.log('CLARITY_API_TOKEN is not set; preserving accumulated Clarity views.');
     return { available: false, changed: false };
@@ -218,7 +221,7 @@ async function updateClarityState(state, knownSlugs) {
   try {
     const threeDays = await fetchClarity(3, knownSlugs);
     if (threeDays.dated.size > 0) {
-      for (const [date, bucket] of threeDays.dated) state.days[date] = mapToObject(bucket);
+      replaceClarityDays(state, threeDays.dated, runDate);
     } else {
       // The API normally returns rolling aggregates rather than dates. Cumulative 1/2/3-day
       // windows are differenced so each run persists overlap-safe daily buckets.
@@ -230,11 +233,12 @@ async function updateClarityState(state, knownSlugs) {
         new Date(Date.parse(`${runDate}T00:00:00Z`) - offset * dayMilliseconds)
           .toISOString().slice(0, 10)
       );
-      state.days[dates[0]] = mapToObject(oneDay.totals);
-      state.days[dates[1]] = mapToObject(subtractClarityTotals(twoDays.totals, oneDay.totals));
-      state.days[dates[2]] = mapToObject(subtractClarityTotals(threeDays.totals, twoDays.totals));
+      replaceClarityDays(state, new Map([
+        [dates[0], oneDay.totals],
+        [dates[1], subtractClarityTotals(twoDays.totals, oneDay.totals)],
+        [dates[2], subtractClarityTotals(threeDays.totals, twoDays.totals)]
+      ]), runDate);
     }
-    state.lastRun = runDate;
     return { available: true, changed: true };
   } catch (error) {
     console.warn(`Warning: could not collect Clarity views: ${error.message}`);
@@ -279,9 +283,13 @@ async function githubGraphql(query, variables) {
 
 async function collectDiscussions() {
   const discussions = [];
+  if (renderOnly) {
+    console.log('Render-only mode; using persisted discussion stats without collecting.');
+    return { available: false, discussions };
+  }
   if (!githubToken) {
     console.log('GITHUB_TOKEN or GH_TOKEN is not set; skipping discussion upvotes.');
-    return discussions;
+    return { available: false, discussions };
   }
 
   const query = `query($owner:String!,$name:String!,$after:String) {
@@ -311,9 +319,9 @@ async function collectDiscussions() {
     } while (after);
   } catch (error) {
     console.warn(`Warning: could not collect GitHub Discussions: ${error.message}`);
-    return [];
+    return { available: false, discussions: [] };
   }
-  return discussions;
+  return { available: true, discussions };
 }
 
 function mapDiscussions(discussions, knownSlugs, explicitMap) {
@@ -347,6 +355,20 @@ function mapDiscussions(discussions, knownSlugs, explicitMap) {
   return result;
 }
 
+function previousUpvotes(previousStats, knownSlugs) {
+  const result = new Map();
+  for (const [slug, stats] of Object.entries(previousStats?.resources ?? {})) {
+    if (!knownSlugs.has(slug) || !stats || typeof stats !== 'object') continue;
+    const upvotes = numberFrom(stats.upvotes);
+    if (upvotes === undefined) continue;
+    result.set(slug, {
+      upvotes: Math.round(upvotes),
+      ...(stats.discussion ? { discussion: stats.discussion } : {})
+    });
+  }
+  return result;
+}
+
 const catalog = readJson(catalogPath);
 const knownSlugs = new Set(catalog.resources.map(resource => resource.slug));
 const resourceFolders = catalog.resources
@@ -365,14 +387,17 @@ const discussionConfig = readOptionalJson(discussionsPath, {
   note: 'Maps gallery resource slug -> GitHub Discussion number in the Resource Votes category.',
   map: {}
 });
+const previousStats = readOptionalJson(statsPath, {});
 const clarityState = readOptionalJson(clarityStatePath, { lastRun: '', days: {} });
 if (!clarityState.days || typeof clarityState.days !== 'object') clarityState.days = {};
 
 const clarityResult = await updateClarityState(clarityState, knownSlugs);
 const clarityViews = collectClarityViews(clarityState, knownSlugs);
 const fallbackViews = collectFallbackViews(resourceFolders);
-const discussions = await collectDiscussions();
-const upvoteTotals = mapDiscussions(discussions, knownSlugs, discussionConfig.map ?? {});
+const discussionResult = await collectDiscussions();
+const upvoteTotals = discussionResult.available
+  ? mapDiscussions(discussionResult.discussions, knownSlugs, discussionConfig.map ?? {})
+  : previousUpvotes(previousStats, knownSlugs);
 const resources = {};
 let viewCount = 0;
 let upvoteCount = 0;
@@ -399,18 +424,21 @@ for (const resource of catalog.resources) {
   if (Object.keys(stats).length > 0) resources[resource.slug] = stats;
 }
 
-const output = `${JSON.stringify({
-  generatedAt: new Date().toISOString(),
+const payload = {
   sources: {
     views: 'clarity',
     upvotes: 'github-discussions'
   },
   resources
-}, null, 2)}\n`;
+};
+const generatedAt = unchangedStatsPayload(previousStats, payload) && previousStats.generatedAt
+  ? previousStats.generatedAt
+  : new Date().toISOString();
+const output = `${JSON.stringify({ generatedAt, ...payload }, null, 2)}\n`;
 
 if (!checkOnly) {
   mkdirSync(trafficDirectory, { recursive: true });
-  writeFileSync(join(repositoryRoot, 'resource-stats.json'), output);
+  writeFileSync(statsPath, output);
   if (!existsSync(discussionsPath)) {
     writeFileSync(discussionsPath, `${JSON.stringify(discussionConfig, null, 2)}\n`);
   }

tools/catalog-build/package.json

@@ -5,9 +5,11 @@
   "type": "module",
   "scripts": {
     "build": "node index.js",
-    "check": "node index.js --check && node test-traffic-workflow.js && node test-category-filtering.js",
+    "check": "node index.js --check && node test-traffic-workflow.js && node test-category-filtering.js && node test-stats-state.js && node test-traffic-git-integration.js",
     "check:traffic-workflow": "node test-traffic-workflow.js",
     "check:category-filtering": "node test-category-filtering.js",
+    "check:stats-state": "node test-stats-state.js",
+    "check:traffic-git": "node test-traffic-git-integration.js",
     "build:stats": "node build-stats.js",
     "check:stats": "node build-stats.js --check",
     "create:vote-discussions": "node create-vote-discussions.js"

tools/catalog-build/reconcile-stats-conflict.js

@@ -0,0 +1,26 @@
+import { execFileSync } from 'node:child_process';
+import { writeFileSync } from 'node:fs';
+import { reconcileClarityStates } from './stats-state.js';
+
+const path = process.argv[2];
+if (path !== 'traffic-data/clarity-views.json') {
+  console.error(`Unsupported durable stats conflict: ${path ?? '(missing path)'}`);
+  process.exit(1);
+}
+
+function readStage(stage) {
+  try {
+    return JSON.parse(execFileSync('git', ['show', `:${stage}:${path}`], { encoding: 'utf8' }));
+  } catch (error) {
+    throw new Error(`Could not read merge stage ${stage} for ${path}: ${error.message}`);
+  }
+}
+
+try {
+  const { state, relation } = reconcileClarityStates(readStage(2), readStage(3));
+  writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`);
+  console.log(`Reconciled ${path} using ${relation}; overlapping daily resource values were identical.`);
+} catch (error) {
+  console.error(`Refusing to auto-resolve ${path}: ${error.message}`);
+  process.exit(1);
+}

tools/catalog-build/stats-state.js

@@ -0,0 +1,122 @@
+export function replaceClarityDays(state, buckets, lastRun) {
+  for (const [date, metrics] of buckets) {
+    state.days[date] = Object.fromEntries(
+      [...metrics].sort(([left], [right]) => left.localeCompare(right))
+    );
+  }
+  state.lastRun = lastRun;
+}
+
+function isRecord(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function sortedRecord(entries) {
+  return Object.fromEntries([...entries].sort(([left], [right]) => left.localeCompare(right)));
+}
+
+function validateClarityState(state, label) {
+  if (!isRecord(state) || !isRecord(state.days)) {
+    throw new Error(`${label} Clarity state must contain a days object.`);
+  }
+  if (typeof state.lastRun !== 'string' ||
+      (state.lastRun !== '' && !/^\d{4}-\d{2}-\d{2}$/.test(state.lastRun))) {
+    throw new Error(`${label} Clarity state has an invalid lastRun.`);
+  }
+  for (const [date, resources] of Object.entries(state.days)) {
+    if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || !isRecord(resources)) {
+      throw new Error(`${label} Clarity state has an invalid day bucket: ${date}.`);
+    }
+    for (const [slug, metrics] of Object.entries(resources)) {
+      if (!slug || !isRecord(metrics)) {
+        throw new Error(`${label} Clarity state has invalid metrics for ${date}/${slug}.`);
+      }
+      for (const [metric, value] of Object.entries(metrics)) {
+        if (!['views', 'uniques'].includes(metric) ||
+            !Number.isFinite(value) || value < 0 || !Number.isInteger(value)) {
+          throw new Error(`${label} Clarity state has invalid ${metric} for ${date}/${slug}.`);
+        }
+      }
+    }
+  }
+}
+
+function equalJson(left, right) {
+  return JSON.stringify(left) === JSON.stringify(right);
+}
+
+export function clarityStateContains(container, candidate) {
+  validateClarityState(container, 'Container');
+  validateClarityState(candidate, 'Candidate');
+  return Object.entries(candidate.days).every(([date, resources]) =>
+    Object.entries(resources).every(([slug, metrics]) =>
+      equalJson(container.days[date]?.[slug], metrics)
+    )
+  );
+}
+
+export function reconcileClarityStates(ours, theirs) {
+  validateClarityState(ours, 'Ours');
+  validateClarityState(theirs, 'Theirs');
+
+  const oursContainsTheirs = clarityStateContains(ours, theirs);
+  const theirsContainsOurs = clarityStateContains(theirs, ours);
+  const newerState = ours.lastRun === theirs.lastRun
+    ? null
+    : (ours.lastRun > theirs.lastRun ? ours : theirs);
+  const days = {};
+
+  for (const date of [...new Set([...Object.keys(ours.days), ...Object.keys(theirs.days)])].sort()) {
+    const resources = new Map();
+    for (const [state, source] of [
+      [ours, ours.days[date] ?? {}],
+      [theirs, theirs.days[date] ?? {}]
+    ]) {
+      for (const [slug, metrics] of Object.entries(source)) {
+        const existing = resources.get(slug);
+        if (existing && !equalJson(existing, metrics)) {
+          if (!newerState) {
+            resources.set(slug, sortedRecord(new Map(
+              [...new Set([...Object.keys(existing), ...Object.keys(metrics)])]
+                .map(metric => [metric, Math.max(existing[metric] ?? 0, metrics[metric] ?? 0)])
+            )));
+            continue;
+          }
+          if (state !== newerState) continue;
+        }
+        resources.set(slug, metrics);
+      }
+    }
+    days[date] = sortedRecord(resources);
+  }
+
+  const extraKeys = new Set([
+    ...Object.keys(ours).filter(key => !['lastRun', 'days'].includes(key)),
+    ...Object.keys(theirs).filter(key => !['lastRun', 'days'].includes(key))
+  ]);
+  const extras = {};
+  for (const key of extraKeys) {
+    if (Object.hasOwn(ours, key) && Object.hasOwn(theirs, key) &&
+        !equalJson(ours[key], theirs[key])) {
+      throw new Error(`Clarity state has competing top-level ${key} values.`);
+    }
+    extras[key] = Object.hasOwn(theirs, key) ? theirs[key] : ours[key];
+  }
+
+  return {
+    state: {
+      ...sortedRecord(Object.entries(extras)),
+      lastRun: [ours.lastRun, theirs.lastRun].sort().at(-1),
+      days
+    },
+    relation: oursContainsTheirs
+      ? (theirsContainsOurs ? 'equal' : 'ours-superset')
+      : (theirsContainsOurs ? 'theirs-superset' : 'structured-union')
+  };
+}
+
+export function unchangedStatsPayload(previous, next) {
+  if (!previous || typeof previous !== 'object') return false;
+  return JSON.stringify(previous.sources) === JSON.stringify(next.sources) &&
+    JSON.stringify(previous.resources) === JSON.stringify(next.resources);
+}