Add bad-scan reports for the nutrition label scanner

apps/api/package.json

@@ -8,7 +8,8 @@
     "test": "tsx --test src/**/*.test.ts",
     "db:generate": "prisma generate",
     "db:migrate": "prisma migrate dev",
-    "make-admin": "tsx src/make-admin.ts"
+    "make-admin": "tsx src/make-admin.ts",
+    "export-scan-reports": "tsx src/export-scan-reports.ts"
   },
   "dependencies": {
     "@prisma/adapter-mariadb": "^7.0.0",

apps/api/prisma/migrations/20261110120000_scan_reports/migration.sql

@@ -0,0 +1,18 @@
+-- CreateTable
+CREATE TABLE `ScanReport` (
+    `id` VARCHAR(191) NOT NULL,
+    `userId` VARCHAR(191) NOT NULL,
+    `photo` LONGBLOB NOT NULL,
+    `rawText` TEXT NOT NULL,
+    `parsed` JSON NOT NULL,
+    `note` VARCHAR(500) NOT NULL DEFAULT '',
+    `language` VARCHAR(8) NOT NULL,
+    `resolvedAt` DATETIME(3) NULL,
+    `createdAt` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+
+    INDEX `ScanReport_resolvedAt_createdAt_idx`(`resolvedAt`, `createdAt`),
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+-- AddForeignKey
+ALTER TABLE `ScanReport` ADD CONSTRAINT `ScanReport_userId_fkey` FOREIGN KEY (`userId`) REFERENCES `User`(`id`) ON DELETE CASCADE ON UPDATE CASCADE;

apps/api/prisma/schema.prisma

@@ -87,6 +87,7 @@ model User {
   fastingSessions FastingSession[]
   healthSettings  HealthSettings?
   healthDays      HealthDay[]
+  scanReports     ScanReport[]
 }
 
 model Session {
@@ -243,3 +244,20 @@ model FastingSession {
 
   @@index([userId, startedAt])
 }
+
+// A label scan the user flagged as wrong. Kept so real failures can become
+// parser tests (see `npm run export-scan-reports`).
+model ScanReport {
+  id         String    @id @default(cuid())
+  userId     String
+  user       User      @relation(fields: [userId], references: [id], onDelete: Cascade)
+  photo      Bytes
+  rawText    String    @db.Text
+  parsed     Json
+  note       String    @default("") @db.VarChar(500)
+  language   String    @db.VarChar(8)
+  resolvedAt DateTime?
+  createdAt  DateTime  @default(now())
+
+  @@index([resolvedAt, createdAt])
+}

apps/api/src/app.ts

@@ -9,6 +9,7 @@ import { fastingRouter } from "./routes/fasting.js";
 import { foodsRouter } from "./routes/foods.js";
 import { profileRouter } from "./routes/profile.js";
 import { rbacRouter } from "./routes/rbac.js";
+import { scanReportsRouter } from "./routes/scan-reports.js";
 import { weeksRouter } from "./routes/weeks.js";
 import { weightsRouter } from "./routes/weights.js";
 
@@ -29,8 +30,10 @@ export function createApp() {
       credentials: true,
     }),
   );
-  app.use(express.json({ limit: "1mb" }));
   app.use(cookieParser());
+  // Mounted before the global JSON parser: it takes photos, so it has its own limit.
+  app.use("/api/scan-reports", scanReportsRouter);
+  app.use(express.json({ limit: "1mb" }));
   app.get("/api/health", (_req, res) => {
     res.json({ ok: true });
   });

apps/api/src/export-scan-reports.ts

@@ -0,0 +1,31 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { prisma } from "./db.js";
+
+// Writes every open scan report to a folder (photo, recognized text, parsed
+// values) so it can be copied off the server and turned into a parser test.
+const dir = process.argv[2];
+if (!dir) {
+  console.error("usage: npm run export-scan-reports -- <folder>");
+  process.exit(1);
+}
+await mkdir(dir, { recursive: true });
+const reports = await prisma.scanReport.findMany({
+  where: { resolvedAt: null },
+  orderBy: { createdAt: "asc" },
+});
+for (const report of reports) {
+  const base = join(dir, report.id);
+  await writeFile(`${base}.jpg`, Buffer.from(report.photo));
+  await writeFile(`${base}.txt`, report.rawText);
+  await writeFile(
+    `${base}.json`,
+    JSON.stringify(
+      { note: report.note, language: report.language, parsed: report.parsed, createdAt: report.createdAt },
+      null,
+      2,
+    ),
+  );
+}
+console.log(`exported ${reports.length} report(s) to ${dir}`);
+await prisma.$disconnect();

apps/api/src/routes/scan-reports.ts

@@ -0,0 +1,93 @@
+import express, { Router } from "express";
+import { currentUser, requireUser } from "../auth.js";
+import { prisma } from "../db.js";
+import { asyncRoute, HttpError } from "../http.js";
+import { requireAdmin } from "../rbac.js";
+import { MAX_SCAN_PHOTO_BYTES, scanReportSchema } from "../schemas.js";
+
+export const scanReportsRouter = Router();
+scanReportsRouter.use(requireUser);
+
+// Photos are far bigger than the 1 MB the rest of the API accepts.
+const bigJson = express.json({ limit: "7mb" });
+
+const SELECT = {
+  id: true,
+  userId: true,
+  rawText: true,
+  parsed: true,
+  note: true,
+  language: true,
+  resolvedAt: true,
+  createdAt: true,
+} as const;
+
+// Any signed-in user can report a bad scan.
+scanReportsRouter.post(
+  "/",
+  bigJson,
+  asyncRoute(async (req, res) => {
+    const user = currentUser(res);
+    const body = scanReportSchema.parse(req.body);
+    const photo = Buffer.from(body.photo, "base64");
+    if (photo.length === 0 || photo.length > MAX_SCAN_PHOTO_BYTES) {
+      throw new HttpError(400, "invalid_body");
+    }
+    const report = await prisma.scanReport.create({
+      data: {
+        userId: user.id,
+        photo,
+        rawText: body.rawText,
+        parsed: body.parsed as object,
+        note: body.note,
+        language: body.language,
+      },
+      select: { id: true },
+    });
+    res.status(201).json({ id: report.id });
+  }),
+);
+
+scanReportsRouter.get(
+  "/",
+  requireAdmin,
+  asyncRoute(async (req, res) => {
+    const open = req.query.status !== "resolved";
+    const reports = await prisma.scanReport.findMany({
+      where: { resolvedAt: open ? null : { not: null } },
+      orderBy: { createdAt: "desc" },
+      take: 100,
+      select: SELECT,
+    });
+    res.json({ reports });
+  }),
+);
+
+scanReportsRouter.get(
+  "/:id/photo",
+  requireAdmin,
+  asyncRoute(async (req, res) => {
+    const report = await prisma.scanReport.findUnique({
+      where: { id: String(req.params.id) },
+      select: { photo: true },
+    });
+    if (!report) throw new HttpError(404, "not_found");
+    res.type("image/jpeg").send(Buffer.from(report.photo));
+  }),
+);
+
+scanReportsRouter.post(
+  "/:id/resolve",
+  requireAdmin,
+  asyncRoute(async (req, res) => {
+    const id = String(req.params.id);
+    const found = await prisma.scanReport.findUnique({ where: { id }, select: { id: true } });
+    if (!found) throw new HttpError(404, "not_found");
+    const report = await prisma.scanReport.update({
+      where: { id },
+      data: { resolvedAt: new Date() },
+      select: SELECT,
+    });
+    res.json({ report });
+  }),
+);

apps/api/src/schemas.ts

@@ -160,3 +160,14 @@ export const catalogQuerySchema = z.object({
   q: z.string().trim().max(64).default(""),
   offset: z.coerce.number().int().min(0).max(100000).default(0),
 });
+
+// A photo is sent as base64 (about 4/3 of its size); the app downsizes it first.
+export const MAX_SCAN_PHOTO_BYTES = 4 * 1024 * 1024;
+
+export const scanReportSchema = z.object({
+  photo: z.string().min(1).max(Math.ceil((MAX_SCAN_PHOTO_BYTES * 4) / 3) + 8),
+  rawText: z.string().max(20000),
+  parsed: z.record(z.unknown()),
+  note: z.string().trim().max(500).default(""),
+  language: z.string().max(8),
+});

apps/mobile/lib/copy.dart

@@ -146,6 +146,17 @@ class Copy {
     'The camera is unavailable. Check camera access in Settings.',
     'המצלמה אינה זמינה. בדקו את הרשאת המצלמה בהגדרות.',
   );
+  String get reportScan => pick('Report a bad scan', 'דיווח על סריקה שגויה');
+  String get reportScanHint => pick(
+    'Sends the photo and the recognized text so the scanner can be improved.',
+    'שולח את התמונה ואת הטקסט שזוהה כדי לשפר את הסורק.',
+  );
+  String get reportScanNote =>
+      pick('What was wrong? (optional)', 'מה הייתה הטעות? (לא חובה)');
+  String get reportScanSend => pick('Send', 'שליחה');
+  String get reportScanThanks => pick('Thanks, report sent', 'תודה, הדיווח נשלח');
+  String get reportScanFailed =>
+      pick('Could not send the report', 'שליחת הדיווח נכשלה');
   String get showScannedText => pick('Show recognized text', 'הצגת הטקסט שזוהה');
   String get close => pick('Close', 'סגירה');
   String get scannedCheck => pick('Scanned – please check', 'נסרק – נא לבדוק');

apps/mobile/lib/main.dart

@@ -4058,11 +4058,14 @@ class _FoodScreenState extends State<FoodScreen> {
   final scanned = <TextEditingController>{};
   bool scannedFromServing = false;
   String scannedText = '';
+  NutritionScan? lastScan;
 
   Future<void> scanLabel() async {
     final scan = await Navigator.push<NutritionScan>(
       context,
-      MaterialPageRoute(builder: (_) => NutritionScanScreen(t: widget.t)),
+      MaterialPageRoute(
+        builder: (_) => NutritionScanScreen(api: widget.api, t: widget.t),
+      ),
     );
     if (scan == null) return;
     String text(num? v) => v == null ? '' : _plainNumber(v);
@@ -4084,6 +4087,7 @@ class _FoodScreenState extends State<FoodScreen> {
       basis = scan.liquid ? 'liquid' : 'solid';
       scannedFromServing = scan.fromServing;
       scannedText = scan.raw;
+      lastScan = scan;
     });
   }
 
@@ -4139,6 +4143,14 @@ class _FoodScreenState extends State<FoodScreen> {
                 child: Text(t.showScannedText),
               ),
             ),
+          if (lastScan != null)
+            Align(
+              alignment: AlignmentDirectional.centerStart,
+              child: TextButton(
+                onPressed: () => reportBadScan(context, widget.api, t, lastScan!),
+                child: Text(t.reportScan),
+              ),
+            ),
           if (scannedFromServing)
             Text(
               t.scannedFromServing,

apps/mobile/lib/nutrition_parser.dart

@@ -20,6 +20,7 @@ class NutritionScan {
     this.liquid = false,
     this.fromServing = false,
     this.raw = '',
+    this.photo,
   });
 
   final num? calories;
@@ -41,6 +42,9 @@ class NutritionScan {
   /// The OCR text this was parsed from (for the "show text" debug view).
   final String raw;
 
+  /// The (upright) photo this was read from, kept so a bad scan can be reported.
+  final String? photo;
+
   int get found => [
     calories,
     protein,
@@ -54,7 +58,11 @@ class NutritionScan {
   /// Enough to be worth prefilling the form with.
   bool get usable => found >= 2;
 
-  NutritionScan withRaw(String text) => NutritionScan(
+  NutritionScan withRaw(String text) => _copy(raw: text);
+
+  NutritionScan withPhoto(String path) => _copy(photo: path);
+
+  NutritionScan _copy({String? raw, String? photo}) => NutritionScan(
     calories: calories,
     protein: protein,
     carbohydrates: carbohydrates,
@@ -64,8 +72,22 @@ class NutritionScan {
     sodium: sodium,
     liquid: liquid,
     fromServing: fromServing,
-    raw: text,
+    raw: raw ?? this.raw,
+    photo: photo ?? this.photo,
   );
+
+  /// The parsed values, for a scan report.
+  Map<String, Object?> toJson() => {
+    'calories': calories,
+    'protein': protein,
+    'carbohydrates': carbohydrates,
+    'fat': fat,
+    'fiber': fiber,
+    'sugars': sugars,
+    'sodium': sodium,
+    'liquid': liquid,
+    'fromServing': fromServing,
+  };
 }
 
 enum _Field { calories, protein, carbs, fat, fiber, sugars, sodium, salt }