From 4f330790680b416b387eb088e6801066fd767f0f Mon Sep 17 00:00:00 2001 From: Yarin Levi <41652571+YarinQuapi@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:03:19 +0200 Subject: [PATCH 1/2] Add ideas backlog Co-Authored-By: Claude Sonnet 5.5 --- docs/ideas.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/ideas.md diff --git a/docs/ideas.md b/docs/ideas.md new file mode 100644 index 0000000..566bf33 --- /dev/null +++ b/docs/ideas.md @@ -0,0 +1,29 @@ +# Ideas backlog + +Ideas for improving Drop It, to revisit. Roughly in priority order. + +1. **Scanner feedback loop** (in progress, branch `scanner-feedback-loop`) + A "report a bad scan" button that saves the photo and the recognized text. + Real failures become parser tests (like the Pringles one) and a test set + builds up over time. Liquid / per-100 ml labels still fail. + +2. **Auto-fill foods from a barcode database** + Look the barcode up in Open Food Facts (good Israeli coverage) before asking + for a label photo. The label scanner becomes the fallback. Also seeds the + catalog. + +3. **Review step for scanned values** + Show the photo beside the parsed fields and highlight low-confidence values + (repaired decimals, Atwater mismatch) instead of a generic "please check". + +4. **Faster logging** + Recent and frequent foods at the top of add-meal, "copy yesterday's meals", + saved meals / combos. + +5. **Home screen widget and reminders** + Widget with today's calories and the fasting timer; optional notification + when a fast ends. + +6. **Catalog trust signals** + Prominent verified badges, a way for users to flag wrong values, per-barcode + change history, and an admin queue of flagged entries. From 259a91fbfde6b0fb5a0949a52388541bbf483830 Mon Sep 17 00:00:00 2001 From: Yarin Levi <41652571+YarinQuapi@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:09:13 +0200 Subject: [PATCH 2/2] Add bad-scan reports: upload photo and OCR text to the API Users can report a wrong nutrition-label scan from the food form or the scan-failed screen. The app sends a downsized JPEG, the recognized text and the parsed values to POST /api/scan-reports. Admins can list, view photos and resolve reports, and npm run export-scan-reports writes them to a folder so real failures can become parser tests. Co-Authored-By: Claude Sonnet 5.5 --- apps/api/package.json | 3 +- .../20261110120000_scan_reports/migration.sql | 18 ++++ apps/api/prisma/schema.prisma | 18 ++++ apps/api/src/app.ts | 5 +- apps/api/src/export-scan-reports.ts | 31 ++++++ apps/api/src/routes/scan-reports.ts | 93 ++++++++++++++++ apps/api/src/schemas.ts | 11 ++ apps/mobile/lib/copy.dart | 11 ++ apps/mobile/lib/main.dart | 14 ++- apps/mobile/lib/nutrition_parser.dart | 26 ++++- apps/mobile/lib/nutrition_scan.dart | 101 +++++++++++++++++- apps/mobile/test/nutrition_parser_test.dart | 11 ++ 12 files changed, 334 insertions(+), 8 deletions(-) create mode 100644 apps/api/prisma/migrations/20261110120000_scan_reports/migration.sql create mode 100644 apps/api/src/export-scan-reports.ts create mode 100644 apps/api/src/routes/scan-reports.ts diff --git a/apps/api/package.json b/apps/api/package.json index 3a728ce..8f34b1c 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -8,7 +8,8 @@ "test": "tsx --test src/**/*.test.ts", "db:generate": "prisma generate", "db:migrate": "prisma migrate dev", - "make-admin": "tsx src/make-admin.ts" + "make-admin": "tsx src/make-admin.ts", + "export-scan-reports": "tsx src/export-scan-reports.ts" }, "dependencies": { "@prisma/adapter-mariadb": "^7.0.0", diff --git a/apps/api/prisma/migrations/20261110120000_scan_reports/migration.sql b/apps/api/prisma/migrations/20261110120000_scan_reports/migration.sql new file mode 100644 index 0000000..251c570 --- /dev/null +++ b/apps/api/prisma/migrations/20261110120000_scan_reports/migration.sql @@ -0,0 +1,18 @@ +-- CreateTable +CREATE TABLE `ScanReport` ( + `id` VARCHAR(191) NOT NULL, + `userId` VARCHAR(191) NOT NULL, + `photo` LONGBLOB NOT NULL, + `rawText` TEXT NOT NULL, + `parsed` JSON NOT NULL, + `note` VARCHAR(500) NOT NULL DEFAULT '', + `language` VARCHAR(8) NOT NULL, + `resolvedAt` DATETIME(3) NULL, + `createdAt` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + + INDEX `ScanReport_resolvedAt_createdAt_idx`(`resolvedAt`, `createdAt`), + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +-- AddForeignKey +ALTER TABLE `ScanReport` ADD CONSTRAINT `ScanReport_userId_fkey` FOREIGN KEY (`userId`) REFERENCES `User`(`id`) ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 0f0966e..e24472b 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -87,6 +87,7 @@ model User { fastingSessions FastingSession[] healthSettings HealthSettings? healthDays HealthDay[] + scanReports ScanReport[] } model Session { @@ -243,3 +244,20 @@ model FastingSession { @@index([userId, startedAt]) } + +// A label scan the user flagged as wrong. Kept so real failures can become +// parser tests (see `npm run export-scan-reports`). +model ScanReport { + id String @id @default(cuid()) + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + photo Bytes + rawText String @db.Text + parsed Json + note String @default("") @db.VarChar(500) + language String @db.VarChar(8) + resolvedAt DateTime? + createdAt DateTime @default(now()) + + @@index([resolvedAt, createdAt]) +} diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index dbe093e..b04f064 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -9,6 +9,7 @@ import { fastingRouter } from "./routes/fasting.js"; import { foodsRouter } from "./routes/foods.js"; import { profileRouter } from "./routes/profile.js"; import { rbacRouter } from "./routes/rbac.js"; +import { scanReportsRouter } from "./routes/scan-reports.js"; import { weeksRouter } from "./routes/weeks.js"; import { weightsRouter } from "./routes/weights.js"; @@ -29,8 +30,10 @@ export function createApp() { credentials: true, }), ); - app.use(express.json({ limit: "1mb" })); app.use(cookieParser()); + // Mounted before the global JSON parser: it takes photos, so it has its own limit. + app.use("/api/scan-reports", scanReportsRouter); + app.use(express.json({ limit: "1mb" })); app.get("/api/health", (_req, res) => { res.json({ ok: true }); }); diff --git a/apps/api/src/export-scan-reports.ts b/apps/api/src/export-scan-reports.ts new file mode 100644 index 0000000..da87db7 --- /dev/null +++ b/apps/api/src/export-scan-reports.ts @@ -0,0 +1,31 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { prisma } from "./db.js"; + +// Writes every open scan report to a folder (photo, recognized text, parsed +// values) so it can be copied off the server and turned into a parser test. +const dir = process.argv[2]; +if (!dir) { + console.error("usage: npm run export-scan-reports -- "); + process.exit(1); +} +await mkdir(dir, { recursive: true }); +const reports = await prisma.scanReport.findMany({ + where: { resolvedAt: null }, + orderBy: { createdAt: "asc" }, +}); +for (const report of reports) { + const base = join(dir, report.id); + await writeFile(`${base}.jpg`, Buffer.from(report.photo)); + await writeFile(`${base}.txt`, report.rawText); + await writeFile( + `${base}.json`, + JSON.stringify( + { note: report.note, language: report.language, parsed: report.parsed, createdAt: report.createdAt }, + null, + 2, + ), + ); +} +console.log(`exported ${reports.length} report(s) to ${dir}`); +await prisma.$disconnect(); diff --git a/apps/api/src/routes/scan-reports.ts b/apps/api/src/routes/scan-reports.ts new file mode 100644 index 0000000..47433c8 --- /dev/null +++ b/apps/api/src/routes/scan-reports.ts @@ -0,0 +1,93 @@ +import express, { Router } from "express"; +import { currentUser, requireUser } from "../auth.js"; +import { prisma } from "../db.js"; +import { asyncRoute, HttpError } from "../http.js"; +import { requireAdmin } from "../rbac.js"; +import { MAX_SCAN_PHOTO_BYTES, scanReportSchema } from "../schemas.js"; + +export const scanReportsRouter = Router(); +scanReportsRouter.use(requireUser); + +// Photos are far bigger than the 1 MB the rest of the API accepts. +const bigJson = express.json({ limit: "7mb" }); + +const SELECT = { + id: true, + userId: true, + rawText: true, + parsed: true, + note: true, + language: true, + resolvedAt: true, + createdAt: true, +} as const; + +// Any signed-in user can report a bad scan. +scanReportsRouter.post( + "/", + bigJson, + asyncRoute(async (req, res) => { + const user = currentUser(res); + const body = scanReportSchema.parse(req.body); + const photo = Buffer.from(body.photo, "base64"); + if (photo.length === 0 || photo.length > MAX_SCAN_PHOTO_BYTES) { + throw new HttpError(400, "invalid_body"); + } + const report = await prisma.scanReport.create({ + data: { + userId: user.id, + photo, + rawText: body.rawText, + parsed: body.parsed as object, + note: body.note, + language: body.language, + }, + select: { id: true }, + }); + res.status(201).json({ id: report.id }); + }), +); + +scanReportsRouter.get( + "/", + requireAdmin, + asyncRoute(async (req, res) => { + const open = req.query.status !== "resolved"; + const reports = await prisma.scanReport.findMany({ + where: { resolvedAt: open ? null : { not: null } }, + orderBy: { createdAt: "desc" }, + take: 100, + select: SELECT, + }); + res.json({ reports }); + }), +); + +scanReportsRouter.get( + "/:id/photo", + requireAdmin, + asyncRoute(async (req, res) => { + const report = await prisma.scanReport.findUnique({ + where: { id: String(req.params.id) }, + select: { photo: true }, + }); + if (!report) throw new HttpError(404, "not_found"); + res.type("image/jpeg").send(Buffer.from(report.photo)); + }), +); + +scanReportsRouter.post( + "/:id/resolve", + requireAdmin, + asyncRoute(async (req, res) => { + const id = String(req.params.id); + const found = await prisma.scanReport.findUnique({ where: { id }, select: { id: true } }); + if (!found) throw new HttpError(404, "not_found"); + const report = await prisma.scanReport.update({ + where: { id }, + data: { resolvedAt: new Date() }, + select: SELECT, + }); + res.json({ report }); + }), +); diff --git a/apps/api/src/schemas.ts b/apps/api/src/schemas.ts index dd6068a..1fe15e1 100644 --- a/apps/api/src/schemas.ts +++ b/apps/api/src/schemas.ts @@ -160,3 +160,14 @@ export const catalogQuerySchema = z.object({ q: z.string().trim().max(64).default(""), offset: z.coerce.number().int().min(0).max(100000).default(0), }); + +// A photo is sent as base64 (about 4/3 of its size); the app downsizes it first. +export const MAX_SCAN_PHOTO_BYTES = 4 * 1024 * 1024; + +export const scanReportSchema = z.object({ + photo: z.string().min(1).max(Math.ceil((MAX_SCAN_PHOTO_BYTES * 4) / 3) + 8), + rawText: z.string().max(20000), + parsed: z.record(z.unknown()), + note: z.string().trim().max(500).default(""), + language: z.string().max(8), +}); diff --git a/apps/mobile/lib/copy.dart b/apps/mobile/lib/copy.dart index 2f1f7cf..ff04523 100644 --- a/apps/mobile/lib/copy.dart +++ b/apps/mobile/lib/copy.dart @@ -146,6 +146,17 @@ class Copy { 'The camera is unavailable. Check camera access in Settings.', 'המצלמה אינה זמינה. בדקו את הרשאת המצלמה בהגדרות.', ); + String get reportScan => pick('Report a bad scan', 'דיווח על סריקה שגויה'); + String get reportScanHint => pick( + 'Sends the photo and the recognized text so the scanner can be improved.', + 'שולח את התמונה ואת הטקסט שזוהה כדי לשפר את הסורק.', + ); + String get reportScanNote => + pick('What was wrong? (optional)', 'מה הייתה הטעות? (לא חובה)'); + String get reportScanSend => pick('Send', 'שליחה'); + String get reportScanThanks => pick('Thanks, report sent', 'תודה, הדיווח נשלח'); + String get reportScanFailed => + pick('Could not send the report', 'שליחת הדיווח נכשלה'); String get showScannedText => pick('Show recognized text', 'הצגת הטקסט שזוהה'); String get close => pick('Close', 'סגירה'); String get scannedCheck => pick('Scanned – please check', 'נסרק – נא לבדוק'); diff --git a/apps/mobile/lib/main.dart b/apps/mobile/lib/main.dart index e75772f..abe68cd 100644 --- a/apps/mobile/lib/main.dart +++ b/apps/mobile/lib/main.dart @@ -4058,11 +4058,14 @@ class _FoodScreenState extends State { final scanned = {}; bool scannedFromServing = false; String scannedText = ''; + NutritionScan? lastScan; Future scanLabel() async { final scan = await Navigator.push( context, - MaterialPageRoute(builder: (_) => NutritionScanScreen(t: widget.t)), + MaterialPageRoute( + builder: (_) => NutritionScanScreen(api: widget.api, t: widget.t), + ), ); if (scan == null) return; String text(num? v) => v == null ? '' : _plainNumber(v); @@ -4084,6 +4087,7 @@ class _FoodScreenState extends State { basis = scan.liquid ? 'liquid' : 'solid'; scannedFromServing = scan.fromServing; scannedText = scan.raw; + lastScan = scan; }); } @@ -4139,6 +4143,14 @@ class _FoodScreenState extends State { child: Text(t.showScannedText), ), ), + if (lastScan != null) + Align( + alignment: AlignmentDirectional.centerStart, + child: TextButton( + onPressed: () => reportBadScan(context, widget.api, t, lastScan!), + child: Text(t.reportScan), + ), + ), if (scannedFromServing) Text( t.scannedFromServing, diff --git a/apps/mobile/lib/nutrition_parser.dart b/apps/mobile/lib/nutrition_parser.dart index 56a3092..216c2df 100644 --- a/apps/mobile/lib/nutrition_parser.dart +++ b/apps/mobile/lib/nutrition_parser.dart @@ -20,6 +20,7 @@ class NutritionScan { this.liquid = false, this.fromServing = false, this.raw = '', + this.photo, }); final num? calories; @@ -41,6 +42,9 @@ class NutritionScan { /// The OCR text this was parsed from (for the "show text" debug view). final String raw; + /// The (upright) photo this was read from, kept so a bad scan can be reported. + final String? photo; + int get found => [ calories, protein, @@ -54,7 +58,11 @@ class NutritionScan { /// Enough to be worth prefilling the form with. bool get usable => found >= 2; - NutritionScan withRaw(String text) => NutritionScan( + NutritionScan withRaw(String text) => _copy(raw: text); + + NutritionScan withPhoto(String path) => _copy(photo: path); + + NutritionScan _copy({String? raw, String? photo}) => NutritionScan( calories: calories, protein: protein, carbohydrates: carbohydrates, @@ -64,8 +72,22 @@ class NutritionScan { sodium: sodium, liquid: liquid, fromServing: fromServing, - raw: text, + raw: raw ?? this.raw, + photo: photo ?? this.photo, ); + + /// The parsed values, for a scan report. + Map toJson() => { + 'calories': calories, + 'protein': protein, + 'carbohydrates': carbohydrates, + 'fat': fat, + 'fiber': fiber, + 'sugars': sugars, + 'sodium': sodium, + 'liquid': liquid, + 'fromServing': fromServing, + }; } enum _Field { calories, protein, carbs, fat, fiber, sugars, sodium, salt } diff --git a/apps/mobile/lib/nutrition_scan.dart b/apps/mobile/lib/nutrition_scan.dart index f3a4e75..7672b78 100644 --- a/apps/mobile/lib/nutrition_scan.dart +++ b/apps/mobile/lib/nutrition_scan.dart @@ -1,3 +1,4 @@ +import 'dart:convert'; import 'dart:io'; import 'package:flutter/foundation.dart'; @@ -7,6 +8,7 @@ import 'package:image/image.dart' as img; import 'package:image_picker/image_picker.dart'; import 'package:path_provider/path_provider.dart'; +import 'api.dart'; import 'copy.dart'; import 'nutrition_parser.dart'; import 'theme.dart'; @@ -102,13 +104,96 @@ Future scanNutritionImage( if (best == null || scan.found > best.found) best = scan; if (best.found >= 5) break; } - return best!; + return best!.withPhoto(full); +} + +/// A report photo: JPEG, at most 1600 px on the long side, as base64. Phone +/// photos are several MB; this keeps an upload to a few hundred KB. +Future _reportPhoto(String path) => compute(_encodeReportPhoto, path); + +String _encodeReportPhoto(String path) { + final decoded = img.decodeImage(File(path).readAsBytesSync()); + if (decoded == null) throw const FormatException('bad image'); + return base64Encode(img.encodeJpg(_fit(decoded, 1600), quality: 80)); +} + +/// Sends a scan the user flagged as wrong, with its photo, to the server. +Future sendScanReport( + Api api, + NutritionScan scan, + String note, + String language, +) async { + final path = scan.photo; + if (path == null) throw StateError('no photo'); + await api.send( + 'POST', + '/api/scan-reports', + body: { + 'photo': await _reportPhoto(path), + 'rawText': scan.raw.length > 20000 ? scan.raw.substring(0, 20000) : scan.raw, + 'parsed': scan.toJson(), + 'note': note.trim(), + 'language': language, + }, + ); +} + +/// Asks for an optional note and uploads [scan] as a bad-scan report. +Future reportBadScan( + BuildContext context, + Api api, + Copy t, + NutritionScan scan, +) async { + final note = TextEditingController(); + final send = await showDialog( + context: context, + builder: (_) => AlertDialog( + title: Text(t.reportScan), + content: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text(t.reportScanHint), + const SizedBox(height: Gaps.md), + TextField( + controller: note, + maxLength: 500, + maxLines: 3, + decoration: InputDecoration(hintText: t.reportScanNote), + ), + ], + ), + actions: [ + TextButton( + onPressed: () => Navigator.pop(context, false), + child: Text(t.cancel), + ), + TextButton( + onPressed: () => Navigator.pop(context, true), + child: Text(t.reportScanSend), + ), + ], + ), + ); + final text = note.text; + note.dispose(); + if (send != true || !context.mounted) return; + final messenger = ScaffoldMessenger.of(context); + try { + await sendScanReport(api, scan, text, t.lang); + messenger.showSnackBar(SnackBar(content: Text(t.reportScanThanks))); + } catch (_) { + messenger.showSnackBar(SnackBar(content: Text(t.reportScanFailed))); + } } /// Asks for a photo of a nutrition table, reads it, and pops a /// [NutritionScan] (or nothing if the user backs out). class NutritionScanScreen extends StatefulWidget { - const NutritionScanScreen({super.key, required this.t}); + const NutritionScanScreen({super.key, required this.api, required this.t}); + final Api api; final Copy t; @override @@ -121,6 +206,7 @@ class _NutritionScanScreenState extends State { String error = ''; String? photo; String rawText = ''; + NutritionScan? failed; @override void initState() { @@ -129,7 +215,10 @@ class _NutritionScanScreenState extends State { } Future pick(ImageSource source) async { - setState(() => error = ''); + setState(() { + error = ''; + failed = null; + }); XFile? file; try { file = await picker.pickImage(source: source, imageQuality: 95); @@ -155,6 +244,7 @@ class _NutritionScanScreenState extends State { setState(() { busy = false; rawText = scan.raw; + failed = scan; error = widget.t.scanNothingFound; }); } catch (_) { @@ -202,6 +292,11 @@ class _NutritionScanScreenState extends State { style: TextStyle(fontSize: 12, color: apple.labelSecondary), textDirection: TextDirection.ltr, ), + if (!busy && failed != null) + TextButton( + onPressed: () => reportBadScan(context, widget.api, t, failed!), + child: Text(t.reportScan), + ), if (!busy) ...[ SizedBox( height: Gaps.hit, diff --git a/apps/mobile/test/nutrition_parser_test.dart b/apps/mobile/test/nutrition_parser_test.dart index ac69d9d..66dcaa2 100644 --- a/apps/mobile/test/nutrition_parser_test.dart +++ b/apps/mobile/test/nutrition_parser_test.dart @@ -228,4 +228,15 @@ Nutrition Information per 100g expect(scan.sugars, 2.8); expect(scan.protein, 4); }); + + test('a scan keeps its photo and exports its values for a report', () { + final scan = parseNutritionText('Per 100 g\nEnergy 120 kcal\nProtein 8 g') + .withRaw('text') + .withPhoto('/tmp/label.png'); + expect(scan.photo, '/tmp/label.png'); + expect(scan.raw, 'text'); + expect(scan.toJson()['calories'], 120); + expect(scan.toJson()['protein'], 8); + expect(scan.toJson()['fat'], isNull); + }); }