fix(whatsapp): add ACK timeout sweep and retry

docs/architecture/cloud-integrations.md

@@ -368,6 +368,20 @@ never re-parsed as tokens. All five automated rules ship disabled (`enabled: fal
 enforce the blocklist, a per-recipient hourly cap (`RATE_LIMIT_MAX_PER_HOUR`), a 10-minute duplicate-body
 filter, simulated typing delay, and a content filter that rejects non-HTTPS URLs and URL shorteners.
 
+**ACK timeout, late ACKs, and retry.** Outbound messages remain in `typing` (refreshing `typing_at` when
+`socket.sendMessage` completes) until a Baileys `messages.update` server ACK (`status = 2/3/4`) arrives.
+`sweepAckTimeouts()` runs every 60 seconds (`ACK_SWEEP_INTERVAL_MS`) while connected, on socket open,
+during scheduled automation sweeps, and on startup (`initFromDb()`) to mark non-in-flight outbound rows
+still in `queued`, `seen`, or `typing` past `ACK_TIMEOUT_MS` (5 minutes, or any unfinished pre-restart or
+unsubmitted rows on startup) as `failed` with `error = 'ack_timeout'`. If a late ACK arrives after a row
+was marked `failed`, `advanceStatus()` promotes the row to `sent`, `delivered`, or `read` by
+`external_message_id` (including prior retry-attempt IDs), clears `error` and `failed_at`, backfills
+earlier timestamps via `COALESCE`, and ignores out-of-order lower-rank updates.
+`POST /api/whatsapp/messages/:messageId/retry` (`retryMessage()`) coalesces concurrent retry requests and
+atomically re-queues a `failed` row in place without creating duplicate rows, enforcing Offline mode
+(`offline_mode`) and all safety controls while excluding the retried row from self-blocking in rate,
+weekly-cap, and duplicate-body checks.
+
 **Offline behaviour.** No work is started when the module is disabled, when Offline mode is on, when
 it is in a terminal shutdown state, or when the request signal is aborted. Work already in flight is
 cancelled when Offline mode turns on or during shutdown rather than left to fail late.

docs/reference/api.md

@@ -766,6 +766,7 @@ Router: `main/routes/whatsapp.ts`. Full path: `/api/whatsapp`.
 | `POST`   | `/automation/broadcast`   | `ROLE_ACCESS.ownerManager`        | body: `template_id`, `min_visits`, `inactive_days`, `min_points`, `limit` | `{ ok: true, matched, attempted, sent, skipped, errors }` for up to `limit` (1-50, default 25) matching active customers; `503` when disconnected.                                               |
 | `POST`   | `/automation/run-sweep`   | `ROLE_ACCESS.ownerManager`        | none                                                                      | `{ ok: true, feedbackSent, winbackSent }` from an immediate scheduled-rule sweep.                                                                                                                |
 | `GET`    | `/messages`               | `whatsapp.use`                    | query: `?limit`, `?offset`, `?direction`, `?status`, `?phone`, `?bill_id` | Message history with `trigger_rule` attribution; entries with `kind: "z_report"` are omitted unless the caller also has `reports.view`. This permission filter is applied before pagination. |
+| `POST`   | `/messages/:messageId/retry` | `whatsapp.use`                 | path: `messageId`                                                         | `{ ok: true, messageId }` after retrying a `failed` outbound message in place (`403` unless the caller also has `reports.view` when `kind: "z_report"`); `503` when disconnected, in cooldown, or in Offline mode; `404` when not found; `400` when invalid, not `failed`, or blocked by safety checks. |
 | `GET`    | `/inbox`                  | `ROLE_ACCESS.ownerManager`        | query: `?limit`, `?offset`                                                | -                                                                                                                                                                                                |
 | `POST`   | `/inbox/:messageId/reply` | `ROLE_ACCESS.ownerManager`        | path: `messageId`; body: `body`                                           | Replies to an inbound message.                                                                                                                                                                   |
 | `GET`    | `/blocklist`              | `ROLE_ACCESS.ownerManager`        | none                                                                      | -                                                                                                                                                                                                |

frontend/src/app/(dashboard)/whatsapp/page.tsx

@@ -65,6 +65,7 @@ interface WhatsAppStatus {
   lastError: string | null;
   lastErrorReason: string | null;
   cooldownUntil: string | null;
+  offlineMode?: boolean;
 }
 
 type WhatsAppApiErrorKey = keyof AppConfig['Messages']['whatsapp']['apiError'];
@@ -264,6 +265,9 @@ export default function WhatsAppPage() {
   const tAutomation = useTranslations('whatsapp.automation');
   const tLastError = useTranslations('whatsapp.lastError');
   const tApiError = useTranslations('whatsapp.apiError');
+  const tSend = useTranslations('whatsapp.send');
+  const tStatus = useTranslations('whatsapp.status');
+  const tSettings = useTranslations('settings');
   const { confirm, ConfirmDialog } = useConfirm();
   const setWhatsappEnabled = usePosSettingsStore((s) => s.setWhatsappEnabled);
   const { currentTenant } = useAuthStore();
@@ -288,6 +292,7 @@ export default function WhatsAppPage() {
   const [pairingPhone, setPairingPhone] = useState(dialCode);
 
   const [sentMessages, setSentMessages] = useState<SentMessage[]>([]);
+  const [retryingIds, setRetryingIds] = useState<Set<number>>(() => new Set());
   const [inbox, setInbox] = useState<InboxMessage[]>([]);
   const [blocklist, setBlocklist] = useState<BlocklistRow[]>([]);
   const [blockPhone, setBlockPhone] = useState('');
@@ -409,9 +414,10 @@ export default function WhatsAppPage() {
   useEffect(() => {
     const id = setInterval(() => {
       void refreshStatus();
+      void refreshSent();
     }, 5000);
     return () => clearInterval(id);
-  }, [refreshStatus]);
+  }, [refreshStatus, refreshSent]);
 
   useEffect(() => {
     if (!isAdmin) return;
@@ -489,6 +495,46 @@ export default function WhatsAppPage() {
     }
   };
 
+  const retrySentMessage = async (messageId: number) => {
+    if (retryingIds.has(messageId)) return;
+    if (status?.offlineMode) {
+      toast.error(tSettings('offlineModeTitle'));
+      return;
+    }
+    setRetryingIds((prev) => {
+      const next = new Set(prev);
+      next.add(messageId);
+      return next;
+    });
+    try {
+      await api.post(`/whatsapp/messages/${messageId}/retry`);
+      toast.success(tStatus('queued'));
+      await refreshSent();
+    } catch (err) {
+      const reason = (err as { response?: { data?: { reason?: string } } })?.response?.data?.reason;
+      if (reason === 'offline_mode') {
+        toast.error(tSettings('offlineModeTitle'));
+      } else if (reason === 'not_connected' || reason === 'cooldown') {
+        toast.error(tSend('error.notConnected'));
+      } else if (reason === 'blocked') {
+        toast.error(tSend('error.blocked'));
+      } else if (reason === 'not_on_whatsapp') {
+        toast.error(tSend('error.notOnWhatsapp'));
+      } else if (reason === 'rate_limited' || reason === 'weekly_cap') {
+        toast.error(tSend('error.rateLimited'));
+      } else {
+        toastApiError(err, tSend('failed'), tApiError);
+      }
+      await refreshSent();
+    } finally {
+      setRetryingIds((prev) => {
+        const next = new Set(prev);
+        next.delete(messageId);
+        return next;
+      });
+    }
+  };
+
   const sendBroadcastNow = async () => {
     setBroadcasting(true);
     try {
@@ -1048,9 +1094,30 @@ export default function WhatsAppPage() {
                             </div>
                           </TableCell>
                           <TableCell>
-                            <div className="flex flex-col gap-1">
+                            <div className="flex flex-col items-start gap-1">
                               <StatusStepper status={m.status} />
                               {m.error && <div className="text-xs text-destructive">{m.error}</div>}
+                              {m.status === 'failed' && (
+                                <Button
+                                  type="button"
+                                  size="sm"
+                                  variant="outline"
+                                  className="h-7 px-2 text-xs"
+                                  disabled={
+                                    retryingIds.has(m.id) ||
+                                    Boolean(status?.offlineMode) ||
+                                    status?.state !== 'connected'
+                                  }
+                                  onClick={() => void retrySentMessage(m.id)}
+                                >
+                                  {retryingIds.has(m.id) ? (
+                                    <Loader2 className="h-3 w-3 me-1 animate-spin" />
+                                  ) : (
+                                    <RotateCcw className="h-3 w-3 me-1" />
+                                  )}
+                                  {tSettings('retry')}
+                                </Button>
+                              )}
                             </div>
                           </TableCell>
                           <TableCell className="text-sm max-w-md">

main/routes/whatsapp.ts

@@ -2,6 +2,7 @@ import { Router, Request, Response } from 'express';
 import { hasPermission, requirePermission } from '../services/authorization';
 import { asyncHandler } from '../middleware/async-handler';
 import { getDatabase, getSettingValue, upsertSettings } from '../db';
+import { isOfflineMode } from '../services/offline-mode';
 import { getHttpRequestSignal, trackHttpRequestWork } from '../shutdown';
 import * as whatsapp from '../services/whatsapp';
 import * as QRCode from 'qrcode';
@@ -155,14 +156,77 @@ router.post(
       }),
     );
     if (!result.ok) {
-      const status = result.reason === 'not_connected' || result.reason === 'cooldown' ? 503 : 400;
+      const status =
+        result.reason === 'not_connected' ||
+        result.reason === 'cooldown' ||
+        result.reason === 'offline_mode'
+          ? 503
+          : 400;
       res.status(status).json({ error: result.error, reason: result.reason });
       return;
     }
     res.json({ ok: true, messageId: result.messageId });
   }),
 );
 
+router.post(
+  '/messages/:messageId/retry',
+  requirePermission('whatsapp.use'),
+  asyncHandler(async (req, res) => {
+    if (isOfflineMode()) {
+      res.status(503).json({
+        error: 'Offline mode is on. Turn it off in Settings to use WhatsApp.',
+        reason: 'offline_mode',
+      });
+      return;
+    }
+    const rawId = Number(req.params.messageId);
+    if (!Number.isInteger(rawId) || rawId <= 0) {
+      res.status(400).json({ error: 'messageId must be a positive integer', reason: 'not_found' });
+      return;
+    }
+    const userId = String((req as Request & { user?: { userId?: string } }).user?.userId || '');
+    const db = getDatabase();
+    const row = db
+      .prepare('SELECT kind FROM whatsapp_messages WHERE id = ? AND direction = ?')
+      .get(rawId, 'outbound') as { kind: string } | undefined;
+    if (!row) {
+      res.status(404).json({ error: 'Message not found.', reason: 'not_found' });
+      return;
+    }
+    if (row.kind === 'z_report' && !hasPermission(userId, 'reports.view')) {
+      res.status(403).json({
+        error: 'Insufficient permissions',
+        required: 'reports.view',
+        code: 'PERMISSION_DENIED',
+      });
+      return;
+    }
+    const result = await trackHttpRequestWork(
+      req,
+      whatsapp.retryMessage(rawId, {
+        userId: userId || null,
+        signal: getHttpRequestSignal(req),
+      }),
+    );
+    if (!result.ok) {
+      const status =
+        result.reason === 'not_found'
+          ? 404
+          : result.reason === 'not_connected' ||
+              result.reason === 'cooldown' ||
+              result.reason === 'offline_mode'
+            ? 503
+            : 400;
+      res
+        .status(status)
+        .json({ error: result.error, reason: result.reason, messageId: result.messageId });
+      return;
+    }
+    res.json({ ok: true, messageId: result.messageId });
+  }),
+);
+
 router.get('/automation', requirePermission('whatsapp.use'), (_req: Request, res: Response) => {
   res.json({
     config: whatsapp.getAutomationConfig(),
@@ -289,7 +353,12 @@ router.post(
       }),
     );
     if (!result.ok) {
-      const status = result.reason === 'not_connected' || result.reason === 'cooldown' ? 503 : 400;
+      const status =
+        result.reason === 'not_connected' ||
+        result.reason === 'cooldown' ||
+        result.reason === 'offline_mode'
+          ? 503
+          : 400;
       res.status(status).json({ error: result.error, reason: result.reason });
       return;
     }