Improve board calibration, Lichess recovery and voice controls

src/components/common/cornersButton.tsx

@@ -1,24 +1,29 @@
 import { findCorners } from "../../utils/findCorners";
 import { useDispatch } from 'react-redux';
 import SidebarButton from "./sidebarButton";
+import { useState } from "react";
 
 const CornersButton = ({ piecesModelRef, xcornersModelRef, videoRef, canvasRef, setText}: 
   {piecesModelRef: any, xcornersModelRef: any, videoRef: any, canvasRef: any, setText: any}) => {
   const dispatch = useDispatch();
+  const [finding, setFinding] = useState(false);
 
   const handleClick = (e: any) => {
     e.preventDefault();
+    if (finding) return;
+    setFinding(true);
 
     void findCorners(piecesModelRef, xcornersModelRef, videoRef, canvasRef, dispatch, setText)
       .catch((error: unknown) => {
         console.error("Unable to find chessboard corners", error);
         setText(["Unable to find chessboard corners"]);
-      });
+      })
+      .finally(() => setFinding(false));
   }
 
   return (
-    <SidebarButton onClick={handleClick}>
-      Find Corners
+    <SidebarButton onClick={handleClick} disabled={finding}>
+      {finding ? "Finding Corners..." : "Find Corners"}
     </SidebarButton>
   );
 };

src/components/common/sidebarButton.tsx

@@ -1,9 +1,9 @@
 const SidebarButton = (props: any) => {
   return (
-    <button onClick={props.onClick} className="btn btn-dark btn-sm btn-outline-light w-100">
+    <button onClick={props.onClick} disabled={props.disabled} className="btn btn-dark btn-sm btn-outline-light w-100">
       {props.children}
     </button>
   )
 }
 
-export default SidebarButton;
+export default SidebarButton;

src/components/common/toast.tsx

@@ -41,6 +41,8 @@ const Toast = () => {
 
   if (!show || !game.error) return null;
 
+  const message = typeof game.error === "string" ? game.error : String(game.error);
+
   return (
     <div
       className="position-fixed top-0 end-0 m-3"
@@ -57,7 +59,7 @@ const Toast = () => {
         }}
       >
         <div className="toast-body d-flex justify-content-between align-items-center">
-          {game.error}
+          {message}
           <button
             type="button"
             className="btn-close btn-close-white"

src/components/common/video.tsx

@@ -24,6 +24,7 @@ const Video = ({ piecesModelRef, canvasRef, videoRef, sidebarRef, playing,
   const boardRef = useRef<any>(makeBoard(game));
   const movesPairsRef = useRef<MovesPair[]>(getMovesPairs(boardRef.current));
   const lastMoveRef = useRef<string>(game.lastMove);
+  const syncRequiredRef = useRef<boolean>(game.syncRequired ?? false);
   const moveTextRef = useRef<string>("");
   const [canPlay, setCanPlay] = useState(false);
 
@@ -41,6 +42,7 @@ const Video = ({ piecesModelRef, canvasRef, videoRef, sidebarRef, playing,
     }
     boardRef.current = board;
     lastMoveRef.current = game.lastMove;
+    syncRequiredRef.current = game.syncRequired ?? false;
   }, [game])
 
   const getMoveText = (board: any): string => {
@@ -111,7 +113,7 @@ const Video = ({ piecesModelRef, canvasRef, videoRef, sidebarRef, playing,
     }
 
     const stopDetection = findPieces(piecesModelRef, videoRef, canvasRef, playingRef, setText, dispatch,
-      cornersRef, boardRef, movesPairsRef, lastMoveRef, moveTextRef, mode);
+      cornersRef, boardRef, movesPairsRef, lastMoveRef, moveTextRef, syncRequiredRef, mode);
 
     const stopWebcam = async () => {
       const stream = await streamPromise;

src/components/common/videoAndSidebar.tsx

@@ -4,7 +4,7 @@ import { useOutletContext } from "react-router-dom";
 import { useDispatch } from 'react-redux';
 import { cornersReset, useCorners } from '../../slices/cornersSlice';
 import { Container } from "../common";
-import { CornersDict, Mode, ModelRefs, Study } from "../../types";
+import { CornersDict, Mode, ModelRefs, PlayInputMode, Study, VoiceLanguage } from "../../types";
 import RecordSidebar from "../record/recordSidebar";
 import UploadSidebar from "../upload/uploadSidebar";
 import BroadcastSidebar from "../broadcast/broadcastSidebar";
@@ -33,6 +33,8 @@ const VideoAndSidebar = ({ mode }: { mode: Mode }) => {
 
   const [text, setText] = useState<string[]>([]);
   const [playing, setPlaying] = useState<boolean>(false);
+  const [playInputMode, setPlayInputMode] = useState<PlayInputMode>("camera");
+  const [voiceLanguage, setVoiceLanguage] = useState<VoiceLanguage>("es-ES");
   const [study, setStudy] = useState<Study | null>(null);
   const [boardNumber, setBoardNumber] = useState<number>(-1);
 
@@ -63,8 +65,8 @@ const VideoAndSidebar = ({ mode }: { mode: Mode }) => {
   }, [boardNumber, mode, moves, study, token])
 
   useEffect(() => {
-    playingRef.current = playing;
-  }, [playing]);
+    playingRef.current = playing && (mode !== "play" || playInputMode === "camera");
+  }, [mode, playInputMode, playing]);
 
   useEffect(() => {
     cornersRef.current = corners;
@@ -82,6 +84,10 @@ const VideoAndSidebar = ({ mode }: { mode: Mode }) => {
     "text": text,
     "study": study,
     "setPlaying": setPlaying,
+    "playInputMode": playInputMode,
+    "setPlayInputMode": setPlayInputMode,
+    "voiceLanguage": voiceLanguage,
+    "setVoiceLanguage": setVoiceLanguage,
     "setText": setText,
     "setBoardNumber": setBoardNumber,
     "setStudy": setStudy,

src/components/play/playSidebar.tsx

@@ -1,24 +1,31 @@
 import { CornersButton, Sidebar, RecordButton, DeviceButton } from "../common";
-import { Game, SetBoolean, SetStringArray } from "../../types";
+import { Game, PlayInputMode, SetBoolean, SetStringArray, VoiceLanguage } from "../../types";
 import { useUser } from "../../slices/userSlice";
 import { useEffect, useRef, useState } from "react";
-import { lichessPlayMove, lichessStreamGame } from "../../utils/lichess";
+import { BoardStreamEvent, errorMessage, lichessPlayMove, lichessStreamGame } from "../../utils/lichess";
 import { Color } from "chessops/types";
 import { useDispatch } from "react-redux";
-import { gameUpdate, gameSetError, makeBoard, makeUpdatePayload, useGame } from "../../slices/gameSlice";
+import { gameUpdate, gameSetError, gameSetStart, gameSetSyncRequired, makeBoardFromUci, makeUpdatePayload, useGame } from "../../slices/gameSlice";
 import GamesButton from "./gamesButton";
+import { START_FEN } from "../../utils/constants";
+import VoiceControl from "./voiceControl";
 
 const PlaySidebar = ({ piecesModelRef, xcornersModelRef, videoRef, canvasRef, sidebarRef,
-  playing, setPlaying, text, setText }: {
+  playing, setPlaying, text, setText, playInputMode, setPlayInputMode, voiceLanguage, setVoiceLanguage }: {
     piecesModelRef: any, xcornersModelRef: any, videoRef: any, canvasRef: any, sidebarRef: any,
     playing: boolean, setPlaying: SetBoolean,
-    text: string[], setText: SetStringArray
+    text: string[], setText: SetStringArray,
+    playInputMode: PlayInputMode, setPlayInputMode: (mode: PlayInputMode) => void,
+    voiceLanguage: VoiceLanguage, setVoiceLanguage: (language: VoiceLanguage) => void
   }) => {
   const token: string = useUser().token;
   const game: Game = useGame();
   const gameRef = useRef<Game>(game);
   const [gameId, setGameId] = useState<string>();
   const [color, setColor] = useState<Color>();
+  const [streamRevision, setStreamRevision] = useState(0);
+  const remoteMovesRef = useRef<string[]>([]);
+  const remoteStartRef = useRef(START_FEN);
   const dispatch = useDispatch();
   const inputStyle = {
     display: playing ? "none" : "inline-block"
@@ -28,59 +35,111 @@ const PlaySidebar = ({ piecesModelRef, xcornersModelRef, videoRef, canvasRef, si
     gameRef.current = game;
   }, [game]);
 
+  useEffect(() => {
+    remoteMovesRef.current = [];
+    remoteStartRef.current = START_FEN;
+  }, [gameId]);
+
   useEffect(() => {
     const colorToMove = game.fen.split(" ")[1];
     const lastMove = game.lastMove;
     const fromOpponent = game.fromOpponent;
-    if ((colorToMove === color) || (lastMove === "") || (gameId === undefined) || (color === undefined) || fromOpponent) {
+    if ((colorToMove === color) || (lastMove === "") || (gameId === undefined) || (color === undefined) || fromOpponent || game.syncRequired) {
       return;
     }
 
     lichessPlayMove(token, gameId, lastMove)
-      .catch((err: string) => {
-        dispatch(gameSetError(err));
+      .catch((error: unknown) => {
+        dispatch(gameSetError(errorMessage(error)));
+        dispatch(gameSetSyncRequired(playInputMode === "camera"));
+        remoteMovesRef.current = [];
+        setStreamRevision((revision) => revision + 1);
       });
-  }, [color, dispatch, game, gameId, token])
+  }, [color, dispatch, game, gameId, playInputMode, token])
+
+  useEffect(() => {
+    if (playInputMode === "voice" && game.syncRequired) {
+      dispatch(gameSetSyncRequired(false));
+    }
+  }, [dispatch, game.syncRequired, playInputMode]);
 
   useEffect(() => {
     if (gameId === undefined) {
       return;
     }
 
-    const streamGameCallback = async (response: any) => {
-      // The selected game is already initialized from nowPlaying.fen.
-      if (response.type === "gameFull") {
+    const streamGameCallback = async (response: BoardStreamEvent) => {
+      const movesText = response.type === "gameFull" ? response.state?.moves : response.moves;
+      if (movesText === undefined) {
         return;
       }
 
-      const moves = response.moves;
-      if (moves === undefined) {
-        return;
+      if (response.type === "gameFull") {
+        remoteStartRef.current = response.initialFen && response.initialFen !== "startpos"
+          ? response.initialFen
+          : START_FEN;
       }
 
-      const splitMoves = moves.split(" ");
-      const lastMove = splitMoves[splitMoves.length - 1];
-      if (lastMove === gameRef.current.lastMove) {
+      const nextMoves = movesText.trim().split(/\s+/).filter(Boolean);
+      const previousMoves = remoteMovesRef.current;
+      const isInitialSnapshot = response.type === "gameFull" && previousMoves.length === 0;
+      const isSimpleAppend = nextMoves.length >= previousMoves.length
+        && previousMoves.every((move, index) => nextMoves[index] === move);
+      const changed = nextMoves.length !== previousMoves.length
+        || nextMoves.some((move, index) => move !== previousMoves[index]);
+      if (!changed && !isInitialSnapshot) {
         return;
       }
 
-      const board = makeBoard(gameRef.current);
-      board.playUci(lastMove);
-      const payload = makeUpdatePayload(board, false, true);
-      console.log("payload", payload);
-      dispatch(gameUpdate(payload));
+      try {
+        const board = makeBoardFromUci(remoteStartRef.current, movesText);
+        const requiresSync = playInputMode === "camera" && !isInitialSnapshot && !isSimpleAppend;
+        const payload = {
+          ...makeUpdatePayload(board, false, true),
+          syncRequired: requiresSync || gameRef.current.syncRequired
+        };
+        remoteMovesRef.current = nextMoves;
+        dispatch(gameSetStart(remoteStartRef.current));
+        dispatch(gameUpdate(payload));
+        if (requiresSync) {
+          setText(["Desynchronization detected", "Rearrange the pieces to match Lichess"]);
+        }
+      } catch (error: unknown) {
+        dispatch(gameSetError(errorMessage(error)));
+        dispatch(gameSetSyncRequired(playInputMode === "camera"));
+      }
     };
 
-    const controller = lichessStreamGame(token, streamGameCallback, gameId);
+    const controller = lichessStreamGame(token, streamGameCallback, gameId, () => {
+      setText(["Lichess connection interrupted", "Reconnecting automatically..."]);
+    });
     return () => controller.abort();
-  }, [dispatch, gameId, token]);
+  }, [dispatch, gameId, playInputMode, setText, streamRevision, token]);
 
   return (
     <Sidebar sidebarRef={sidebarRef} playing={playing} text={text} setText={setText} >
       <li className="my-1" style={inputStyle}>
+        <div className="btn-group w-100" role="group" aria-label="Modo de entrada">
+          <button type="button" className={`btn btn-sm btn-outline-light ${playInputMode === "camera" ? "btn-light text-dark" : "btn-dark"}`}
+            onClick={() => setPlayInputMode("camera")}>Cámara</button>
+          <button type="button" className={`btn btn-sm btn-outline-light ${playInputMode === "voice" ? "btn-light text-dark" : "btn-dark"}`}
+            onClick={() => setPlayInputMode("voice")}>🎙 Voz</button>
+        </div>
+      </li>
+      {playInputMode === "voice" && !playing && (
+        <li className="my-1">
+          <div className="btn-group w-100" role="group" aria-label="Idioma de voz">
+            <button type="button" className={`btn btn-sm btn-outline-light ${voiceLanguage === "es-ES" ? "btn-light text-dark" : "btn-dark"}`}
+              onClick={() => setVoiceLanguage("es-ES")}>Español</button>
+            <button type="button" className={`btn btn-sm btn-outline-light ${voiceLanguage === "en-US" ? "btn-light text-dark" : "btn-dark"}`}
+              onClick={() => setVoiceLanguage("en-US")}>English</button>
+          </div>
+        </li>
+      )}
+      <li className="my-1" style={{ ...inputStyle, display: !playing && playInputMode === "camera" ? "inline-block" : "none" }}>
         <DeviceButton videoRef={videoRef} />
       </li>
-      <li className="my-1" style={inputStyle}>
+      <li className="my-1" style={{ ...inputStyle, display: !playing && playInputMode === "camera" ? "inline-block" : "none" }}>
         <GamesButton setGameId={setGameId} setColor={setColor} setText={setText} />
       </li>
       <li className="my-1" style={inputStyle}>
@@ -92,6 +151,13 @@ const PlaySidebar = ({ piecesModelRef, xcornersModelRef, videoRef, canvasRef, si
           <RecordButton playing={playing} setPlaying={setPlaying} />
         </div>
       </li>
+      {playInputMode === "voice" && <VoiceControl active={playing} color={color} language={voiceLanguage} setText={setText} />}
+      {playInputMode === "camera" && game.syncRequired && (
+        <li className="alert alert-warning py-2 px-3 my-2" role="alert">
+          <strong>Desynchronization</strong><br />
+          Rearrange the physical pieces to match Lichess. Detection will resume automatically.
+        </li>
+      )}
     </Sidebar>
   );
 };