diff --git a/src/components/common/cornersButton.tsx b/src/components/common/cornersButton.tsx index 8a9e0e9..8d1c1d5 100644 --- a/src/components/common/cornersButton.tsx +++ b/src/components/common/cornersButton.tsx @@ -1,24 +1,29 @@ import { findCorners } from "../../utils/findCorners"; import { useDispatch } from 'react-redux'; import SidebarButton from "./sidebarButton"; +import { useState } from "react"; const CornersButton = ({ piecesModelRef, xcornersModelRef, videoRef, canvasRef, setText}: {piecesModelRef: any, xcornersModelRef: any, videoRef: any, canvasRef: any, setText: any}) => { const dispatch = useDispatch(); + const [finding, setFinding] = useState(false); const handleClick = (e: any) => { e.preventDefault(); + if (finding) return; + setFinding(true); void findCorners(piecesModelRef, xcornersModelRef, videoRef, canvasRef, dispatch, setText) .catch((error: unknown) => { console.error("Unable to find chessboard corners", error); setText(["Unable to find chessboard corners"]); - }); + }) + .finally(() => setFinding(false)); } return ( - - Find Corners + + {finding ? "Finding Corners..." : "Find Corners"} ); }; diff --git a/src/components/common/sidebarButton.tsx b/src/components/common/sidebarButton.tsx index 6f672fb..27629a2 100644 --- a/src/components/common/sidebarButton.tsx +++ b/src/components/common/sidebarButton.tsx @@ -1,9 +1,9 @@ const SidebarButton = (props: any) => { return ( - ) } -export default SidebarButton; \ No newline at end of file +export default SidebarButton; diff --git a/src/components/common/toast.tsx b/src/components/common/toast.tsx index e5b47a2..c9e9025 100644 --- a/src/components/common/toast.tsx +++ b/src/components/common/toast.tsx @@ -41,6 +41,8 @@ const Toast = () => { if (!show || !game.error) return null; + const message = typeof game.error === "string" ? game.error : String(game.error); + return (
{ }} >
- {game.error} + {message} + +
+ + {playInputMode === "voice" && !playing && ( +
  • +
    + + +
    +
    • + )} +
    • -
  • +
  • @@ -92,6 +151,13 @@ const PlaySidebar = ({ piecesModelRef, xcornersModelRef, videoRef, canvasRef, si
    • + {playInputMode === "voice" && } + {playInputMode === "camera" && game.syncRequired && ( +
  • + Desynchronization
    + Rearrange the physical pieces to match Lichess. Detection will resume automatically. +
    • + )} ); }; diff --git a/src/components/play/voiceControl.tsx b/src/components/play/voiceControl.tsx new file mode 100644 index 0000000..bcb3297 --- /dev/null +++ b/src/components/play/voiceControl.tsx @@ -0,0 +1,168 @@ +import { useEffect, useRef, useState } from "react"; +import { useDispatch } from "react-redux"; +import { Color } from "chessops/types"; +import { makeUci } from "chessops/util"; +import { Game, SetStringArray, VoiceLanguage } from "../../types"; +import { gameUpdate, makeBoard, makeUpdatePayload, useGame } from "../../slices/gameSlice"; +import { resolveVoiceMove } from "../../utils/voice"; + +type SpeechRecognitionResultLike = { + isFinal: boolean; + 0: { transcript: string; confidence: number }; +}; + +type SpeechRecognitionEventLike = Event & { + resultIndex: number; + results: { length: number; [index: number]: SpeechRecognitionResultLike }; +}; + +type SpeechRecognitionErrorLike = Event & { error: string }; + +type SpeechRecognitionLike = { + lang: string; + continuous: boolean; + interimResults: boolean; + maxAlternatives: number; + start: () => void; + stop: () => void; + onresult: ((event: SpeechRecognitionEventLike) => void) | null; + onerror: ((event: SpeechRecognitionErrorLike) => void) | null; + onend: (() => void) | null; +}; + +type SpeechRecognitionConstructor = new () => SpeechRecognitionLike; + +const getSpeechRecognition = (): SpeechRecognitionConstructor | undefined => { + const speechWindow = window as unknown as { + SpeechRecognition?: SpeechRecognitionConstructor; + webkitSpeechRecognition?: SpeechRecognitionConstructor; + }; + return speechWindow.SpeechRecognition ?? speechWindow.webkitSpeechRecognition; +}; + +const VoiceControl = ({ active, color, language, setText }: { + active: boolean; + color?: Color; + language: VoiceLanguage; + setText: SetStringArray; +}) => { + const game: Game = useGame(); + const gameRef = useRef(game); + const lastCommandRef = useRef({ transcript: "", time: 0 }); + const dispatch = useDispatch(); + const [listening, setListening] = useState(false); + const supported = getSpeechRecognition() !== undefined; + const english = language === "en-US"; + + useEffect(() => { + gameRef.current = game; + }, [game]); + + useEffect(() => { + const Recognition = getSpeechRecognition(); + if (!active || !Recognition || color === undefined) { + setListening(false); + return; + } + + const recognition = new Recognition(); + let stopped = false; + let mayRestart = true; + recognition.lang = language; + recognition.continuous = true; + recognition.interimResults = false; + recognition.maxAlternatives = 1; + + recognition.onresult = (event) => { + for (let index = event.resultIndex; index < event.results.length; index++) { + const result = event.results[index]; + if (!result.isFinal) continue; + const transcript = result[0].transcript.trim(); + const now = Date.now(); + if (transcript === lastCommandRef.current.transcript && now - lastCommandRef.current.time < 1500) continue; + lastCommandRef.current = { transcript, time: now }; + + const board = makeBoard(gameRef.current); + if (board.turn !== color) { + setText([`${english ? "Voice" : "Voz"}: “${transcript}”`, english ? "It is the opponent's turn" : "Es el turno del rival"]); + continue; + } + + const resolution = resolveVoiceMove(board, transcript, language); + if (!resolution.move) { + setText([`${english ? "Voice" : "Voz"}: “${transcript}”`, resolution.message]); + continue; + } + + const played = board.playUci(makeUci(resolution.move)); + if (!played) { + setText([`${english ? "Voice" : "Voz"}: “${transcript}”`, english ? "The move could not be applied" : "No se pudo aplicar la jugada"]); + continue; + } + dispatch(gameUpdate(makeUpdatePayload(board))); + setText([`${english ? "Voice" : "Voz"}: “${transcript}”`, `${english ? "Move" : "Jugada"}: ${resolution.message}`]); + } + }; + + recognition.onerror = (event) => { + if (event.error === "no-speech" || event.error === "aborted") return; + if (event.error === "not-allowed" || event.error === "service-not-allowed") { + mayRestart = false; + setText(english + ? ["The microphone could not be used", "Allow microphone access in the browser"] + : ["No se pudo usar el micrófono", "Permite el acceso al micrófono en el navegador"]); + } else { + setText([english ? "Voice recognition error" : "Error de reconocimiento de voz", event.error]); + } + }; + + recognition.onend = () => { + setListening(false); + if (!stopped && mayRestart) { + window.setTimeout(() => { + if (stopped) return; + try { + recognition.start(); + setListening(true); + } catch (_) { + // The browser can still be transitioning from the previous session. + } + }, 250); + } + }; + + try { + recognition.start(); + setListening(true); + setText(english + ? ["Voice mode active", "Say for example: “bishop C3” or “C4 to C5”"] + : ["Modo voz activo", "Di por ejemplo: “alfil C3” o “C4 a C5”"]); + } catch (_) { + setText([english ? "Voice recognition could not be started" : "No se pudo iniciar el reconocimiento de voz"]); + } + + return () => { + stopped = true; + recognition.onend = null; + recognition.stop(); + setListening(false); + }; + }, [active, color, dispatch, english, language, setText]); + + const status = !supported + ? (english ? "This browser does not support voice recognition" : "El navegador no soporta reconocimiento de voz") + : color === undefined + ? (english ? "Select a game first" : "Selecciona una partida primero") + : active && listening + ? (english ? "Listening in English…" : "Escuchando en español…") + : (english ? "Press ▶ to start listening" : "Pulsa ▶ para comenzar a escuchar"); + + return ( +
  • + {english ? "Voice mode" : "Modo voz"}
    + {status} +
    • + ); +}; + +export default VoiceControl; diff --git a/src/slices/gameSlice.tsx b/src/slices/gameSlice.tsx index 5cef16c..98bbe1b 100644 --- a/src/slices/gameSlice.tsx +++ b/src/slices/gameSlice.tsx @@ -18,7 +18,8 @@ const initialState: Game = { "lastMove": "", "greedy": false, "fromOpponent": false, - "error": null + "error": null, + "syncRequired": false }; const gameSlice = createSlice({ @@ -52,7 +53,13 @@ const gameSlice = createSlice({ state.lastMove = initialState.lastMove; }, gameSetError(state, action) { - state.error = action.payload; + const error = action.payload; + state.error = error === null || typeof error === "string" + ? error + : error instanceof Error ? error.message : String(error); + }, + gameSetSyncRequired(state, action) { + state.syncRequired = Boolean(action.payload); }, gameUpdate(state, action) { const newState: Game = { @@ -62,7 +69,8 @@ const gameSlice = createSlice({ "lastMove": action.payload.lastMove, "greedy": action.payload.greedy, "fromOpponent": action.payload.fromOpponent ?? false, - "error": action.payload.error ?? null + "error": action.payload.error ?? null, + "syncRequired": action.payload.syncRequired ?? state.syncRequired ?? false } return newState } @@ -107,7 +115,8 @@ export const makeUpdatePayload = (board: any, greedy: boolean = false, fromOppon "lastMove": lastMove, "greedy": greedy, "fromOpponent": fromOpponent, - "error": error + "error": error, + "syncRequired": false } return payload @@ -171,11 +180,27 @@ export const makeBoard = (game: Game): any => { return board; } +export const makeBoardFromUci = (startFen: string, moves: string): any => { + const normalizedStart = startFen === "startpos" ? START_FEN : startFen; + const seed: Game = { + ...initialState, + start: normalizedStart, + fen: normalizedStart + }; + const board = makeBoard(seed); + for (const uci of moves.trim().split(/\s+/).filter(Boolean)) { + if (board.playUci(uci) === null) { + throw new Error(`Lichess sent an invalid move: ${uci}`); + } + } + return board; +}; + export const { gameSetMoves, gameResetMoves, gameSetFen, gameResetFen, gameSetStart, gameResetStart, gameSetLastMove, gameResetLastMove, - gameUpdate, gameSetError + gameUpdate, gameSetError, gameSetSyncRequired } = gameSlice.actions export default gameSlice.reducer diff --git a/src/types.tsx b/src/types.tsx index cac94c1..5dc75d0 100644 --- a/src/types.tsx +++ b/src/types.tsx @@ -34,7 +34,8 @@ interface Game { lastMove: string, greedy: boolean, fromOpponent: boolean, - error: string | null + error: string | null, + syncRequired: boolean } interface User { @@ -49,6 +50,8 @@ interface RootState { } type Mode = "record" | "upload" | "broadcast" | "play"; +type PlayInputMode = "camera" | "voice"; +type VoiceLanguage = "es-ES" | "en-US"; type SetBoolean = React.Dispatch> type SetString = React.Dispatch> @@ -60,5 +63,5 @@ export type { RootState, Study, ModelRefs, MovesData, MovesPair, CornersDict, CornersKey, CornersPayload, Game, SetBoolean, SetString, SetStringArray, SetNumber, Mode, - SetStudy -} \ No newline at end of file + SetStudy, PlayInputMode, VoiceLanguage +} diff --git a/src/utils/findCorners.tsx b/src/utils/findCorners.tsx index 1907762..cc82c19 100644 --- a/src/utils/findCorners.tsx +++ b/src/utils/findCorners.tsx @@ -7,7 +7,7 @@ import { cornersSet } from '../slices/cornersSlice'; import { MODEL_WIDTH, MODEL_HEIGHT, CORNER_KEYS } from "./constants"; import { clamp } from "./math"; import { CornersDict, CornersPayload } from "../types"; -import { NDArray } from "vectorious"; +import { array, NDArray } from "vectorious"; const x: number[] = Array.from({ length: 7 }, (_, i) => i); const y: number[] = Array.from({ length: 7 }, (_, i) => i); @@ -63,28 +63,34 @@ const getQuads = (xCorners: number[][]) => { const intXcorners = xCorners.flat().map(x => Math.round(x)); const delaunay = new Delaunator(intXcorners); const triangles = delaunay.triangles; - const quads = []; + const quads: number[][][] = []; + const seen = new Set(); for (let i = 0; i < triangles.length; i += 3) { - const t1 = triangles[i]; - const t2 = triangles[i + 1]; - const t3 = triangles[i + 2]; - const quad = [t1, t2, t3, -1]; + const first = [triangles[i], triangles[i + 1], triangles[i + 2]]; - for (let j = 0; j < triangles.length; j += 3) { + for (let j = i + 3; j < triangles.length; j += 3) { if (i === j) { continue; } - const cond1 = (t1 === triangles[j] && t2 === triangles[j + 1]) || (t1 === triangles[j + 1] && t2 === triangles[j]); - const cond2 = (t2 === triangles[j] && t3 === triangles[j + 1]) || (t2 === triangles[j + 1] && t3 === triangles[j]); - const cond3 = (t3 === triangles[j] && t1 === triangles[j + 1]) || (t3 === triangles[j + 1] && t1 === triangles[j]); - if ((cond1 || cond2 || cond3)) { - quad[3] = triangles[j + 2]; - break; - } - } - - if (quad[3] !== -1) { - quads.push(quad.map(x => xCorners[x])); + const second = [triangles[j], triangles[j + 1], triangles[j + 2]]; + if (first.filter(index => second.includes(index)).length !== 2) continue; + + const indices = Array.from(new Set([...first, ...second])); + if (indices.length !== 4) continue; + const key = [...indices].sort((a, b) => a - b).join(','); + if (seen.has(key)) continue; + seen.add(key); + + const points = indices.map(index => xCorners[index]); + const center = getCenter(points); + points.sort((a, b) => Math.atan2(a[1] - center[1], a[0] - center[0]) + - Math.atan2(b[1] - center[1], b[0] - center[0])); + const signedArea = points.reduce((area, point, index) => { + const next = points[(index + 1) % points.length]; + return area + point[0] * next[1] - next[0] * point[1]; + }, 0); + if (signedArea > 0) points.reverse(); + quads.push(points); } } return quads; @@ -102,41 +108,95 @@ const cdist = (a: number[][], b: number[][]) => { return dist; } +type GridMatch = { grid: number[]; image: number[]; error: number }; + +const getGridMatches = (warpedXcorners: number[][], xCorners: number[][], shift: number[], threshold = 0.34) => { + const matches = new Map(); + for (let index = 0; index < warpedXcorners.length; index++) { + const warped = warpedXcorners[index]; + const gx = Math.round(warped[0]); + const gy = Math.round(warped[1]); + if (gx < shift[0] || gx > shift[0] + 6 || gy < shift[1] || gy > shift[1] + 6) continue; + + const error = Math.hypot(warped[0] - gx, warped[1] - gy); + if (error > threshold) continue; + const key = `${gx},${gy}`; + const current = matches.get(key); + if (!current || error < current.error) { + matches.set(key, { grid: [gx, gy], image: xCorners[index], error }); + } + } + return Array.from(matches.values()); +}; + +// Robust projective fit using every visible intersection of the 7 x 7 inner lattice. +const fitGridHomography = (matches: GridMatch[]): NDArray => { + const normal = Array.from({ length: 8 }, () => Array(8).fill(0)); + const rhs = Array(8).fill(0); + + const accumulate = (row: number[], value: number) => { + for (let i = 0; i < 8; i++) { + rhs[i] += row[i] * value; + for (let j = 0; j < 8; j++) normal[i][j] += row[i] * row[j]; + } + }; + + matches.forEach(({ grid: [u, v], image: [x, y] }) => { + accumulate([u, v, 1, 0, 0, 0, -x * u, -x * v], x); + accumulate([0, 0, 0, u, v, 1, -y * u, -y * v], y); + }); + + const solution = array(normal).solve(array(rhs, { shape: [8, 1] })).toArray(); + return array([...solution, 1], { shape: [3, 3] }); +}; + +const refineGridHomography = (matches: GridMatch[]) => { + let transform = fitGridHomography(matches); + const projected = perspectiveTransform(matches.map(match => match.grid), transform); + const errors = matches.map((match, index) => euclidean(projected[index], match.image)); + const sortedErrors = [...errors].sort((a, b) => a - b); + const medianError = sortedErrors[Math.floor(sortedErrors.length / 2)]; + const robustLimit = Math.max(2, medianError * 2.5); + const inliers = matches.filter((_, index) => errors[index] <= robustLimit); + if (inliers.length >= 8) transform = fitGridHomography(inliers); + return { transform, inlierCount: inliers.length, medianError }; +}; + const calculateOffsetScore = (warpedXcorners: number[][], shift: number[]) => { const grid = GRID.map(x => [x[0] + shift[0], x[1] + shift[1]]); const dist = cdist(grid, warpedXcorners); - let assignmentCost = 0; - for (let i = 0; i < dist.length; i++) { - assignmentCost += Math.min(...dist[i]); - } - const score = 1 / (1 + assignmentCost); + const gridCost = dist.reduce((sum, row) => sum + Math.min(1.5, Math.min(...row)), 0) / grid.length; + const detectionCost = warpedXcorners.reduce((sum, _, detectionIndex) => { + const nearest = Math.min(...dist.map(row => row[detectionIndex])); + return sum + Math.min(1.5, nearest); + }, 0) / warpedXcorners.length; + const score = 1 / (1 + gridCost + detectionCost); return score; } -const findOffset = (warpedXcorners: number[][]) => { - const bestOffset = [0, 0]; - for (let i = 0; i < 2; i++) { - let low = -7; - let high = 1; - const scores: any = {}; - while ((high - low) > 1) { - const mid = (high + low) >> 1; - [mid, mid + 1].forEach(x => { - if (!(x in scores)) { - const shift = [0, 0]; - shift[i] = x; - scores[x] = calculateOffsetScore(warpedXcorners, shift); - } - }); - if (scores[mid] > scores[mid + 1]) { - high = mid - } else { - low = mid +const calculateLatticeScore = (warpedXcorners: number[][], xCorners: number[][], shift: number[]) => { + const matches = getGridMatches(warpedXcorners, xCorners, shift); + if (matches.length < 4) return Number.NEGATIVE_INFINITY; + const xs = matches.map(match => match.grid[0]); + const ys = matches.map(match => match.grid[1]); + const span = (Math.max(...xs) - Math.min(...xs)) + (Math.max(...ys) - Math.min(...ys)); + const meanError = matches.reduce((sum, match) => sum + match.error, 0) / matches.length; + return matches.length * 10 + span - meanError * 5 + calculateOffsetScore(warpedXcorners, shift); +}; + +const findOffset = (warpedXcorners: number[][], xCorners: number[][]) => { + let bestOffset = [0, 0]; + let bestScore = Number.NEGATIVE_INFINITY; + for (let dx = -7; dx <= 1; dx++) { + for (let dy = -7; dy <= 1; dy++) { + const score = calculateLatticeScore(warpedXcorners, xCorners, [dx, dy]); + if (score > bestScore) { + bestScore = score; + bestOffset = [dx, dy]; } } - bestOffset[i] = low + 1; } return bestOffset; @@ -145,9 +205,9 @@ const findOffset = (warpedXcorners: number[][]) => { const scoreQuad = (quad: number[][], xCorners: number[][]): [number, NDArray, number[]] => { const M: NDArray = getPerspectiveTransform(IDEAL_QUAD, quad); const warpedXcorners: number[][] = perspectiveTransform(xCorners, M); - const offset: number[] = findOffset(warpedXcorners); + const offset: number[] = findOffset(warpedXcorners, xCorners); - const score: number = calculateOffsetScore(warpedXcorners, offset); + const score: number = calculateLatticeScore(warpedXcorners, xCorners, offset); return [score, M, offset] } @@ -157,25 +217,52 @@ const findCornersFromXcorners = (xCorners: number[][]) => { return; } - let bestScore: number; - let bestM: NDArray; - let bestOffset: number[]; - [bestScore, bestM, bestOffset] = scoreQuad(quads[0], xCorners); - for (let i = 1; i < quads.length; i++) { - const [score, M, offset] = scoreQuad(quads[i], xCorners); - if (score > bestScore) { - bestScore = score; - bestM = M; - bestOffset = offset; + let bestScore = Number.NEGATIVE_INFINITY; + let bestM: NDArray | undefined; + let bestOffset: number[] | undefined; + for (const quad of quads) { + try { + const [score, M, offset] = scoreQuad(quad, xCorners); + if (score > bestScore) { + bestScore = score; + bestM = M; + bestOffset = offset; + } + } catch (_) { + // Degenerate Delaunay quads can produce a singular homography. } } - const invM = bestM.inv() + if (!bestM || !bestOffset) return; + + const warpedXcorners = perspectiveTransform(xCorners, bestM); + const matches = getGridMatches(warpedXcorners, xCorners, bestOffset); + let gridToImage = bestM.inv(); + if (matches.length >= 8) { + try { + const refined = refineGridHomography(matches); + if (refined.inlierCount >= 8 && refined.medianError < 8) { + gridToImage = refined.transform; + } + } catch (_) { + // Keep the four-point hypothesis when the all-grid fit is ill-conditioned. + } + } const warpedCorners = [[bestOffset[0] - 1, bestOffset[1] - 1], [bestOffset[0] - 1, bestOffset[1] + 7], [bestOffset[0] + 7, bestOffset[1] + 7], [bestOffset[0] + 7, bestOffset[1] - 1]] - const corners = perspectiveTransform(warpedCorners, invM); + const corners = perspectiveTransform(warpedCorners, gridToImage); + + const area = Math.abs(corners.reduce((sum, point, index) => { + const next = corners[(index + 1) % corners.length]; + return sum + point[0] * next[1] - next[0] * point[1]; + }, 0)) / 2; + const shortestEdge = Math.min(...corners.map((point, index) => euclidean(point, corners[(index + 1) % 4]))); + if (!Number.isFinite(bestScore) || area < MODEL_WIDTH * MODEL_HEIGHT * 0.04 || shortestEdge < 20 + || corners.flat().some(value => !Number.isFinite(value))) { + return; + } // Clip bad corners for (let i = 0; i < 4; i++) { @@ -226,7 +313,41 @@ const calculateKeypoints = (blackPieces: number[][], whitePieces: number[][], co return keypoints } -export const _findCorners = async (piecesModelRef: any, xcornersModelRef: any, videoRef: any, +const waitForVideoFrame = () => new Promise((resolve) => requestAnimationFrame(() => resolve())); + +const median = (values: number[]) => { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +}; + +const consensusKeypoints = (samples: CornersDict[]): CornersDict => { + const result = {} as CornersDict; + CORNER_KEYS.forEach((key) => { + result[key] = [ + median(samples.map(sample => sample[key][0])), + median(samples.map(sample => sample[key][1])) + ]; + }); + return result; +}; + +const detectCornersSample = async (piecesModelRef: any, xcornersModelRef: any, videoRef: any) => { + const pieces = await runPiecesModel(videoRef, piecesModelRef); + const blackPieces = pieces.filter(x => (x[2] <= 5)); + const whitePieces = pieces.filter(x => (x[2] > 5)); + if (blackPieces.length === 0 || whitePieces.length === 0) return null; + + const xCorners = await runXcornersModel(videoRef, xcornersModelRef, pieces); + if (xCorners.length < 5) return null; + const corners = findCornersFromXcorners(xCorners); + if (!corners) return null; + return { keypoints: calculateKeypoints(blackPieces, whitePieces, corners), xCorners }; +}; + +export const findCornersSingleFrame = async (piecesModelRef: any, xcornersModelRef: any, videoRef: any, canvasRef: any, dispatch: any, setText: any) => { if (invalidVideo(videoRef)) { return; @@ -267,6 +388,47 @@ export const _findCorners = async (piecesModelRef: any, xcornersModelRef: any, v setText(["Found corners", "Ready to record"]) } +export const _findCorners = async (piecesModelRef: any, xcornersModelRef: any, videoRef: any, + canvasRef: any, dispatch: any, setText: any) => { + if (invalidVideo(videoRef)) return; + + const samples: { keypoints: CornersDict, xCorners: number[][] }[] = []; + const sampleCount = 5; + setText(["Finding corners", "Hold the camera still..."]); + for (let index = 0; index < sampleCount; index++) { + try { + const sample = await detectCornersSample(piecesModelRef, xcornersModelRef, videoRef); + if (sample) samples.push(sample); + } catch (error) { + console.warn(`Corner sample ${index + 1} failed`, error); + } + await waitForVideoFrame(); + } + + if (samples.length < 3) { + setText(["Could not find stable corners", `Valid frames: ${samples.length}/${sampleCount}`]); + return; + } + + const roughConsensus = consensusKeypoints(samples.map(sample => sample.keypoints)); + const deviations = samples.map(sample => CORNER_KEYS.reduce((sum, key) => + sum + euclidean(sample.keypoints[key], roughConsensus[key]), 0) / CORNER_KEYS.length); + const deviationLimit = Math.max(10, median(deviations) * 2.5); + const stableSamples = samples.filter((_, index) => deviations[index] <= deviationLimit); + const keypoints = consensusKeypoints(stableSamples.map(sample => sample.keypoints)); + const bestSampleIndex = deviations.indexOf(Math.min(...deviations)); + + CORNER_KEYS.forEach((key) => { + const payload: CornersPayload = { + xy: getMarkerXY(keypoints[key], canvasRef.current.height, canvasRef.current.width), + key + }; + dispatch(cornersSet(payload)); + }); + renderCorners(canvasRef.current, samples[bestSampleIndex].xCorners); + setText(["Found stable corners", `${stableSamples.length}/${sampleCount} frames agreed`]); +}; + export const findCorners = async (piecesModelRef: any, xcornersModelRef: any, videoRef: any, canvasRef: any, dispatch: any, setText: any) => { const startTensors = tf.memory().numTensors; @@ -281,4 +443,4 @@ export const findCorners = async (piecesModelRef: any, xcornersModelRef: any, vi return () => { tf.disposeVariables(); }; -} \ No newline at end of file +} diff --git a/src/utils/findPieces.tsx b/src/utils/findPieces.tsx index 0ae08a1..c5474c1 100644 --- a/src/utils/findPieces.tsx +++ b/src/utils/findPieces.tsx @@ -1,11 +1,11 @@ import { renderState } from "./render/renderState"; import * as tf from "@tensorflow/tfjs-core"; import { getInvTransform, transformBoundary, transformCenters } from "./warp"; -import { gameUpdate, makeUpdatePayload } from "../slices/gameSlice"; +import { gameSetSyncRequired, gameUpdate, makeUpdatePayload } from "../slices/gameSlice"; import { getBoxesAndScores, getInput, getXY, invalidVideo } from "./detect"; import { Mode, MovesData, MovesPair } from "../types"; import { zeros } from "./math"; -import { CORNER_KEYS } from "./constants"; +import { CORNER_KEYS, LABEL_MAP } from "./constants"; import { parseSan } from "chessops/san"; import { makeUci } from "chessops/util"; @@ -139,6 +139,24 @@ export const getUpdate = (scoresTensor: tf.Tensor2D, squares: number[]) => { return update; } +const physicalBoardMatches = (state: number[][], board: any): boolean => { + for (let square = 0; square < 64; square++) { + const scores = state[square]; + const bestScore = Math.max(...scores); + const piece = board.board.get(square); + if (!piece) { + if (bestScore > 0.28) return false; + continue; + } + + const roleLetter = piece.role === "knight" ? "n" : piece.role[0]; + const label = piece.color === "white" ? roleLetter.toUpperCase() : roleLetter; + const expectedScore = scores[LABEL_MAP[label]]; + if (expectedScore < 0.32 || expectedScore + 0.08 < bestScore) return false; + } + return true; +}; + const updateState = (state: number[][], update: number[][], decay: number = 0.5) => { for (let i = 0; i < 64; i++) { for (let j = 0; j < 12; j++) { @@ -176,7 +194,7 @@ export const getKeypoints = (cornersRef: any, canvasRef: any): number[][] => { export const findPieces = (modelRef: any, videoRef: any, canvasRef: any, playingRef: any, setText: any, dispatch: any, cornersRef: any, boardRef: any, - movesPairsRef: any, lastMoveRef: any, moveTextRef: any, mode: Mode) => { + movesPairsRef: any, lastMoveRef: any, moveTextRef: any, syncRequiredRef: any, mode: Mode) => { let centers: number[][] | null = null; let boundary: number[][]; let centers3D: tf.Tensor3D; @@ -187,6 +205,7 @@ export const findPieces = (modelRef: any, videoRef: any, canvasRef: any, let requestId: number; let greedyMoveToTime: { [move: string]: number }; let active = true; + let synchronizedFrames = 0; const loop = async () => { try { @@ -209,6 +228,26 @@ export const findPieces = (modelRef: any, videoRef: any, canvasRef: any, const squares: number[] = getSquares(boxes, centers3D, boundary3D); const update: number[][] = getUpdate(scores, squares); state = updateState(state, update); + + if (syncRequiredRef.current) { + synchronizedFrames = physicalBoardMatches(state, boardRef.current) + ? synchronizedFrames + 1 + : 0; + setText(["Desynchronization detected", "Rearrange the pieces to match Lichess"]); + if (synchronizedFrames >= 6) { + syncRequiredRef.current = false; + synchronizedFrames = 0; + possibleMoves.clear(); + greedyMoveToTime = {}; + dispatch(gameSetSyncRequired(false)); + setText(["Board synchronized", "Detection resumed"]); + } + renderState(canvasRef.current, centers, boundary, state); + tf.dispose([boxes, scores]); + return; + } + + synchronizedFrames = 0; const { bestScore1, bestScore2, bestJointScore, bestMove, bestMoves } = processState(state, movesPairsRef.current, possibleMoves); const endTime: number = performance.now(); @@ -228,17 +267,18 @@ export const findPieces = (modelRef: any, videoRef: any, canvasRef: any, let hasGreedyMove: boolean = false; if (bestMove !== null && !(hasMove) && (bestScore1 > 0)) { const move: string = bestMove.sans[0]; - if (!(move in greedyMoveToTime)) { - greedyMoveToTime[move] = endTime; - } + const firstSeen = greedyMoveToTime[move] ?? endTime; + greedyMoveToTime = { [move]: firstSeen }; - const secondElapsed = (endTime - greedyMoveToTime[move]) > 1000; + const secondElapsed = (endTime - firstSeen) > 1000; const newMove = sanToLan(boardRef.current, move) !== lastMoveRef.current; hasGreedyMove = secondElapsed && newMove; if (hasGreedyMove) { boardRef.current.playSan(move); - greedyMoveToTime = { greedyMove: greedyMoveToTime[move] }; + greedyMoveToTime = {}; } + } else if (!hasMove) { + greedyMoveToTime = {}; } if (hasMove || hasGreedyMove) { diff --git a/src/utils/lichess.tsx b/src/utils/lichess.tsx index 8ac3229..0d9b441 100644 --- a/src/utils/lichess.tsx +++ b/src/utils/lichess.tsx @@ -86,7 +86,22 @@ type Account = { username: string }; type Playing = { nowPlaying: unknown[] }; type ImportResult = { id: string; url: string }; type BroadcastPushResult = { games: { error?: string }[] }; -type BoardStreamEvent = { type: string; moves?: string; state?: { moves?: string } }; +export type BoardStreamEvent = { + type: string; + moves?: string; + initialFen?: string; + state?: { moves?: string }; +}; + +export const errorMessage = (error: unknown): string => { + if (error instanceof Error) return error.message; + if (typeof error === 'string') return error; + try { + return JSON.stringify(error); + } catch (_) { + return String(error); + } +}; const setBroadcastlessStudies = async (token: string, username: string, setStudies: (studies: Study[]) => void, broadcasts: Study[]) => { const path = `/api/study/by/${username}`; @@ -171,19 +186,42 @@ export const lichessPushRound = async (token: string, pgn: string, roundId: stri return result; } -export const lichessStreamGame = (token: string, callback: (event: BoardStreamEvent) => void | Promise, gameId: string) => { +export const lichessStreamGame = (token: string, callback: (event: BoardStreamEvent) => void | Promise, gameId: string, + onError?: (error: unknown) => void) => { const path = `/api/board/game/stream/${gameId}`; const controller = new AbortController(); - void fetchResponse(token, path, { - signal: controller.signal, - headers: { Accept: 'application/x-ndjson' } - }) - .then(readStream(callback)) - .catch((error: unknown) => { + + const wait = (milliseconds: number) => new Promise((resolve) => { + const timeout = window.setTimeout(resolve, milliseconds); + controller.signal.addEventListener('abort', () => { + window.clearTimeout(timeout); + resolve(); + }, { once: true }); + }); + + const connect = async () => { + let retryDelay = 1000; + while (!controller.signal.aborted) { + try { + const response = await fetchResponse(token, path, { + signal: controller.signal, + headers: { Accept: 'application/x-ndjson' } + }); + retryDelay = 1000; + await readStream(callback)(response); + } catch (error: unknown) { + if (controller.signal.aborted) break; + console.error('Lichess game stream failed; reconnecting.', error); + onError?.(error); + } if (!controller.signal.aborted) { - console.error('Lichess game stream failed.', error); + await wait(retryDelay); + retryDelay = Math.min(retryDelay * 2, 10000); } - }); + } + }; + + void connect(); return controller; } diff --git a/src/utils/moves.tsx b/src/utils/moves.tsx index 1aa767e..9da4345 100644 --- a/src/utils/moves.tsx +++ b/src/utils/moves.tsx @@ -6,7 +6,7 @@ import { kingCastlesTo } from "chessops/util"; import { SQUARE_MAP, LABEL_MAP, SQUARE_NAMES } from "./constants"; import { MovesData, MovesPair } from "../types"; -function* legalMoves(pos: Position): Generator { +export function* legalMoves(pos: Position): Generator { const ctx = pos.ctx(); for (const [from, dests] of pos.allDests(ctx)) { for (const to of dests) { diff --git a/src/utils/voice.ts b/src/utils/voice.ts new file mode 100644 index 0000000..4844e24 --- /dev/null +++ b/src/utils/voice.ts @@ -0,0 +1,220 @@ +import { Move, Role, isNormal } from "chessops/types"; +import { SQUARE_NAMES } from "./constants"; +import { legalMoves } from "./moves"; + +export type VoiceMoveResolution = { + move?: Move; + message: string; + normalized: string; +}; + +const roleNames: Array<[RegExp, Role]> = [ + [/\b(?:rey)\b/, "king"], + [/\b(?:dama|reina)\b/, "queen"], + [/\b(?:torre)\b/, "rook"], + [/\b(?:alfil)\b/, "bishop"], + [/\b(?:caballo)\b/, "knight"], + [/\b(?:peon)\b/, "pawn"] +]; + +const spokenTokens: Record = { + "be": "b", "ce": "c", "de": "d", "efe": "f", "ge": "g", "hache": "h", + "uno": "1", "un": "1", "dos": "2", "tres": "3", "cuatro": "4", + "cinco": "5", "seis": "6", "siete": "7", "ocho": "8" +}; + +const englishSpokenTokens: Record = { + "bee": "b", "see": "c", "sea": "c", "dee": "d", "eff": "f", "gee": "g", "aitch": "h", + "one": "1", "two": "2", "three": "3", "four": "4", + "five": "5", "six": "6", "seven": "7", "eight": "8" +}; + +export const normalizeSpanishChessCommand = (transcript: string) => transcript + .toLowerCase() + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .replace(/[^a-z0-9\s]/g, " ") + .trim() + .split(/\s+/) + .map(token => spokenTokens[token] ?? token) + .join(" "); + +export const normalizeEnglishChessCommand = (transcript: string) => transcript + .toLowerCase() + .replace(/[^a-z0-9\s]/g, " ") + .trim() + .split(/\s+/) + .map(token => englishSpokenTokens[token] ?? token) + .join(" "); + +const getRole = (command: string): Role | undefined => + roleNames.find(([pattern]) => pattern.test(command))?.[1]; + +const getPromotion = (command: string): Role | undefined => { + if (/\b(?:dama|reina)\b/.test(command)) return "queen"; + if (/\btorre\b/.test(command)) return "rook"; + if (/\balfil\b/.test(command)) return "bishop"; + if (/\bcaballo\b/.test(command)) return "knight"; + return undefined; +}; + +const getSquares = (command: string): number[] => { + const squares: number[] = []; + const pattern = /(?:^|\s)([a-h])\s*([1-8])(?=\s|$)/g; + let match: RegExpExecArray | null; + while ((match = pattern.exec(command)) !== null) { + const square = SQUARE_NAMES.indexOf(`${match[1]}${match[2]}` as typeof SQUARE_NAMES[number]); + if (square >= 0) squares.push(square); + } + return squares; +}; + +const originList = (moves: Move[], separator = " o ") => moves + .filter(isNormal) + .map(move => SQUARE_NAMES[move.from].toUpperCase()) + .filter((square, index, all) => all.indexOf(square) === index) + .join(separator); + +export const resolveSpanishVoiceMove = (board: any, transcript: string): VoiceMoveResolution => { + const normalized = normalizeSpanishChessCommand(transcript); + const moves = Array.from(legalMoves(board)).filter(isNormal); + + const castling = normalized.match(/\benroque\s+(corto|largo)\b/); + if (castling) { + const targetFile = castling[1] === "corto" ? 6 : 2; + const candidates = moves.filter(move => { + const piece = board.board.get(move.from); + return piece?.role === "king" && move.to % 8 === targetFile; + }); + return candidates.length === 1 + ? { move: candidates[0], message: `Enroque ${castling[1]}`, normalized } + : { message: `El enroque ${castling[1]} no es legal en esta posición`, normalized }; + } + + const squares = getSquares(normalized); + if (squares.length >= 2) { + const [from, to] = squares; + let candidates = moves.filter(move => move.from === from && move.to === to); + if (candidates.length > 1) { + const promotion = getPromotion(normalized) ?? "queen"; + candidates = candidates.filter(move => move.promotion === promotion); + } + if (candidates.length === 1) { + return { + move: candidates[0], + message: `${SQUARE_NAMES[from].toUpperCase()} a ${SQUARE_NAMES[to].toUpperCase()}`, + normalized + }; + } + return { + message: `${SQUARE_NAMES[from].toUpperCase()} a ${SQUARE_NAMES[to].toUpperCase()} no es una jugada legal`, + normalized + }; + } + + const role = getRole(normalized); + if (role && squares.length === 1) { + const target = squares[0]; + const candidates = moves.filter(move => board.board.get(move.from)?.role === role && move.to === target); + if (candidates.length === 1) { + return { move: candidates[0], message: `${normalized} reconocido`, normalized }; + } + if (candidates.length > 1) { + return { + message: `Jugada ambigua: indica el origen (${originList(candidates)}) y el destino`, + normalized + }; + } + return { message: `No hay una jugada legal que corresponda a “${normalized}”`, normalized }; + } + + return { + message: "No entendí el comando. Prueba “alfil C3” o “C4 a C5”", + normalized + }; +}; + +const englishRoleNames: Array<[RegExp, Role]> = [ + [/\bking\b/, "king"], + [/\bqueen\b/, "queen"], + [/\brook\b/, "rook"], + [/\bbishop\b/, "bishop"], + [/\bknight\b/, "knight"], + [/\bpawn\b/, "pawn"] +]; + +const getEnglishRole = (command: string): Role | undefined => + englishRoleNames.find(([pattern]) => pattern.test(command))?.[1]; + +const getEnglishPromotion = (command: string): Role | undefined => { + if (/\bqueen\b/.test(command)) return "queen"; + if (/\brook\b/.test(command)) return "rook"; + if (/\bbishop\b/.test(command)) return "bishop"; + if (/\bknight\b/.test(command)) return "knight"; + return undefined; +}; + +export const resolveEnglishVoiceMove = (board: any, transcript: string): VoiceMoveResolution => { + const normalized = normalizeEnglishChessCommand(transcript); + const moves = Array.from(legalMoves(board)).filter(isNormal); + + const castling = normalized.match(/\b(?:castle|castling)\s+(king\s*side|queen\s*side)\b/); + if (castling) { + const side = castling[1].replace(/\s/g, ""); + const targetFile = side === "kingside" ? 6 : 2; + const candidates = moves.filter(move => { + const piece = board.board.get(move.from); + return piece?.role === "king" && move.to % 8 === targetFile; + }); + return candidates.length === 1 + ? { move: candidates[0], message: `${side === "kingside" ? "Kingside" : "Queenside"} castle`, normalized } + : { message: `${side === "kingside" ? "Kingside" : "Queenside"} castling is not legal`, normalized }; + } + + const squares = getSquares(normalized); + if (squares.length >= 2) { + const [from, to] = squares; + let candidates = moves.filter(move => move.from === from && move.to === to); + if (candidates.length > 1) { + const promotion = getEnglishPromotion(normalized) ?? "queen"; + candidates = candidates.filter(move => move.promotion === promotion); + } + if (candidates.length === 1) { + return { + move: candidates[0], + message: `${SQUARE_NAMES[from].toUpperCase()} to ${SQUARE_NAMES[to].toUpperCase()}`, + normalized + }; + } + return { + message: `${SQUARE_NAMES[from].toUpperCase()} to ${SQUARE_NAMES[to].toUpperCase()} is not a legal move`, + normalized + }; + } + + const role = getEnglishRole(normalized); + if (role && squares.length === 1) { + const target = squares[0]; + const candidates = moves.filter(move => board.board.get(move.from)?.role === role && move.to === target); + if (candidates.length === 1) { + return { move: candidates[0], message: `${normalized} recognized`, normalized }; + } + if (candidates.length > 1) { + return { + message: `Ambiguous move: say the origin (${originList(candidates, " or ")}) and destination`, + normalized + }; + } + return { message: `There is no legal move matching “${normalized}”`, normalized }; + } + + return { + message: "I did not understand. Try “bishop C3” or “C4 to C5”", + normalized + }; +}; + +export const resolveVoiceMove = (board: any, transcript: string, language: "es-ES" | "en-US") => + language === "en-US" + ? resolveEnglishVoiceMove(board, transcript) + : resolveSpanishVoiceMove(board, transcript);