From 5cabc032d92c7d3da737570a114a0077a8f3bebc Mon Sep 17 00:00:00 2001 From: Sanan507 <227714367+Sanan507@users.noreply.github.com> Date: Tue, 4 Aug 2026 07:09:19 +0000 Subject: [PATCH] feat(audio): implement pitch synthesizer for visualization feedback --- .jules/palette.md | 3 ++ frontend/src/App.tsx | 4 +- frontend/src/context/AudioContext.tsx | 6 +-- frontend/src/hooks/useAudioSettings.ts | 2 + frontend/src/hooks/usePlayback.ts | 52 +++++++++++++++++++++++--- frontend/src/hooks/useSound.ts | 41 +++++++++++++------- frontend/src/pages/PathfindingPage.tsx | 20 ++-------- frontend/src/pages/SearchingPage.tsx | 22 ++--------- frontend/src/pages/SortingPage.tsx | 20 ++-------- 9 files changed, 95 insertions(+), 75 deletions(-) create mode 100644 .jules/palette.md diff --git a/.jules/palette.md b/.jules/palette.md new file mode 100644 index 0000000..e2fdb2f --- /dev/null +++ b/.jules/palette.md @@ -0,0 +1,3 @@ +## 2024-05-30 - Web Audio API Synthesis +**Learning:** Adding sound variation (e.g. mapping values to pentatonic scales) increases user engagement and makes abstract data processes sound more natural and musical. +**Action:** Used `OscillatorNode` dynamically mapping array values to an A-minor pentatonic scale (220Hz - 880Hz) with differing wave types (sine/triangle) and attack velocities for `compare` vs `swap` actions to improve UX. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index d5b84c1..e20c7b0 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -58,7 +58,7 @@ export default function App() { const [mobileMenuOpen, setMobileMenuOpen] = useState(false); const { settings: audioSettings, setSettings: setAudioSettings } = useAudioSettings(); - const { play, playValueTone } = useSound(audioSettings); + const { play, playToneForValue } = useSound(audioSettings); // Listen to browser back/forward and URL hash updates useEffect(() => { @@ -131,7 +131,7 @@ export default function App() { return ( - + {active === 'landing' ? ( ) : ( diff --git a/frontend/src/context/AudioContext.tsx b/frontend/src/context/AudioContext.tsx index 0c795ac..3129646 100644 --- a/frontend/src/context/AudioContext.tsx +++ b/frontend/src/context/AudioContext.tsx @@ -4,15 +4,15 @@ import type { AudioSettings } from '../hooks/useAudioSettings'; export interface AudioContextValue { play: (name: SoundName) => void; - playValueTone?: (value: number, maxVal?: number) => void; + playToneForValue?: (value: number, minValue: number, maxValue: number, type: 'compare' | 'swap') => void; audioSettings: AudioSettings; setAudioSettings: (patch: Partial) => void; } export const AudioCtx = createContext({ play: () => {}, - playValueTone: () => {}, - audioSettings: { soundEnabled: true, masterVolume: 0.6, effectsVolume: 0.7 }, + playToneForValue: () => {}, + audioSettings: { soundEnabled: true, synthEnabled: true, masterVolume: 0.6, effectsVolume: 0.7 }, setAudioSettings: () => {}, }); diff --git a/frontend/src/hooks/useAudioSettings.ts b/frontend/src/hooks/useAudioSettings.ts index 5041673..7037c84 100644 --- a/frontend/src/hooks/useAudioSettings.ts +++ b/frontend/src/hooks/useAudioSettings.ts @@ -2,6 +2,7 @@ import { useCallback, useEffect, useState } from 'react'; export interface AudioSettings { soundEnabled: boolean; + synthEnabled: boolean; masterVolume: number; // 0–1 effectsVolume: number; // 0–1 } @@ -10,6 +11,7 @@ const STORAGE_KEY = 'algorace:audio'; const DEFAULTS: AudioSettings = { soundEnabled: true, + synthEnabled: true, masterVolume: 0.6, effectsVolume: 0.7, }; diff --git a/frontend/src/hooks/usePlayback.ts b/frontend/src/hooks/usePlayback.ts index 69c8e19..9146281 100644 --- a/frontend/src/hooks/usePlayback.ts +++ b/frontend/src/hooks/usePlayback.ts @@ -1,5 +1,6 @@ import { useEffect, useMemo, useState } from 'react'; import type { RaceResponse } from '../models/types'; +import { useAudio } from '../context/AudioContext'; export type FrameEvent = 'compare' | 'swap' | 'hit' | 'miss' | 'step'; @@ -10,6 +11,7 @@ export function usePlayback( ) { const [playing, setPlaying] = useState(false); const [frameIndex, setFrameIndex] = useState(0); + const { play, playToneForValue, audioSettings } = useAudio(); const maxFrames = useMemo(() => { if (!response?.lanes || response.lanes.length === 0) return 0; @@ -40,12 +42,23 @@ export function usePlayback( return current; } const next = current + 1; - if (onFrame && response) { + if (response) { let hasSwap = false; let hasHit = false; let hasMiss = false; let isAnyLaneActive = false; + let activeValue = 0; + let minVal = 0; + let maxVal = 100; + + if (response.dataset && response.dataset.length > 0) { + minVal = Math.min(...response.dataset); + maxVal = Math.max(...response.dataset); + } + + let extractedValue = false; + for (const lane of response.lanes) { if (next < lane.frames.length) { const frame = lane.frames[next] as Record; @@ -63,25 +76,52 @@ export function usePlayback( if (frame.found === false && frame.done === true) { hasMiss = true; } + + if (!extractedValue && frame.array && Array.isArray(frame.array)) { + if (frame.comparing && Array.isArray(frame.comparing) && frame.comparing.length > 0) { + activeValue = frame.array[frame.comparing[0]]; + extractedValue = true; + } else if (frame.highlight && Array.isArray(frame.highlight) && frame.highlight.length > 0) { + activeValue = frame.array[frame.highlight[0]]; + extractedValue = true; + } + } } } } + if (!extractedValue) { + activeValue = Math.floor(Math.random() * (maxVal - minVal + 1)) + minVal; + } + if (hasHit) { - onFrame('hit', next); + if (onFrame) onFrame('hit', next); + if (response.type === 'pathfinding') play('pathFound'); + else play('searchHit'); } else if (hasSwap) { - onFrame('swap', next); + if (onFrame) onFrame('swap', next); + if (audioSettings.synthEnabled && playToneForValue) { + playToneForValue(activeValue, minVal, maxVal, 'swap'); + } else { + play('swap'); + } } else if (hasMiss) { - onFrame('miss', next); + if (onFrame) onFrame('miss', next); + play('searchMiss'); } else if (isAnyLaneActive) { - onFrame('compare', next); + if (onFrame) onFrame('compare', next); + if (audioSettings.synthEnabled && playToneForValue) { + playToneForValue(activeValue, minVal, maxVal, 'compare'); + } else { + play('compare'); + } } } return next; }); }, delay); return () => window.clearInterval(id); - }, [playing, maxFrames, speed, onFrame, response]); + }, [playing, maxFrames, speed, onFrame, response, play, playToneForValue, audioSettings.synthEnabled]); function stepForward() { setPlaying(false); diff --git a/frontend/src/hooks/useSound.ts b/frontend/src/hooks/useSound.ts index 78b0bf2..807e95f 100644 --- a/frontend/src/hooks/useSound.ts +++ b/frontend/src/hooks/useSound.ts @@ -187,40 +187,53 @@ export function useSound(settings: AudioSettings) { }; }, []); - const playValueTone = useCallback( - (value: number, maxValue: number = 100) => { - if (!settings.soundEnabled) return; + const playToneForValue = useCallback( + (value: number, minValue: number, maxValue: number, type: 'compare' | 'swap') => { + if (!settings.soundEnabled || !settings.synthEnabled) return; try { const ctx = ensureCtx(); if (!ctx || !masterGainRef.current) return; - // Map value ratio (0..1) to A-minor pentatonic scale frequencies (220Hz to 880Hz) - const ratio = Math.max(0, Math.min(1, value / Math.max(1, maxValue))); - const baseFreq = 220; // A3 - const octaveMultiplier = Math.pow(2, ratio * 2); // 2 octaves range - const freq = baseFreq * octaveMultiplier; + // Map value to pentatonic scale frequencies between 220Hz and 880Hz + const range = Math.max(1, maxValue - minValue); + const ratio = Math.max(0, Math.min(1, (value - minValue) / range)); + + // A-minor pentatonic notes from A3 (220Hz) to A5 (880Hz) + // 11 distinct frequencies across 2 octaves + const freqs = [ + 220.00, 261.63, 293.66, 329.63, 392.00, + 440.00, 523.25, 587.33, 659.25, 783.99, + 880.00 + ]; + + const noteIndex = Math.floor(ratio * (freqs.length - 1)); + const freq = freqs[noteIndex]; const osc = ctx.createOscillator(); const gain = ctx.createGain(); - osc.type = 'sine'; + const isSwap = type === 'swap'; + osc.type = isSwap ? 'triangle' : 'sine'; osc.frequency.setValueAtTime(freq, ctx.currentTime); - const peakGain = 0.25 * settings.effectsVolume; + // Swaps produce higher velocity tones + const peakGain = (isSwap ? 0.45 : 0.25) * settings.effectsVolume; + const duration = isSwap ? 0.12 : 0.08; + gain.gain.setValueAtTime(peakGain, ctx.currentTime); - gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + 0.08); + gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + duration); osc.connect(gain); gain.connect(masterGainRef.current); osc.start(ctx.currentTime); - osc.stop(ctx.currentTime + 0.09); + osc.stop(ctx.currentTime + duration + 0.01); } catch { // silent fallback } }, - [settings.soundEnabled, settings.effectsVolume, ensureCtx] + [settings.soundEnabled, settings.synthEnabled, settings.effectsVolume, ensureCtx] ); - return { play, playValueTone }; + return { play, playToneForValue }; } diff --git a/frontend/src/pages/PathfindingPage.tsx b/frontend/src/pages/PathfindingPage.tsx index f6d7563..96cc2e5 100644 --- a/frontend/src/pages/PathfindingPage.tsx +++ b/frontend/src/pages/PathfindingPage.tsx @@ -42,7 +42,7 @@ export function PathfindingPage({ catalog }: { catalog: CatalogResponse }) { const [speed, setSpeed] = useState(6); const [loading, setLoading] = useState(false); - const { play, playValueTone } = useAudio(); + const { play } = useAudio(); const winnerAnnouncedRef = useRef(false); const initialized = useRef(false); const latestFetchIdRef = useRef(0); @@ -57,21 +57,9 @@ export function PathfindingPage({ catalog }: { catalog: CatalogResponse }) { return defaultMazeTypes; }, [catalog]); - const onFrame = useCallback( - (event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => { - if (event === 'hit') { - play('pathFound'); - } else { - if (playValueTone) { - const val = Math.floor(Math.random() * 70) + 20; - playValueTone(val, 100); - } else { - play('compare'); - } - } - }, - [play, playValueTone] - ); + const onFrame = useCallback((event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => { + // Audio is now handled centrally in usePlayback hook + }, []); const playback = usePlayback(response, speed, onFrame); diff --git a/frontend/src/pages/SearchingPage.tsx b/frontend/src/pages/SearchingPage.tsx index c6fb764..e31b34f 100644 --- a/frontend/src/pages/SearchingPage.tsx +++ b/frontend/src/pages/SearchingPage.tsx @@ -32,7 +32,7 @@ export function SearchingPage({ catalog }: { catalog: CatalogResponse }) { const [validationError, setValidationError] = useState(null); const [isModalOpen, setIsModalOpen] = useState(false); - const { play, playValueTone } = useAudio(); + const { play } = useAudio(); const winnerAnnouncedRef = useRef(false); const requestIdRef = useRef(0); const initialized = useRef(false); @@ -170,23 +170,9 @@ export function SearchingPage({ catalog }: { catalog: CatalogResponse }) { }; }, [isCustomMode, parsedCustomArray, response, algorithms, target, catalog, size]); - const onFrame = useCallback( - (event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => { - if (event === 'hit') { - play('searchHit'); - } else if (event === 'miss') { - play('searchMiss'); - } else if (event === 'compare') { - if (playValueTone) { - const val = Math.floor(Math.random() * 80) + 15; - playValueTone(val, 100); - } else { - play('compare'); - } - } - }, - [play, playValueTone] - ); + const onFrame = useCallback((event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => { + // Audio is now handled centrally in usePlayback hook + }, []); const playback = usePlayback(activeResponse, speed, onFrame); diff --git a/frontend/src/pages/SortingPage.tsx b/frontend/src/pages/SortingPage.tsx index 27d541f..9cf3764 100644 --- a/frontend/src/pages/SortingPage.tsx +++ b/frontend/src/pages/SortingPage.tsx @@ -31,7 +31,7 @@ export function SortingPage({ catalog }: { catalog: CatalogResponse }) { const [speed, setSpeed] = useState(6); const [validationError, setValidationError] = useState(null); - const { play, playValueTone } = useAudio(); + const { play } = useAudio(); const winnerAnnouncedRef = useRef(false); const requestIdRef = useRef(0); const initialized = useRef(false); @@ -177,21 +177,9 @@ export function SortingPage({ catalog }: { catalog: CatalogResponse }) { }; }, [isCustomMode, parsedCustomArray, response, algorithms, catalog, size]); - const onFrame = useCallback( - (event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => { - if (event === 'swap') { - play('swap'); - } else if (event === 'compare') { - if (playValueTone) { - const val = Math.floor(Math.random() * 80) + 10; - playValueTone(val, 100); - } else { - play('compare'); - } - } - }, - [play, playValueTone] - ); + const onFrame = useCallback((event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => { + // Audio is now handled centrally in usePlayback hook + }, []); const playback = usePlayback(activeResponse, speed, onFrame);