feat: implement web audio pitch synthesizer for visualization feedback

.jules/palette.md

@@ -0,0 +1,3 @@
+## 2024-05-30 - Web Audio API Synthesis
+**Learning:** Adding sound variation (e.g. mapping values to pentatonic scales) increases user engagement and makes abstract data processes sound more natural and musical.
+**Action:** Used `OscillatorNode` dynamically mapping array values to an A-minor pentatonic scale (220Hz - 880Hz) with differing wave types (sine/triangle) and attack velocities for `compare` vs `swap` actions to improve UX.

frontend/src/App.tsx

@@ -58,7 +58,7 @@ export default function App() {
   const [mobileMenuOpen, setMobileMenuOpen] = useState(false);
 
   const { settings: audioSettings, setSettings: setAudioSettings } = useAudioSettings();
-  const { play, playValueTone } = useSound(audioSettings);
+  const { play, playToneForValue } = useSound(audioSettings);
 
   // Listen to browser back/forward and URL hash updates
   useEffect(() => {
@@ -131,7 +131,7 @@ export default function App() {
 
   return (
     <ThemeProvider>
-      <AudioCtx.Provider value={{ play, playValueTone, audioSettings, setAudioSettings }}>
+      <AudioCtx.Provider value={{ play, playToneForValue, audioSettings, setAudioSettings }}>
         {active === 'landing' ? (
           <LandingPage onNavigate={setActive} darkMode={darkMode} setDarkMode={setDarkMode} />
         ) : (

frontend/src/context/AudioContext.tsx

@@ -4,15 +4,15 @@ import type { AudioSettings } from '../hooks/useAudioSettings';
 
 export interface AudioContextValue {
   play: (name: SoundName) => void;
-  playValueTone?: (value: number, maxVal?: number) => void;
+  playToneForValue?: (value: number, minValue: number, maxValue: number, type: 'compare' | 'swap') => void;
   audioSettings: AudioSettings;
   setAudioSettings: (patch: Partial<AudioSettings>) => void;
 }
 
 export const AudioCtx = createContext<AudioContextValue>({
   play: () => {},
-  playValueTone: () => {},
-  audioSettings: { soundEnabled: true, masterVolume: 0.6, effectsVolume: 0.7 },
+  playToneForValue: () => {},
+  audioSettings: { soundEnabled: true, synthEnabled: true, masterVolume: 0.6, effectsVolume: 0.7 },
   setAudioSettings: () => {},
 });
 

frontend/src/hooks/useAudioSettings.ts

@@ -2,6 +2,7 @@ import { useCallback, useEffect, useState } from 'react';
 
 export interface AudioSettings {
   soundEnabled: boolean;
+  synthEnabled: boolean;
   masterVolume: number;  // 0–1
   effectsVolume: number; // 0–1
 }
@@ -10,6 +11,7 @@ const STORAGE_KEY = 'algorace:audio';
 
 const DEFAULTS: AudioSettings = {
   soundEnabled: true,
+  synthEnabled: true,
   masterVolume: 0.6,
   effectsVolume: 0.7,
 };

frontend/src/hooks/usePlayback.ts

@@ -1,5 +1,6 @@
 import { useEffect, useMemo, useState } from 'react';
 import type { RaceResponse } from '../models/types';
+import { useAudio } from '../context/AudioContext';
 
 export type FrameEvent = 'compare' | 'swap' | 'hit' | 'miss' | 'step';
 
@@ -10,6 +11,7 @@ export function usePlayback(
 ) {
   const [playing, setPlaying] = useState(false);
   const [frameIndex, setFrameIndex] = useState(0);
+  const { play, playToneForValue, audioSettings } = useAudio();
 
   const maxFrames = useMemo(() => {
     if (!response?.lanes || response.lanes.length === 0) return 0;
@@ -40,12 +42,23 @@ export function usePlayback(
           return current;
         }
         const next = current + 1;
-        if (onFrame && response) {
+        if (response) {
           let hasSwap = false;
           let hasHit = false;
           let hasMiss = false;
           let isAnyLaneActive = false;
 
+          let activeValue = 0;
+          let minVal = 0;
+          let maxVal = 100;
+
+          if (response.dataset && response.dataset.length > 0) {
+            minVal = Math.min(...response.dataset);
+            maxVal = Math.max(...response.dataset);
+          }
+
+          let extractedValue = false;
+
           for (const lane of response.lanes) {
             if (next < lane.frames.length) {
               const frame = lane.frames[next] as Record<string, unknown>;
@@ -63,25 +76,52 @@ export function usePlayback(
                 if (frame.found === false && frame.done === true) {
                   hasMiss = true;
                 }
+
+                if (!extractedValue && frame.array && Array.isArray(frame.array)) {
+                  if (frame.comparing && Array.isArray(frame.comparing) && frame.comparing.length > 0) {
+                    activeValue = frame.array[frame.comparing[0]];
+                    extractedValue = true;
+                  } else if (frame.highlight && Array.isArray(frame.highlight) && frame.highlight.length > 0) {
+                    activeValue = frame.array[frame.highlight[0]];
+                    extractedValue = true;
+                  }
+                }
               }
             }
           }
 
+          if (!extractedValue) {
+            activeValue = Math.floor(Math.random() * (maxVal - minVal + 1)) + minVal;
+          }
+
           if (hasHit) {
-            onFrame('hit', next);
+            if (onFrame) onFrame('hit', next);
+            if (response.type === 'pathfinding') play('pathFound');
+            else play('searchHit');
           } else if (hasSwap) {
-            onFrame('swap', next);
+            if (onFrame) onFrame('swap', next);
+            if (audioSettings.synthEnabled && playToneForValue) {
+              playToneForValue(activeValue, minVal, maxVal, 'swap');
+            } else {
+              play('swap');
+            }
           } else if (hasMiss) {
-            onFrame('miss', next);
+            if (onFrame) onFrame('miss', next);
+            play('searchMiss');
           } else if (isAnyLaneActive) {
-            onFrame('compare', next);
+            if (onFrame) onFrame('compare', next);
+            if (audioSettings.synthEnabled && playToneForValue) {
+              playToneForValue(activeValue, minVal, maxVal, 'compare');
+            } else {
+              play('compare');
+            }
           }
         }
         return next;
       });
     }, delay);
     return () => window.clearInterval(id);
-  }, [playing, maxFrames, speed, onFrame, response]);
+  }, [playing, maxFrames, speed, onFrame, response, play, playToneForValue, audioSettings.synthEnabled]);
 
   function stepForward() {
     setPlaying(false);

frontend/src/hooks/useSound.ts

@@ -187,40 +187,53 @@ export function useSound(settings: AudioSettings) {
     };
   }, []);
 
-  const playValueTone = useCallback(
-    (value: number, maxValue: number = 100) => {
-      if (!settings.soundEnabled) return;
+  const playToneForValue = useCallback(
+    (value: number, minValue: number, maxValue: number, type: 'compare' | 'swap') => {
+      if (!settings.soundEnabled || !settings.synthEnabled) return;
       try {
         const ctx = ensureCtx();
         if (!ctx || !masterGainRef.current) return;
 
-        // Map value ratio (0..1) to A-minor pentatonic scale frequencies (220Hz to 880Hz)
-        const ratio = Math.max(0, Math.min(1, value / Math.max(1, maxValue)));
-        const baseFreq = 220; // A3
-        const octaveMultiplier = Math.pow(2, ratio * 2); // 2 octaves range
-        const freq = baseFreq * octaveMultiplier;
+        // Map value to pentatonic scale frequencies between 220Hz and 880Hz
+        const range = Math.max(1, maxValue - minValue);
+        const ratio = Math.max(0, Math.min(1, (value - minValue) / range));
+
+        // A-minor pentatonic notes from A3 (220Hz) to A5 (880Hz)
+        // 11 distinct frequencies across 2 octaves
+        const freqs = [
+          220.00, 261.63, 293.66, 329.63, 392.00,
+          440.00, 523.25, 587.33, 659.25, 783.99,
+          880.00
+        ];
+
+        const noteIndex = Math.floor(ratio * (freqs.length - 1));
+        const freq = freqs[noteIndex];
 
         const osc = ctx.createOscillator();
         const gain = ctx.createGain();
 
-        osc.type = 'sine';
+        const isSwap = type === 'swap';
+        osc.type = isSwap ? 'triangle' : 'sine';
         osc.frequency.setValueAtTime(freq, ctx.currentTime);
 
-        const peakGain = 0.25 * settings.effectsVolume;
+        // Swaps produce higher velocity tones
+        const peakGain = (isSwap ? 0.45 : 0.25) * settings.effectsVolume;
+        const duration = isSwap ? 0.12 : 0.08;
+
         gain.gain.setValueAtTime(peakGain, ctx.currentTime);
-        gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + 0.08);
+        gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + duration);
 
         osc.connect(gain);
         gain.connect(masterGainRef.current);
 
         osc.start(ctx.currentTime);
-        osc.stop(ctx.currentTime + 0.09);
+        osc.stop(ctx.currentTime + duration + 0.01);
       } catch {
         // silent fallback
       }
     },
-    [settings.soundEnabled, settings.effectsVolume, ensureCtx]
+    [settings.soundEnabled, settings.synthEnabled, settings.effectsVolume, ensureCtx]
   );
 
-  return { play, playValueTone };
+  return { play, playToneForValue };
 }

frontend/src/pages/PathfindingPage.tsx

@@ -42,7 +42,7 @@ export function PathfindingPage({ catalog }: { catalog: CatalogResponse }) {
   const [speed, setSpeed] = useState(6);
   const [loading, setLoading] = useState(false);
 
-  const { play, playValueTone } = useAudio();
+  const { play } = useAudio();
   const winnerAnnouncedRef = useRef(false);
   const initialized = useRef(false);
   const latestFetchIdRef = useRef(0);
@@ -57,21 +57,9 @@ export function PathfindingPage({ catalog }: { catalog: CatalogResponse }) {
     return defaultMazeTypes;
   }, [catalog]);
 
-  const onFrame = useCallback(
-    (event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => {
-      if (event === 'hit') {
-        play('pathFound');
-      } else {
-        if (playValueTone) {
-          const val = Math.floor(Math.random() * 70) + 20;
-          playValueTone(val, 100);
-        } else {
-          play('compare');
-        }
-      }
-    },
-    [play, playValueTone]
-  );
+  const onFrame = useCallback((event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => {
+    // Audio is now handled centrally in usePlayback hook
+  }, []);
 
   const playback = usePlayback(response, speed, onFrame);
 

frontend/src/pages/SearchingPage.tsx

@@ -32,7 +32,7 @@ export function SearchingPage({ catalog }: { catalog: CatalogResponse }) {
   const [validationError, setValidationError] = useState<string | null>(null);
   const [isModalOpen, setIsModalOpen] = useState(false);
 
-  const { play, playValueTone } = useAudio();
+  const { play } = useAudio();
   const winnerAnnouncedRef = useRef(false);
   const requestIdRef = useRef(0);
   const initialized = useRef(false);
@@ -170,23 +170,9 @@ export function SearchingPage({ catalog }: { catalog: CatalogResponse }) {
     };
   }, [isCustomMode, parsedCustomArray, response, algorithms, target, catalog, size]);
 
-  const onFrame = useCallback(
-    (event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => {
-      if (event === 'hit') {
-        play('searchHit');
-      } else if (event === 'miss') {
-        play('searchMiss');
-      } else if (event === 'compare') {
-        if (playValueTone) {
-          const val = Math.floor(Math.random() * 80) + 15;
-          playValueTone(val, 100);
-        } else {
-          play('compare');
-        }
-      }
-    },
-    [play, playValueTone]
-  );
+  const onFrame = useCallback((event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => {
+    // Audio is now handled centrally in usePlayback hook
+  }, []);
 
   const playback = usePlayback(activeResponse, speed, onFrame);
 

frontend/src/pages/SortingPage.tsx

@@ -31,7 +31,7 @@ export function SortingPage({ catalog }: { catalog: CatalogResponse }) {
   const [speed, setSpeed] = useState(6);
   const [validationError, setValidationError] = useState<string | null>(null);
 
-  const { play, playValueTone } = useAudio();
+  const { play } = useAudio();
   const winnerAnnouncedRef = useRef(false);
   const requestIdRef = useRef(0);
   const initialized = useRef(false);
@@ -177,21 +177,9 @@ export function SortingPage({ catalog }: { catalog: CatalogResponse }) {
     };
   }, [isCustomMode, parsedCustomArray, response, algorithms, catalog, size]);
 
-  const onFrame = useCallback(
-    (event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => {
-      if (event === 'swap') {
-        play('swap');
-      } else if (event === 'compare') {
-        if (playValueTone) {
-          const val = Math.floor(Math.random() * 80) + 10;
-          playValueTone(val, 100);
-        } else {
-          play('compare');
-        }
-      }
-    },
-    [play, playValueTone]
-  );
+  const onFrame = useCallback((event: 'compare' | 'swap' | 'hit' | 'miss' | 'step') => {
+    // Audio is now handled centrally in usePlayback hook
+  }, []);
 
   const playback = usePlayback(activeResponse, speed, onFrame);