// Lyria RealTime MusicManager — Drop-in replacement for Tone.js synth engine // Sends gesture data to Python backend via WebSocket, receives PCM audio export class MusicManager { constructor() { this.isStarted = false; this._panicMuted = false; this.analyser = null; this.currentStyleLabel = ''; this._handFists = [false, false]; // Track fist state per hand this._bassDropActive = false; // True while both fists held // Spacebar listener — skip if typing in an input document.addEventListener('keydown', (e) => { if (e.code === 'Space' || e.key === ' ') { if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') return; e.preventDefault(); e.stopPropagation(); this.panic(); } }, true); // Genre override — persists even when legend is closed this._genreOverride = ''; window._setGenreOverride = (genre) => { this._genreOverride = genre.trim(); if (this._wsReady) { this._ws.send(JSON.stringify({ type: 'genre_override', genre: this._genreOverride })); } // Show/hide indicator let el = document.getElementById('genre-lock-indicator'); if (!el) { el = document.createElement('div'); el.id = 'genre-lock-indicator'; el.style.cssText = 'position:fixed;top:12px;left:12px;padding:4px 10px;border-radius:6px;font:11px monospace;z-index:9999;pointer-events:none;background:rgba(0,180,255,0.7);color:#000;transition:opacity 0.3s'; document.body.appendChild(el); } if (this._genreOverride) { el.textContent = '\uD83C\uDFB5 ' + this._genreOverride.toUpperCase(); el.style.opacity = '1'; console.log('[Lyria] Genre locked:', this._genreOverride); } else { el.style.opacity = '0'; console.log('[Lyria] Genre lock cleared'); } }; // Gesture state accumulator (sent to backend) this._gestureState = { handY: 0.5, handSpread: 0.5, wristAngle: 0.0, volume: 0.5, handVelocity: { x: 0, y: 0 }, fingerExtensions: {}, shapeActive: false, shapeProximity: 0.5, drumHandActive: false, drumTempo: null, muteDrums: false, muteBass: false, fistGesture: false, }; // WebSocket to backend this._ws = null; this._wsReady = false; // Audio playback via Web Audio API this._audioCtx = null; this._pcmQueue = []; this._isPlaying = false; this._nextPlayTime = 0; // Analyser for visualizer compatibility this._analyserNode = null; this._analyserData = null; // Throttle gesture sends (every 100ms) this._lastSendTime = 0; this._SEND_INTERVAL = 1000; // Send gesture updates every 1s — Lyria keeps playing between // Compat: pad preset cycling this.padPresets = [ { name: 'Ambient Sub' }, { name: 'Dub Growl' }, { name: 'Dream Wash' }, { name: 'Detuned Saw' }, { name: 'Warm Square' }, ]; this.currentSynthIndex = 0; // Smoothing (compat with game.js references) this._smoothDist = { thumb: 0, index: 0, middle: 0, ring: 0, pinky: 0 }; this._smoothDrumDist = { thumb: 0, index: 0, middle: 0, ring: 0, pinky: 0 }; this._SMOOTH_ALPHA = 0.3; this._frameCount = 0; this._prevExtensions = {}; this._prevDrumExtensions = { thumb: 0, index: 0, middle: 0, ring: 0, pinky: 0 }; this._drumFingerCooldowns = { thumb: 0, index: 0, middle: 0, ring: 0, pinky: 0 }; this.handVolumes = new Map(); this.fingerCooldowns = new Map(); this.percCooldown = 0; this.padSynths = new Map(); this.activePatterns = this.padSynths; this.scale = [ 'C1','Eb1','G1','Bb1','C2','Eb2','F2','G2','Bb2', 'C3','Eb3','F3','G3','Bb3','C4','Eb4','F4','G4','Bb4','C5','Eb5','F5' ]; } _debug(msg) { console.log('[Lyria] ' + msg); try { let el = document.getElementById('lyria-debug'); if (!el && document.body) { el = document.createElement('div'); el.id = 'lyria-debug'; el.style.cssText = 'position:fixed;bottom:4px;left:4px;padding:4px 8px;border-radius:4px;font:10px monospace;z-index:9999;pointer-events:none;background:rgba(0,0,0,0.7);color:#0f0;max-width:500px;word-wrap:break-word'; document.body.appendChild(el); } if (el) el.textContent = msg; } catch(e) { /* DOM not ready */ } } async start() { if (this.isStarted) return; this._debug('start() called'); // Create AudioContext for PCM playback this._audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 48000 }); this._debug('AudioContext: ' + this._audioCtx.state); // Create analyser node for visualizer this._analyserNode = this._audioCtx.createAnalyser(); this._analyserNode.fftSize = 2048; this._analyserNode.connect(this._audioCtx.destination); this._analyserData = new Float32Array(this._analyserNode.fftSize); // Expose analyser in Tone.js-compatible format this.analyser = { getValue: () => { this._analyserNode.getFloatTimeDomainData(this._analyserData); return this._analyserData; } }; // Connect to backend WebSocket with auto-reconnect const wsProto = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; this._wsUrl = `${wsProto}//${window.location.host}/ws`; this._debug('WS URL: ' + this._wsUrl); this._reconnectAttempts = 0; this._maxReconnectAttempts = 10; this._connectWebSocket(); } _connectWebSocket() { this._debug('Connecting WS...'); this._ws = new WebSocket(this._wsUrl); this._ws.binaryType = 'arraybuffer'; this._ws.onopen = () => { this._debug('WS CONNECTED'); this._wsReady = true; this._reconnectAttempts = 0; this._nextPlayTime = 0; // reset audio scheduling on reconnect }; this._ws.onmessage = (event) => { if (event.data instanceof ArrayBuffer) { this._handleAudioChunkBinary(event.data); } else if (event.data instanceof Blob) { event.data.arrayBuffer().then(ab => this._handleAudioChunkBinary(ab)); } else if (typeof event.data === 'string') { try { const msg = JSON.parse(event.data); if (msg.type === 'error') { console.error('[Lyria] Backend error:', msg.message); } else if (msg.type === 'connected') { console.log('[Lyria] Backend session ready'); } else if (msg.type === 'fallback') { this._setFallbackIndicator(msg.active); if (msg.style) { this.currentStyleLabel = msg.style; } } else if (msg.type === 'session_restart') { this._showSessionRestart(msg.phase); } } catch (e) { console.warn('[Lyria] Unparseable message:', event.data.substring(0, 50)); } } }; this._ws.onerror = (e) => { this._debug('WS ERROR: ' + (e.message || 'unknown')); }; this._ws.onclose = (e) => { this._debug('WS CLOSED: code=' + e.code + ' reason=' + e.reason); this._wsReady = false; // Auto-reconnect with backoff if (this._reconnectAttempts < this._maxReconnectAttempts) { const delay = Math.min(1000 * Math.pow(1.5, this._reconnectAttempts), 10000); this._reconnectAttempts++; console.log(`[Lyria] Reconnecting in ${(delay/1000).toFixed(1)}s (attempt ${this._reconnectAttempts})...`); setTimeout(() => this._connectWebSocket(), delay); } else { console.error('[Lyria] Max reconnect attempts reached'); } }; this.isStarted = true; } _handleAudioChunkBinary(arrayBuffer) { // Drop audio chunks while paused if (this._panicMuted) return; if (!this._chunkCount) this._chunkCount = 0; this._chunkCount++; if (this._chunkCount <= 3 || this._chunkCount % 20 === 0) { console.log(`[Lyria] Audio chunk ${this._chunkCount}: ${arrayBuffer.byteLength} bytes`); } // Convert 16-bit PCM to Float32 stereo directly from ArrayBuffer const int16 = new Int16Array(arrayBuffer); const numSamples = int16.length / 2; // stereo const audioBuffer = this._audioCtx.createBuffer(2, numSamples, 48000); const left = audioBuffer.getChannelData(0); const right = audioBuffer.getChannelData(1); for (let i = 0; i < numSamples; i++) { left[i] = int16[i * 2] / 32768; right[i] = int16[i * 2 + 1] / 32768; } // Schedule seamless playback — drop chunks if we fall too far behind const now = this._audioCtx.currentTime; if (this._nextPlayTime > 0 && this._nextPlayTime > now + 6) { console.warn('[Lyria] Audio buffer too far ahead, resetting'); this._nextPlayTime = now + 0.05; } const startTime = Math.max(now + 0.05, this._nextPlayTime); const source = this._audioCtx.createBufferSource(); source.buffer = audioBuffer; source.connect(this._analyserNode); source.start(startTime); // Disconnect source after playback to free memory source.onended = () => { source.disconnect(); }; this._nextPlayTime = startTime + audioBuffer.duration; } _sendGestureState() { if (this._panicMuted || this._bassDropActive) return; // Don't send during pause or bass drop const now = performance.now(); if (!this._wsReady || now - this._lastSendTime < this._SEND_INTERVAL) return; this._lastSendTime = now; this._ws.send(JSON.stringify({ type: 'gesture', data: this._gestureState })); } // === Public API matching original MusicManager interface === startArpeggio(handId, note) { // Track that a hand appeared, update base note this._gestureState.handY = this.scale.indexOf(note) / (this.scale.length - 1); if (!this._isPlaying && this._wsReady) { this._ws.send(JSON.stringify({ type: 'play' })); this._isPlaying = true; } } stopArpeggio(handId) { // Hand disappeared } updateGesture(handId, gestureData) { if (!this.isStarted || this._panicMuted) return; // Map gesture data to our state format const rootNote = gestureData.rootNote || 'C3'; const noteIdx = this.scale.indexOf(rootNote); this._gestureState.handY = noteIdx >= 0 ? noteIdx / (this.scale.length - 1) : 0.5; this._gestureState.volume = gestureData.volume || 0; this._gestureState.wristAngle = gestureData.wristAngle || 0; this._gestureState.handSpread = gestureData.handSpread || 0; this._gestureState.handVelocity = gestureData.handVelocity || { x: 0, y: 0 }; this._gestureState.fingerExtensions = gestureData.fingerExtensions || {}; this._gestureState.fistGesture = false; this._sendGestureState(); } updateDrumGesture(gestureData) { if (!this.isStarted || this._panicMuted) return; this._gestureState.drumHandActive = true; // Average finger extension as tempo control const ext = gestureData.fingerExtensions || {}; const vals = Object.values(ext).filter(v => typeof v === 'number'); if (vals.length > 0) { this._gestureState.drumTempo = vals.reduce((a, b) => a + b, 0) / vals.length; } this._sendGestureState(); } updateShapeBass(shapeData) { if (!this.isStarted) return; this._gestureState.shapeActive = true; this._gestureState.shapeProximity = shapeData?.proximity || 0.5; this._sendGestureState(); } stopShapeBass() { this._gestureState.shapeActive = false; this._sendGestureState(); } triggerFingerTouch(finger1, finger2) { // Finger touches are sent as part of gesture state // The Qwen model will interpret the overall gesture } cycleSynth() { // Synth hand made a fist this._handFists[0] = true; this._checkDoubleFist(); if (!this._bassDropActive) { this.currentSynthIndex = (this.currentSynthIndex + 1) % this.padPresets.length; this._gestureState.fistGesture = true; this._sendGestureState(); } } setDrumHandFist(isFist) { this._handFists[1] = isFist; if (isFist) { this._checkDoubleFist(); } else { this._endDoubleFist(); } } // Called by game.js when synth hand opens from fist setSynthHandOpen() { this._handFists[0] = false; this._endDoubleFist(); } _checkDoubleFist() { if (this._handFists[0] && this._handFists[1] && !this._bassDropActive) { this._bassDropActive = true; console.log('[Lyria] BASS DROP LOCKED!'); if (this._wsReady) { this._ws.send(JSON.stringify({ type: 'bassdrop' })); } } } _endDoubleFist() { if (this._bassDropActive && !(this._handFists[0] && this._handFists[1])) { this._bassDropActive = false; console.log('[Lyria] BASS DROP RELEASED'); if (this._wsReady) { this._ws.send(JSON.stringify({ type: 'bassdrop_end' })); } } } panic() { // Toggle pause/resume this._panicMuted = !this._panicMuted; if (this._panicMuted) { // PAUSE everything console.log('[Lyria] PAUSED'); if (this._wsReady) { this._ws.send(JSON.stringify({ type: 'panic' })); } // Suspend audio context to silence all scheduled buffers if (this._audioCtx && this._audioCtx.state === 'running') { this._audioCtx.suspend(); } // Show pause indicator this._setPauseIndicator(true); } else { // RESUME everything console.log('[Lyria] RESUMED'); if (this._audioCtx && this._audioCtx.state === 'suspended') { this._audioCtx.resume(); } // Reset audio scheduling so we don't play stale buffered chunks this._nextPlayTime = 0; if (this._wsReady) { this._ws.send(JSON.stringify({ type: 'resume' })); } this._setPauseIndicator(false); } } _showSessionRestart(phase) { let el = document.getElementById('lyria-restart-indicator'); if (!el) { el = document.createElement('div'); el.id = 'lyria-restart-indicator'; el.style.cssText = 'position:fixed;top:50%;left:50%;transform:translate(-50%,-50%);font:bold 28px monospace;color:rgba(255,200,0,0.95);z-index:9999;pointer-events:none;text-shadow:0 0 30px rgba(255,150,0,0.6);text-align:center;transition:opacity 0.5s'; document.body.appendChild(el); } if (phase === 'starting') { const msgs = [ 'RELOADING THE COSMIC JUKEBOX...', 'REWINDING THE UNIVERSE...', 'REFUELING THE BASS CANNON...', 'SHUFFLING THE QUANTUM DECK...', 'REBOOTING THE GROOVE MATRIX...', 'TUNING THE HYPERSPACE ANTENNA...', 'SUMMONING FRESH FREQUENCIES...', ]; el.textContent = msgs[Math.floor(Math.random() * msgs.length)]; el.style.opacity = '1'; this._nextPlayTime = 0; // reset audio scheduling } else if (phase === 'done') { el.textContent = 'LOCKED BACK IN'; setTimeout(() => { el.style.opacity = '0'; }, 1500); } } _setPauseIndicator(paused) { let el = document.getElementById('lyria-pause-indicator'); if (!el) { el = document.createElement('div'); el.id = 'lyria-pause-indicator'; el.style.cssText = 'position:fixed;top:50%;left:50%;transform:translate(-50%,-50%);font:bold 48px monospace;color:rgba(255,255,255,0.8);z-index:9999;pointer-events:none;text-shadow:0 0 20px rgba(0,0,0,0.8);transition:opacity 0.3s'; document.body.appendChild(el); } if (paused) { el.textContent = '|| PAUSED'; el.style.opacity = '1'; } else { el.style.opacity = '0'; } } handLeft(handId) { if (handId !== 0) { this._gestureState.drumHandActive = false; this._gestureState.drumTempo = null; } // Notify backend if (this._wsReady) { this._ws.send(JSON.stringify({ type: 'hands_lost' })); } } // === Additional stubs for game.js compatibility === setProximityFilter(proximity) { // Proximity affects brightness in the gesture state this._gestureState.shapeProximity = proximity; } updateArpeggio(handId, note) { const noteIdx = this.scale.indexOf(note); this._gestureState.handY = noteIdx >= 0 ? noteIdx / (this.scale.length - 1) : 0.5; } updateArpeggioVolume(handId, volume) { this._gestureState.volume = volume; } updateFingerExpression(data) { // Finger expression data folded into gesture state via updateGesture } getAnalyser() { return this.analyser; } ensureAudioActive() { if (this._audioCtx && this._audioCtx.state === 'suspended') { this._audioCtx.resume(); } } updateShapeAudio(shapeData) { this.updateShapeBass(shapeData); } getAnalyserData() { if (this.analyser) { return this.analyser.getValue(); } return new Float32Array(2048); } _setFallbackIndicator(active) { let el = document.getElementById('qwen-fallback-indicator'); if (!el) { el = document.createElement('div'); el.id = 'qwen-fallback-indicator'; el.style.cssText = 'position:fixed;top:12px;right:12px;padding:4px 10px;border-radius:6px;font:11px monospace;z-index:9999;pointer-events:none;transition:opacity 0.3s'; document.body.appendChild(el); } if (active) { el.textContent = '\u2699 RULES'; el.style.background = 'rgba(255,160,0,0.7)'; el.style.color = '#000'; el.style.opacity = '1'; } else { el.textContent = '\u2728 QWEN'; el.style.background = 'rgba(0,200,120,0.5)'; el.style.color = '#fff'; el.style.opacity = '0.6'; } } dispose() { if (this._ws) { this._ws.close(); } if (this._audioCtx) { this._audioCtx.close(); } } }