Solshine commited on
Commit
bb63f82
·
verified ·
1 Parent(s): 5f1d376

Upload folder using huggingface_hub

Browse files
Files changed (4) hide show
  1. MusicManager.js +37 -1
  2. backend/server.py +30 -1
  3. game.js +9 -3
  4. index.html +12 -10
MusicManager.js CHANGED
@@ -5,6 +5,18 @@ export class MusicManager {
5
  this.isStarted = false;
6
  this._panicMuted = false;
7
  this.analyser = null;
 
 
 
 
 
 
 
 
 
 
 
 
8
 
9
  // Gesture state accumulator (sent to backend)
10
  this._gestureState = {
@@ -125,6 +137,9 @@ export class MusicManager {
125
  console.log('[Lyria] Backend session ready');
126
  } else if (msg.type === 'fallback') {
127
  this._setFallbackIndicator(msg.active);
 
 
 
128
  }
129
  } catch (e) {
130
  console.warn('[Lyria] Unparseable message:', event.data.substring(0, 50));
@@ -273,10 +288,31 @@ export class MusicManager {
273
  }
274
 
275
  cycleSynth() {
 
 
 
276
  this.currentSynthIndex = (this.currentSynthIndex + 1) % this.padPresets.length;
277
  this._gestureState.fistGesture = true;
278
  this._sendGestureState();
279
- console.log(`[Lyria] Preset: ${this.padPresets[this.currentSynthIndex].name}`);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
280
  }
281
 
282
  panic() {
 
5
  this.isStarted = false;
6
  this._panicMuted = false;
7
  this.analyser = null;
8
+ this.currentStyleLabel = '';
9
+ this._handFists = [false, false]; // Track fist state per hand
10
+ this._doubleFistSent = false; // Prevent repeat sends
11
+
12
+ // Spacebar listener at document level — captures regardless of focus
13
+ document.addEventListener('keydown', (e) => {
14
+ if (e.code === 'Space' || e.key === ' ') {
15
+ e.preventDefault();
16
+ e.stopPropagation();
17
+ this.panic();
18
+ }
19
+ }, true); // useCapture=true to catch before anything else
20
 
21
  // Gesture state accumulator (sent to backend)
22
  this._gestureState = {
 
137
  console.log('[Lyria] Backend session ready');
138
  } else if (msg.type === 'fallback') {
139
  this._setFallbackIndicator(msg.active);
140
+ if (msg.style) {
141
+ this.currentStyleLabel = msg.style;
142
+ }
143
  }
144
  } catch (e) {
145
  console.warn('[Lyria] Unparseable message:', event.data.substring(0, 50));
 
288
  }
289
 
290
  cycleSynth() {
291
+ // Synth hand made a fist
292
+ this._handFists[0] = true;
293
+ this._checkDoubleFist();
294
  this.currentSynthIndex = (this.currentSynthIndex + 1) % this.padPresets.length;
295
  this._gestureState.fistGesture = true;
296
  this._sendGestureState();
297
+ // Reset fist state after a short window
298
+ setTimeout(() => { this._handFists[0] = false; this._doubleFistSent = false; }, 1000);
299
+ }
300
+
301
+ setDrumHandFist(isFist) {
302
+ // Called when drum hand makes/releases a fist
303
+ this._handFists[1] = isFist;
304
+ if (isFist) this._checkDoubleFist();
305
+ if (!isFist) this._doubleFistSent = false;
306
+ }
307
+
308
+ _checkDoubleFist() {
309
+ if (this._handFists[0] && this._handFists[1] && !this._doubleFistSent) {
310
+ this._doubleFistSent = true;
311
+ console.log('[Lyria] DOUBLE FIST — BASS DROP!');
312
+ if (this._wsReady) {
313
+ this._ws.send(JSON.stringify({ type: 'bassdrop' }));
314
+ }
315
+ }
316
  }
317
 
318
  panic() {
backend/server.py CHANGED
@@ -162,10 +162,16 @@ async def websocket_endpoint(websocket: WebSocket):
162
  prompt_text = await interpreter.interpret(_gd, force_new_direction=_im)
163
  if prompt_text:
164
  await lyria.update_prompts(prompt_text)
 
 
 
 
 
165
  try:
166
  await websocket.send_json({
167
  "type": "fallback",
168
- "active": interpreter.using_fallback
 
169
  })
170
  except Exception:
171
  pass
@@ -195,6 +201,29 @@ async def websocket_endpoint(websocket: WebSocket):
195
  last_qwen_time = 0 # allow immediate Qwen update on resume
196
  print("[Server] RESUMED")
197
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
198
  elif msg["type"] == "hands_lost":
199
  # All hands gone — fade out
200
  await lyria.update_config({"density": 0.05, "brightness": 0.2})
 
162
  prompt_text = await interpreter.interpret(_gd, force_new_direction=_im)
163
  if prompt_text:
164
  await lyria.update_prompts(prompt_text)
165
+ # Extract top prompt as style label
166
+ style = max(prompt_text, key=lambda p: p.get("weight", 0))
167
+ style_label = style["text"][:30]
168
+ else:
169
+ style_label = ""
170
  try:
171
  await websocket.send_json({
172
  "type": "fallback",
173
+ "active": interpreter.using_fallback,
174
+ "style": style_label
175
  })
176
  except Exception:
177
  pass
 
201
  last_qwen_time = 0 # allow immediate Qwen update on resume
202
  print("[Server] RESUMED")
203
 
204
+ elif msg["type"] == "bassdrop":
205
+ # Double fist = EPIC BASS DROP
206
+ print("[Server] BASS DROP!")
207
+ await lyria.update_prompts([
208
+ {"text": "massive epic bass drop impact", "weight": 3.0},
209
+ {"text": "sub-bass earthquake explosion", "weight": 2.5},
210
+ ])
211
+ await lyria.update_config({"density": 0.95, "brightness": 0.3})
212
+ # Revert after 5 seconds
213
+ async def _revert_drop():
214
+ import asyncio
215
+ await asyncio.sleep(5)
216
+ # Restore to current gesture-based state
217
+ if current_gesture_state:
218
+ cfg = _map_gesture_to_lyria_config(current_gesture_state)
219
+ await lyria.update_config(cfg)
220
+ prompt_text = await interpreter.interpret(
221
+ current_gesture_state, force_new_direction=True
222
+ )
223
+ if prompt_text:
224
+ await lyria.update_prompts(prompt_text)
225
+ asyncio.create_task(_revert_drop())
226
+
227
  elif msg["type"] == "hands_lost":
228
  # All hands gone — fade out
229
  await lyria.update_config({"density": 0.05, "brightness": 0.2})
game.js CHANGED
@@ -971,6 +971,11 @@ export var Game = /*#__PURE__*/ function() {
971
  } else if (isDrumHand) {
972
  var fingerStates = _this1._getFingerStates(smoothedLandmarks);
973
  if (playerIndex === 0) drumManager.updateActiveDrums(fingerStates);
 
 
 
 
 
974
  _this1._updateHandLines(i, smoothedLandmarks, videoParams, canvasWidth, canvasHeight, {
975
  fingerStates: fingerStates
976
  });
@@ -1505,9 +1510,10 @@ export var Game = /*#__PURE__*/ function() {
1505
  });
1506
  volumeLabel.position.set(midPoint.x, midPoint.y - 20, 2);
1507
  lineGroup.add(volumeLabel);
1508
- // Pitch label above wrist
1509
- var pitchLabel = this._createTextSprite(note, {
1510
- fontsize: 20,
 
1511
  backgroundColor: this.labelColors.evaGreen,
1512
  textColor: this.labelColors.black
1513
  });
 
971
  } else if (isDrumHand) {
972
  var fingerStates = _this1._getFingerStates(smoothedLandmarks);
973
  if (playerIndex === 0) drumManager.updateActiveDrums(fingerStates);
974
+ // Detect fist on drum hand for double-fist bass drop
975
+ var drumIsFist = _this1._isFist(smoothedLandmarks);
976
+ if (_this1.musicManager.setDrumHandFist) {
977
+ _this1.musicManager.setDrumHandFist(drumIsFist);
978
+ }
979
  _this1._updateHandLines(i, smoothedLandmarks, videoParams, canvasWidth, canvasHeight, {
980
  fingerStates: fingerStates
981
  });
 
1510
  });
1511
  volumeLabel.position.set(midPoint.x, midPoint.y - 20, 2);
1512
  lineGroup.add(volumeLabel);
1513
+ // Style label above wrist (from Lyria/Qwen)
1514
+ var styleText = (this.musicManager.currentStyleLabel || note).toUpperCase();
1515
+ var pitchLabel = this._createTextSprite(styleText, {
1516
+ fontsize: 16,
1517
  backgroundColor: this.labelColors.evaGreen,
1518
  textColor: this.labelColors.black
1519
  });
index.html CHANGED
@@ -44,18 +44,20 @@
44
  </div>
45
 
46
  <div id="controls-hint" class="hidden">
47
- <div class="hint-title">HYPERSPACE JAM</div>
48
  <div class="hint-grid">
49
- <span class="hint-icon">↕️</span><span class="hint-desc">Hand high = bright · Hand low = DEEP bass</span>
50
- <span class="hint-icon">👌</span><span class="hint-desc">Pinch thumb+index = volume</span>
51
- <span class="hint-icon">🖐️</span><span class="hint-desc">Extend fingers = play chord notes</span>
52
- <span class="hint-icon">✊</span><span class="hint-desc">Make fist = cycle synth sound</span>
53
- <span class="hint-icon">🤲</span><span class="hint-desc">Tilt wrist = filter sweep + detune</span>
54
- <span class="hint-icon">🤚</span><span class="hint-desc">2nd hand = kick · hihat · clap (distance = tempo)</span>
55
- <span class="hint-icon">📏</span><span class="hint-desc">Spread fingers wide = wobble bass</span>
56
- <span class="hint-icon">👆</span><span class="hint-desc">Touch fingertips together = sparkle sounds</span>
 
 
57
  </div>
58
- <div class="hint-fade">spacebar = panic stop · D = toggle displacement</div>
59
  </div>
60
 
61
  </div>
 
44
  </div>
45
 
46
  <div id="controls-hint" class="hidden">
47
+ <div class="hint-title">HYPERSPACE JAM — LYRIA AI</div>
48
  <div class="hint-grid">
49
+ <span class="hint-icon">↕️</span><span class="hint-desc">Hand HIGH = ambient/shoegaze · LOW = DnB/dubstep</span>
50
+ <span class="hint-icon">🖐️</span><span class="hint-desc">Spread wide = dense layers · Closed = minimal</span>
51
+ <span class="hint-icon">👌</span><span class="hint-desc">Pinch thumb+index = intensity</span>
52
+ <span class="hint-icon">👍</span><span class="hint-desc">Thumb = melody · Index = arp · Middle = bass</span>
53
+ <span class="hint-icon">🤟</span><span class="hint-desc">Ring = percussion · Pinky = texture/atmosphere</span>
54
+ <span class="hint-icon">🤲</span><span class="hint-desc">Tilt wrist = filter sweep + dissonance</span>
55
+ <span class="hint-icon">🤚</span><span class="hint-desc">2nd hand = drums + BPM control</span>
56
+ <span class="hint-icon">⬜</span><span class="hint-desc">Both hands shape = sub-bass drone</span>
57
+ <span class="hint-icon">✊✊</span><span class="hint-desc">Both fists = EPIC BASS DROP</span>
58
+ <span class="hint-icon">🧘</span><span class="hint-desc">Hold still = lock the groove</span>
59
  </div>
60
+ <div class="hint-fade">SPACE = pause/resume · D = displacement · ? = this menu</div>
61
  </div>
62
 
63
  </div>