feat: route imported and frozen clips through volume, ring, and pitch modulation (#27)

Author: nosam1998

src/app.js

@@ -217,4 +217,5 @@ refresh();
 
 import { installLibrary } from './library.js';
 import { installAudioLibrary } from './audio-library.js';
-app.ready=installLibrary(app).then(()=>installAudioLibrary(app));
+import { installModulation } from './modulation-ui.js';
+app.ready=installLibrary(app).then(()=>{installAudioLibrary(app);installModulation(app);});

src/library.js

@@ -15,7 +15,7 @@ export async function installLibrary(app){
     if(p.assets.some(k=>!result[k]))throw new Error('A required audio source is missing. Restore its project bundle.');
     return result;
   };
-  async function refreshEntries(){entries=(await store.list('sounds')).sort((a,b)=>a.order-b.order);renderBoard();}
+  async function refreshEntries(){entries=(await store.list('sounds')).sort((a,b)=>a.order-b.order);renderBoard();window.dispatchEvent(new Event('sigint-library-change'));}
   function retain(asset){cache.delete(asset.id);cache.set(asset.id,asset);cacheBytes=[...cache.values()].reduce((n,a)=>n+a.samples.byteLength,0);while(cacheBytes>8*1024**2&&cache.size>1){const key=cache.keys().next().value;cacheBytes-=cache.get(key).samples.byteLength;cache.delete(key);}return asset;}
   async function clip(key){if(cache.has(key))return retain(cache.get(key));const a=await store.get('audio',key);if(!a)throw new Error('Saved audio is missing. Open its project and save a new render.');return retain(a);}
   async function play(entry){

src/modulation-ui.js

@@ -0,0 +1,62 @@
+import { COLORS } from './model.js';
+export function installModulation(app){
+  const el=app.element,panel=app.addPanel('Shape one sound with another');
+  const heading=el('p',{class:'hint'},'Choose a frozen audio clip as the modulator. It can follow volume, multiply the waveform, or bend pitch. Saved sounds are copied by content, so later edits to a pad do not change this sound.');
+  const grid=el('div',{class:'control-grid',id:'modulation-controls'}),formula=el('code',{class:'route-formula',id:'modulation-formula'});
+  panel.append(heading,grid,formula);
+  let refreshToken=0;
+  function field(label,node){return el('label',{},label,node);}
+  function select(id,values,value,change){
+    const node=el('select',{id,'data-mod-key':id,onchange:e=>change(e.target.value)},...values.map(([value,text])=>el('option',{value},text)));node.value=value;return node;
+  }
+  function numeric(id,value,min,max,step,change){return el('input',{id,'data-mod-key':id,type:'number',value,min,max,step,onchange:e=>change(Number(e.target.value))});}
+  function check(id,value,change){return el('input',{id,'data-mod-key':id,type:'checkbox',checked:value,onchange:e=>change(e.target.checked)});}
+  async function attachAudio(id,key){
+    try{
+      if(!id){app.edit(p=>{p.modulation[key]='';p.modulation.enabled=false;});return;}
+      const asset=app.assets[id]||await app.store.get('audio',id);if(!asset)throw new Error('That clip is unavailable');
+      if(!app.project.assets.includes(id)&&app.project.assets.length>=8)throw new Error('Remove an unused clip before adding a ninth audio source');
+      app.assets[id]=asset;app.edit(p=>{if(!p.assets.includes(id))p.assets.push(id);p.modulation[key]=id;});
+    }catch(e){app.status(e.message,true);refresh();}
+  }
+  async function refresh(){
+    const token=++refreshToken,p=app.project,m=p.modulation;
+    const [meta,sounds]=await Promise.all([app.store.list('audioMeta'),app.store.list('sounds')]);
+    if(token!==refreshToken)return;
+    const focus=document.activeElement?.dataset.modKey;
+    const available=new Map();
+    for(const a of meta)if(p.assets.includes(a.id))available.set(a.id,a.name+' · imported/frozen clip');
+    for(const s of sounds.sort((a,b)=>a.order-b.order))if(!available.has(s.audioId))available.set(s.audioId,s.name+' · soundboard');
+    const options=[['','Choose a clip…'],...available.entries()];
+    const set=(key,value)=>app.edit(p=>{p.modulation[key]=value;});
+    grid.replaceChildren(
+      field('Enable modulation',check('mod-enabled',m.enabled,v=>{if(v&&!m.sourceId){app.status('Choose a modulator clip first.',true);refresh();return;}set('enabled',v);})),
+      field('Modulator clip',select('mod-source',options,m.sourceId,id=>attachAudio(id,'sourceId'))),
+      field('Effect',select('mod-mode',[['volume','Volume follower'],['ring','Ring modulation'],['pitch','Pitch / varispeed']],m.mode,value=>set('mode',value))),
+      field('Apply to',select('mod-target',[['mix','Current sound / mix'],...(p.mode==='timeline'?[]:[['oscillator','One oscillator']]),['asset','Another audio clip']],m.target,value=>{
+        app.edit(p=>{p.modulation.target=value;if(value==='asset'&&!p.modulation.carrierId)p.modulation.enabled=false;});
+      }))
+    );
+    if(m.target==='oscillator'){
+      const node=select('mod-oscillator',p.voices.map(v=>[v.id,v.name]),m.oscillatorId,v=>set('oscillatorId',v));node.style.borderColor=COLORS[p.voices.find(v=>v.id===m.oscillatorId).color];
+      grid.append(field('Carrier oscillator',node));
+    }
+    if(m.target==='asset')grid.append(field('Carrier clip',select('mod-carrier',options,m.carrierId,id=>attachAudio(id,'carrierId'))));
+    if(m.mode==='pitch')grid.append(field('Pitch depth · semitones ±',numeric('mod-semitones',m.semitones,0,24,.1,v=>set('semitones',v))));
+    else grid.append(field('Depth · 0–1',numeric('mod-depth',m.depth,0,1,.01,v=>set('depth',v))));
+    if(m.mode==='volume')grid.append(field('Attack · seconds',numeric('mod-attack',m.attack,.001,1,.001,v=>set('attack',v))),field('Release · seconds',numeric('mod-release',m.release,.001,1,.001,v=>set('release',v))));
+    grid.append(field('Modulator starts · seconds',numeric('mod-offset',m.offset,0,10,.001,v=>set('offset',v))),field('Loop modulator',check('mod-loop',m.sourceLoop,v=>set('sourceLoop',v))));
+    if(m.target==='asset'||(m.target==='mix'&&m.mode==='pitch'))grid.append(field('Loop carrier',check('carrier-loop',m.carrierLoop,v=>set('carrierLoop',v))));
+    formula.textContent=m.mode==='volume'?'y(t) = carrier(t) × [1 − d(t) + d(t) × envelope(|modulator(t)|)]'
+      :m.mode==='ring'?'y(t) = carrier(t) × [1 − d(t) + d(t) × modulator(t)]'
+      :m.target==='oscillator'?'f(t) = basePitch(t) × 2^(D(t) × modulator(t) / 12); phase = integral of frequency'
+      :'readhead(t) = integral of 2^(D(t) × modulator(t) / 12); y(t) = carrier(readhead(t)). Pitch and timing change together.';
+    if(!m.sourceLoop)formula.textContent+=' Outside the modulator clip: unchanged carrier.';
+    if(focus)grid.querySelector('[data-mod-key="'+CSS.escape(focus)+'"]')?.focus();
+  }
+  app.onChange(()=>refresh().catch(e=>app.status(e.message,true)));app.onAssetsChanged=()=>refresh().catch(e=>app.status(e.message,true));
+  const oldRefresh=app.library.refresh;app.library.refresh=async()=>{await oldRefresh();await refresh();};
+  // Soundboard operations publish a local event after their atomic transaction.
+  window.addEventListener('sigint-library-change',()=>refresh().catch(e=>app.status(e.message,true)));
+  refresh().catch(e=>app.status(e.message,true));app.modulationUI={refresh};
+}

src/style.css

@@ -51,6 +51,19 @@ try {
         await page.waitForFunction(t=>document.querySelector('#import-state').textContent.startsWith('Kept tone.'+t),type,{timeout:30000});
       }
       assert.equal(await page.locator('.asset-row').count(),2);
+      await page.locator('#mod-mode').selectOption('ring');
+      await page.locator('#mod-depth').fill('1');await page.locator('#mod-depth').press('Tab');
+      await page.locator('#mod-enabled').check();
+      await page.getByRole('button',{name:'Play sound',exact:true}).click();
+      await page.waitForFunction(()=>!!window.testApp.lastRender);
+      assert.ok(await page.evaluate(()=>window.testApp.lastRender.peak>0));
+      await page.locator('#stop').click();
+      await page.locator('#mod-mode').selectOption('pitch');
+      await page.locator('#mod-target').selectOption('oscillator');
+      await page.locator('#mod-oscillator').selectOption(await page.evaluate(()=>window.testApp.project.voices[0].id));
+      const route=await page.evaluate(()=>({target:window.testApp.project.modulation.oscillatorId,selected:window.testApp.project.selectedId}));
+      assert.notEqual(route.target,route.selected);
+      await page.getByRole('button',{name:'Play sound',exact:true}).click();await page.waitForFunction(()=>!!window.testApp.lastRender);await page.locator('#stop').click();
       assert.deepEqual(errors,[]);
       if(name==='chromium')console.log('VISUAL:'+((await page.screenshot({type:'jpeg',quality:55,fullPage:true})).toString('base64')));
       console.log(name+': draw, voices, envelopes, undo/redo, worker audio, and Stop passed');

tests/browser.mjs

@@ -48,3 +48,34 @@ test('sample-rate conversion preserves duration and bounded low-frequency tone',
   const out=resamplePCM(v,44100,48000);assert.equal(out.length,4800);
   assert.ok(Math.max(...out)<.51);
 });
+
+function toneAmplitude(samples,rate,hz,from=.025,to=.085){
+  let re=0,im=0,count=0;
+  for(let i=Math.round(from*rate);i<Math.min(samples.length,Math.round(to*rate));i++){re+=samples[i]*Math.cos(2*Math.PI*hz*i/rate);im+=samples[i]*Math.sin(2*Math.PI*hz*i/rate);count++;}
+  return 2*Math.hypot(re,im)/count;
+}
+test('constant bipolar pitch modulators produce octave ratios with continuous integrated phase',async()=>{
+  for(const sign of [-1,1]){
+    const p=project();p.duration=.12;p.voices[0].hz=1000;p.assets=['dc'];
+    p.modulation={...p.modulation,enabled:true,mode:'pitch',target:'oscillator',sourceId:'dc',semitones:12};
+    const result=await render(p,{dc:{sampleRate:48000,samples:new Float32Array(9600).fill(sign)}});
+    assert.ok(toneAmplitude(result.samples,48000,sign===1?2000:500)>.65);
+  }
+});
+test('clip pitch uses a varispeed readhead and explicit carrier loop behavior',async()=>{
+  const p=project();p.duration=.12;p.assets=['tone','dc'];
+  p.modulation={...p.modulation,enabled:true,mode:'pitch',target:'asset',sourceId:'dc',carrierId:'tone',semitones:12};
+  const assets={tone:{sampleRate:48000,samples:Float32Array.from({length:4800},(_,i)=>.5*Math.sin(2*Math.PI*1000*i/48000))},dc:{sampleRate:48000,samples:new Float32Array(9600).fill(1)}};
+  let result=await render(p,assets);
+  assert.ok(toneAmplitude(result.samples,48000,2000,.02,.04)>.47);
+  assert.ok(result.samples.slice(4000,5000).every(x=>Math.abs(x)<1e-8));
+  p.modulation.carrierLoop=true;result=await render(p,assets);
+  assert.ok(toneAmplitude(result.samples,48000,2000,.07,.10)>.47);
+});
+test('volume follower attack, offset, and dry fallback have distinct audible behavior',async()=>{
+  const p=project();p.duration=.15;p.voices[0].hz=1000;p.assets=['level'];
+  p.modulation={...p.modulation,enabled:true,mode:'volume',target:'mix',sourceId:'level',depth:1,attack:.01,release:.03,offset:.02};
+  const a=new Float32Array(4800).fill(.5),out=await render(p,{level:{sampleRate:48000,samples:a}});
+  const early=toneAmplitude(out.samples,48000,1000,.024,.029),settled=toneAmplitude(out.samples,48000,1000,.07,.09),outside=toneAmplitude(out.samples,48000,1000,.13,.14);
+  assert.ok(early<settled);assert.ok(settled>.32&&settled<.37);assert.ok(outside>.65);
+});