Javascript Audio APIs - Let's make some noise

Javascript Audio APIs Let‘s make some noise! APIs, Demos, Applications
Jan Krutisch <[email protected]> for FrosCon 2011, St. Augustin

Realtime audio?

In/Out

Applications?

Collaboration

>1 Sounds at once

Spatial Audio

Ahem. WebGL. Ahem.

( Build and they will come )

Status Quo.

Nothing.

Nothing serious.

well, Flash.

(insert rant on ﬂash scripting bridge here)

Mac about:ﬂags Win, Linux Canary builds

Code or it didn‘t happen

Audio Data API

reading

<audio id="audio" src="song.ogg"></audio> <script> var audio = document.getElementById("audio"); audio.addEventListener('MozAudioAvailable', audioAvailable,
false); audio.addEventListener('loadedmetadata', audioInfo, false); </script>

var channels, rate, frameBufferLength, samples; function audioInfo() { var audio
= document.getElementById('audio'); // After loadedmetadata event, following media element attributes are known: channels = audio.mozChannels; rate = audio.mozSampleRate; frameBufferLength = audio.mozFrameBufferLength; } function audioAvailable(event) { var samples = event.frameBuffer; var time = event.time; for (var i = 0; i < frameBufferLength; i++) { // Do something with the audio data as it is played. processSample(samples[i], channels, rate); } }

writing

<!DOCTYPE html> <html> <head> <title>JavaScript Audio Write Example</title> </head> <body>
<input type="text" size="4" id="freq" value="440"><label for="hz">Hz</label> <button onclick="start()">play</button> <button onclick="stop()">stop</button> <script type="text/javascript"> function AudioDataDestination(sampleRate, readFn) { // Initialize the audio output. var audio = new Audio(); audio.mozSetup(1, sampleRate); var currentWritePosition = 0; var prebufferSize = sampleRate / 2; // buffer 500ms var tail = null; // The function called with regular interval to populate // the audio output buffer. setInterval(function() { var written; // Check if some data was not written in previous attempts. if(tail) { written = audio.mozWriteAudio(tail); currentWritePosition += written; if(written < tail.length) { // Not all the data was written, saving the tail... tail = tail.slice(written); return; // ... and exit the function. } tail = null; } // Check if we need add some data to the audio output. var currentPosition = audio.mozCurrentSampleOffset(); var available = currentPosition + prebufferSize - currentWritePosition; if(available > 0) { // Request some sound data from the callback function. var soundData = new Float32Array(available); readFn(soundData); // Writting the data. written = audio.mozWriteAudio(soundData); if(written < soundData.length) { // Not all the data was written, saving the tail. tail = soundData.slice(written); } currentWritePosition += written; } }, 100); } // Control and generate the sound. var frequency = 0, currentSoundSample; var sampleRate = 44100; function requestSoundData(soundData) { if (!frequency) { return; // no sound selected } var k = 2* Math.PI * frequency / sampleRate; for (var i=0, size=soundData.length; i<size; i++) { soundData[i] = Math.sin(k * currentSoundSample++); } } var audioDestination = new AudioDataDestination(sampleRate, requestSoundData); function start() { currentSoundSample = 0; frequency = parseFloat(document.getElementById("freq").value); } function stop() { frequency = 0; } </script>

[...] var audio = new Audio(); audio.mozSetup(1, sampleRate); [...] written
= audio.mozWriteAudio(tail); [...]

https://wiki.mozilla.org/Audio_Data_API#Writing_Audio

Web Audio API

reading

var context = new webkitAudioContext(); var source = context.createBufferSource(); var
jsNode = context.createJavaScriptNode(1024 * 8, 1, 1); jsNode.onaudioprocess = processAudio; source.connect(jsNode); jsNode.connect(context.destination); loadSample("data/funkydrummer.mp3");

function loadSample(url) { // Load asynchronously var request = new
XMLHttpRequest(); request.open("GET", url, true); request.responseType = "arraybuffer"; request.onload = function() { var buffer = context.createBuffer(request.response, false); source.buffer = buffer; source.looping = true; source.noteOn(0.0); console.log("Sample loaded"); } request.send(); }

writing

! var context = new webkitAudioContext(); var source = context.createJavaScriptNode(8192,
0, 1); source.onaudioprocess = requestSoundData; source.connect(context.destination);

chaining

var context = new webkitAudioContext(); var source = context.createBufferSource(); var
filter = context.createBiquadFilter(); filter.type = filter.LOWPASS; filter.frequency.value = (12000.0); filter.Q = 2; source.connect(filter); filter.connect(context.destination); loadSample("data/funkydrummer.mp3"); setInterval(function() { filter.frequency.value = Math.random() * 12000; //console.log(filter.frequency); }, 200)

Performance

Software Synthesis in Javascript. Crazy?

Not at all

Depends heavily on

JS runtime

Hardware

Breakthrough

V8 Nitro TraceMonkey Carakan Chakra

Typed Arrays

( Invented for WebGL )

Thank you Mr. Moore.

Curiosities

On The Fly Data URL

http://synth.bitsnbites.eu/play.html

window.btoa()

My little corner

http://github.com/halfbyte/soundbridge.js

Web Audio > Audio Data > Flash

Only Mono/Stereo

Only Output

SoundBridge(2, 44100, '..', function(soundbridge) { soundbridge.setCallback(calc); log("now stating to play.");
soundbridge.play(); });

! var absoluteBufferPos = 0; var calc = function(bridge, bufferSize,
channels) { for(var b=0;b<bufferSize;b++) { var period = ((absoluteBufferPos + b) % 100) / 100; period *= 2 * Math.PI; var val = Math.sin(period); bridge.addToBuffer(val,val); } absoluteBufferPos += bufferSize; };

http://webloop.pixelpoke.de https://github.com/halfbyte/webloop

Demo time

Resources

‣ https://wiki.mozilla.org/Audio_Data_API ‣ https://dvcs.w3.org/hg/audio/raw-ﬁle/tip/webaudio/speciﬁcation.html ‣ https://wiki.mozilla.org/Audio_Data_API#Working_Audio_Data_Demos ‣ http://chromium.googlecode.com/svn/trunk/samples/audio/index.html ‣ https://github.com/halfbyte/webloop
‣ https://github.com/halfbyte/soundbridge.js

thanks. http://jan.krutisch.de/ http://twitter.com/halfbyte http://github.com/halfbyte http://soundcloud.com/halfbyte http://www.mindmatters.de/

Javascript Audio APIs - Let's make some noise

Javascript Audio APIs - Let's make some noise

More Decks by Jan Krutisch

Other Decks in Technology

Featured

Transcript