Signal Processing with JavaScript JSConf Colombia

Nowadays we have native options to deal with media signals on javascript! How can we use the available APIs to process it? This talk is about amazing uses to Web Audio API and the Video tag like sampling, quantization, filtering, etc. Can we synthesize guitar effects?

Transcript

1. Signal Processing w/ JavaScript @rafael_sps
2. Who? rafael specht da silva Telecommunication Technologist Javascript Enthusiast
3. BrazilJS Conf
4. Nodebots Day Porto Alegre
5. “... mathematical manipulation of a signal to modify it or improve it”
6. var numbers = [1, 2, 3, 4]; numbers.map(function (item) { return item + 3; }); // [4, 5, 6, 7]
7. Sampling
10. Why this is important?
11. var playBuffer = function (buffer) { source = context.createBufferSource(); source.buffer = buffer; var processor = context.createScriptProcessor(512, 2, 2); processor.onaudioprocess = onAudioProcess; source.connect(processor); processor.connect(context.destination); source.start(0, startOffset % buffer.duration); }
12. var onAudioProcess = function (ev) { inputBuffer = ev.inputBuffer; outputBuffer = ev.outputBuffer; inputData = inputBuffer.getChannelData(0); outputData0 = outputBuffer.getChannelData(0); outputData1 = outputBuffer.getChannelData(1); for (var sample = 0, len = inputData.length; sample < len; sample++) { outputData0[sample] = (1 - panelX) * inputData[sample]; outputData1[sample] = panelX * inputData[sample]; } }
13. Stereo
16. Images
17. var canvas = document.getElementById('canvas'); var ctx = canvas.getContext('2d'); function make_base(src) { base_image = new Image(); base_image.src = src; base_image.onload = function(){ ctx.drawImage(base_image, 0, 0); imageLoaded(base_image); } }
18. var img = ctx.getImageData( 0, 0, canvas.width, canvas.height );
19. img.height // number img.width // number img.data // Uint8ClampedArray r g b a r g b a [65, 113, 130, 255, 65, 113, 130, 255, ...
20. COMPRESSION
21. // Run Length Encoding (RLE) // Original (19) [1, 2, 2, 2, 2, 3, 3, 3, 3] // Compressed (15) [1, ‘2 4’, ‘3 4’]
22. var red = []; var green = []; var blue = []; var alpha = []; for (var i = 4; i < imgData.data.length; i+=4) { red.push(imgData.data[i]) green.push(imgData.data[i+1]) blue.push(imgData.data[i+2]) alpha.push(imgData.data[i+3]) }
23. Video Compress
24. Video Diff
25. Chroma key
26. Boundaries
27. Turret
28. Theremin
30. Javascript is awesome
31. Thanks @rafael_sps github.com/rssilva
32. References Images: http://3.bp.blogspot.com/-cjGDYx6JGw8/UY07MGGVfoI/AAAAAAAACuk/kQrzGhK9oFM/s1600/noticia44770.jpg http://www.curtopoa.com.br/images/albuns/fotos-aereas-porto-alegre/cais_porto_alegre.jpg http://wvwri.org/wp-content/uploads/2014/01/1391018991.jpg https://www.flickr.com/photos/murilocardoso/6020194265 https://www.flickr.com/photos/josephkennelty/9554300993 https://www.flickr.com/photos/takeabreakwithme/3234099003 https://www.flickr.com/photos/busiguy6/457381735/in/photolist-nQthL-oN8PmF-f29NoU-dCtU8y-pvt5S-9KoqQh-23cgG- 35D6wU-ccbYUC-GqcDZ-azBSCm-p5sZXB-6ib9Wb-56Srho-hH5RD-fc6RYp-gfcqpQ-vmaEe-56NhmD-azUZWy- 5Y95gU-bUu2s-eDhQLn-6YW9JX-8YQVX1-8M8pUQ-bCnrz-8AsQiz-imtb9d-7N9J1y-ogg8U-5NtoYg-4cELXj-bUoJRy- vmaEc-wGktP-fpNZBa-bGQycT-bGQaTk-5KSTpm-dqtWeP-ruZnP4-9cM7SK-qrm1Lr-7rDrFc-7QRNWn-9KEpL9- oVwQbZ-9PxYEY-pDkEs3 https://www.flickr.com/photos/wdig/7176747886 https://www.flickr.com/photos/poulomee/16140153959 https://www.flickr.com/photos/voulez-vous/12076930664
33. References Projects: http://trackingjs.com/ https://github.com/felipenmoura/js-chroma-key http://fellipe.com/demos/lena-js/ http://codepen.io/SitePoint/full/LVpNjp/ http://dashersw.github.io/pedalboard.js/ http://femurdesign.com/theremin/