Digital Signal Processing w/ Javascript

Nowadays we have native options to deal with media signals on javascript! How can we use the available APIs to process it? This talk is about amazing uses to Web Audio API and the Video tag like sampling, quantization, filtering, etc. Can we synthesize guitar effects?

Transcript

1. Digital Signal Processing w/ Javascript @rafael_sps
2. Who? rafael specht da silva Telecommunication Technologist Javascript enthusiast
5. “... mathematical manipulation of an information signal to modify or improve it”
7. Sampling
11. Why this is important?
12. var playBuffer = function (buffer) { source = context.createBufferSource(); source.buffer = buffer; var processor = context.createScriptProcessor(512, 2, 2); processor.onaudioprocess = onAudioProcess; source.connect(processor); processor.connect(context.destination); source.start(0, startOffset % buffer.duration); }
13. var onAudioProcess = function (ev) { inputBuffer = ev.inputBuffer; outputBuffer = ev.outputBuffer; inputData = inputBuffer.getChannelData(0); outputData0 = outputBuffer.getChannelData(0); outputData1 = outputBuffer.getChannelData(1); for (var sample = 0, len = inputData.length; sample < len; sample++) { outputData0[sample] = (1 - panelX) * inputData[sample]; outputData1[sample] = panelX * inputData[sample]; } }
14. DEMO
17. Quantization
19. COMPRESS
20. // Original (37) [1.1, 1.8, 2.3, 2.4, 1.9, 2.7, 3.2, 3.4, 3.1] // Quantized (19) [1, 2, 2, 2, 2, 3, 3, 3, 3] // Compressed (15) [1, ‘2 4’, ‘3 4’]
21. Images
22. var canvas = document.getElementById('canvas'); var ctx = canvas.getContext('2d'); function make_base(src) { base_image = new Image(); base_image.src = src; base_image.onload = function(){ ctx.drawImage(base_image, 0, 0); imageLoaded(base_image); } }
23. var img = ctx.getImageData( 0, 0, canvas.width, canvas.height );
24. img.height // number img.width // number img.data // Uint8ClampedArray r g b a r g b a [65, 113, 130, 255, 65, 113, 130, 255, ...
26. Compressing
27. var red = []; var green = []; var blue = []; var alpha = []; for (var i = 4; i < imgData.data.length; i+=4) { red.push(imgData.data[i]) green.push(imgData.data[i+1]) blue.push(imgData.data[i+2]) alpha.push(imgData.data[i+3]) }
28. Image Diff
29. Video
30. Video Diff
31. “Always bet on JS” Brendan Eich
33. Thank you @rafael_sps github.com/rssilva
34. References Images: https://farm7.staticflickr.com/6075/6068692939_3fa9b50838_b.jpg http://www.curtopoa.com.br/images/albuns/fotos-aereas-porto-alegre/cais_porto_alegre.jpg http://wvwri.org/wp-content/uploads/2014/01/1391018991.jpg https://www.flickr.com/photos/murilocardoso/6020194265 https://www.flickr.com/photos/josephkennelty/9554300993 https://www.flickr.com/photos/takeabreakwithme/3234099003 https://www.flickr.com/photos/busiguy6/457381735/in/photolist-nQthL-oN8PmF-f29NoU-dCtU8y-pvt5S-9KoqQh-23cgG- 35D6wU-ccbYUC-GqcDZ-azBSCm-p5sZXB-6ib9Wb-56Srho-hH5RD-fc6RYp-gfcqpQ-vmaEe-56NhmD-azUZWy- 5Y95gU-bUu2s-eDhQLn-6YW9JX-8YQVX1-8M8pUQ-bCnrz-8AsQiz-imtb9d-7N9J1y-ogg8U-5NtoYg-4cELXj-bUoJRy- vmaEc-wGktP-fpNZBa-bGQycT-bGQaTk-5KSTpm-dqtWeP-ruZnP4-9cM7SK-qrm1Lr-7rDrFc-7QRNWn-9KEpL9- oVwQbZ-9PxYEY-pDkEs3 https://www.flickr.com/photos/wdig/7176747886 https://www.flickr.com/photos/poulomee/16140153959 https://www.flickr.com/photos/voulez-vous/12076930664
35. References Projects: http://trackingjs.com/ https://github.com/felipenmoura/js-chroma-key http://fellipe.com/demos/lena-js/ http://dashersw.github.io/pedalboard.js/