Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Digital Signal Processing w/ Javascript

Digital Signal Processing w/ Javascript

Nowadays we have native options to deal with media signals on javascript! How can we use the available APIs to process it? This talk is about amazing uses to Web Audio API and the Video tag like sampling, quantization, filtering, etc. Can we synthesize guitar effects?

268d8fa031153c23add09489760ce3bb?s=128

Rafael Specht da Silva

April 25, 2015
Tweet

Transcript

  1. Digital Signal Processing w/ Javascript @rafael_sps

  2. Who? rafael specht da silva Telecommunication Technologist Javascript enthusiast

  3. None
  4. None
  5. “... mathematical manipulation of an information signal to modify or

    improve it”
  6. None
  7. Sampling

  8. None
  9. None
  10. None
  11. Why this is important?

  12. var playBuffer = function (buffer) { source = context.createBufferSource(); source.buffer

    = buffer; var processor = context.createScriptProcessor(512, 2, 2); processor.onaudioprocess = onAudioProcess; source.connect(processor); processor.connect(context.destination); source.start(0, startOffset % buffer.duration); }
  13. var onAudioProcess = function (ev) { inputBuffer = ev.inputBuffer; outputBuffer

    = ev.outputBuffer; inputData = inputBuffer.getChannelData(0); outputData0 = outputBuffer.getChannelData(0); outputData1 = outputBuffer.getChannelData(1); for (var sample = 0, len = inputData.length; sample < len; sample++) { outputData0[sample] = (1 - panelX) * inputData[sample]; outputData1[sample] = panelX * inputData[sample]; } }
  14. DEMO

  15. None
  16. None
  17. Quantization

  18. None
  19. COMPRESS

  20. // Original (37) [1.1, 1.8, 2.3, 2.4, 1.9, 2.7, 3.2,

    3.4, 3.1] // Quantized (19) [1, 2, 2, 2, 2, 3, 3, 3, 3] // Compressed (15) [1, ‘2 4’, ‘3 4’]
  21. Images

  22. var canvas = document.getElementById('canvas'); var ctx = canvas.getContext('2d'); function make_base(src)

    { base_image = new Image(); base_image.src = src; base_image.onload = function(){ ctx.drawImage(base_image, 0, 0); imageLoaded(base_image); } }
  23. var img = ctx.getImageData( 0, 0, canvas.width, canvas.height );

  24. img.height // number img.width // number img.data // Uint8ClampedArray r

    g b a r g b a [65, 113, 130, 255, 65, 113, 130, 255, ...
  25. None
  26. Compressing

  27. var red = []; var green = []; var blue

    = []; var alpha = []; for (var i = 4; i < imgData.data.length; i+=4) { red.push(imgData.data[i]) green.push(imgData.data[i+1]) blue.push(imgData.data[i+2]) alpha.push(imgData.data[i+3]) }
  28. Image Diff

  29. Video

  30. Video Diff

  31. “Always bet on JS” Brendan Eich

  32. None
  33. Thank you @rafael_sps github.com/rssilva

  34. References Images: https://farm7.staticflickr.com/6075/6068692939_3fa9b50838_b.jpg http://www.curtopoa.com.br/images/albuns/fotos-aereas-porto-alegre/cais_porto_alegre.jpg http://wvwri.org/wp-content/uploads/2014/01/1391018991.jpg https://www.flickr.com/photos/murilocardoso/6020194265 https://www.flickr.com/photos/josephkennelty/9554300993 https://www.flickr.com/photos/takeabreakwithme/3234099003 https://www.flickr.com/photos/busiguy6/457381735/in/photolist-nQthL-oN8PmF-f29NoU-dCtU8y-pvt5S-9KoqQh-23cgG- 35D6wU-ccbYUC-GqcDZ-azBSCm-p5sZXB-6ib9Wb-56Srho-hH5RD-fc6RYp-gfcqpQ-vmaEe-56NhmD-azUZWy-

    5Y95gU-bUu2s-eDhQLn-6YW9JX-8YQVX1-8M8pUQ-bCnrz-8AsQiz-imtb9d-7N9J1y-ogg8U-5NtoYg-4cELXj-bUoJRy- vmaEc-wGktP-fpNZBa-bGQycT-bGQaTk-5KSTpm-dqtWeP-ruZnP4-9cM7SK-qrm1Lr-7rDrFc-7QRNWn-9KEpL9- oVwQbZ-9PxYEY-pDkEs3 https://www.flickr.com/photos/wdig/7176747886 https://www.flickr.com/photos/poulomee/16140153959 https://www.flickr.com/photos/voulez-vous/12076930664
  35. References Projects: http://trackingjs.com/ https://github.com/felipenmoura/js-chroma-key http://fellipe.com/demos/lena-js/ http://dashersw.github.io/pedalboard.js/