Crazy computer vision stuff in the browser

Crazy computer vision stuff in the browser

E190023b66e2b8aa73a842b106920c93?s=128

Zeno Rocha

April 28, 2015
Tweet

Transcript

  1. CRAZY COMPUTER vision stuff in the browser

  2. .html .css .js

  3. zenorocha

  4. None
  5. None
  6. None
  7. None
  8. None
  9. computer vision ? what is

  10. Persistence of vision

  11. Persistence of vision

  12. Motion perception

  13. Motion perception

  14. vimeo.com/108331968

  15. Face detection

  16. Face recognition

  17. Autonomous robots

  18. Autonomous robots

  19. Virtual reality

  20. Augmented reality

  21. Augmented reality

  22. how to do that?

  23. None
  24. None
  25. how to do that in the browser?

  26. 1. Access the camera

  27. None
  28. Request user’s webcam navigator.getUserMedia({ video: true }, onSuccess, onFail);

  29. Request user’s webcam navigator.getUserMedia({ video: true }, onSuccess, onFail);

  30. Request user’s webcam navigator.getUserMedia({ video: true }, onSuccess, onFail);

  31. 2. Reproduce in a video

  32. <video>

  33. Get User Media callback function onSuccess(stream) { var video =

    document.querySelector('video'); video.src = URL.createObjectURL(stream); video.onloadedmetadata = function(e) { // Done }; }
  34. Get User Media callback function onSuccess(stream) { var video =

    document.querySelector('video'); video.src = URL.createObjectURL(stream); video.onloadedmetadata = function(e) { // Done }; }
  35. Get User Media callback function onSuccess(stream) { var video =

    document.querySelector('video'); video.src = URL.createObjectURL(stream); video.onloadedmetadata = function(e) { // Done }; }
  36. Get User Media callback function onSuccess(stream) { var video =

    document.querySelector('video'); video.src = URL.createObjectURL(stream); video.onloadedmetadata = function(e) { // Done }; }
  37. 3. Obtain pixel matrix

  38. <canvas>

  39. 4. Tracking algorithms

  40. Request accepted Request access to the camera Reproduce
 camera <video>

    Obtain pixel matrix <canvas> Tracking algorithms Result Summary
  41. tracking.js

  42. None
  43. Maira Bello Team eduardo lundgren java pablo carvalho zeno rocha

  44. 100% JavaScript! Open source No dependencies Simple and intuitive API

    Automatic setup Built-in tracking
 algorithms Easily extensible High performance Unit & performance
 regression tests ~ 7 Kb
  45. tracker color

  46. Demo

  47. var tracker = new tracking.ColorTracker('magenta'); tracking.track('#video', tracker, { camera: true

    }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
  48. var tracker = new tracking.ColorTracker('magenta'); tracking.track('#video', tracker, { camera: true

    }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
  49. var tracker = new tracking.ColorTracker('magenta'); tracking.track('#video', tracker, { camera: true

    }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
  50. tracker object

  51. Demo

  52. var tracker = new tracking.ObjectTracker('face'); tracking.track('#video', tracker, { camera: true

    }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
  53. var tracker = new tracking.ObjectTracker('face'); tracking.track('#video', tracker, { camera: true

    }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
  54. var tracker = new tracking.ObjectTracker('face'); tracking.track('#video', tracker, { camera: true

    }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
  55. tracker custom

  56. var MyTracker = function() { MyTracker.base(this, 'constructor'); }; tracking.inherits(MyTracker, tracking.Tracker);

    MyTracker.prototype.track = function(pxs, width, height) { // Your code here this.emit('track', { // Your code here }); };
  57. var MyTracker = function() { MyTracker.base(this, 'constructor'); }; tracking.inherits(MyTracker, tracking.Tracker);

    MyTracker.prototype.track = function(pxs, width, height) { // Your code here this.emit('track', { // Your code here }); };
  58. var MyTracker = function() { MyTracker.base(this, 'constructor'); }; tracking.inherits(MyTracker, tracking.Tracker);

    MyTracker.prototype.track = function(pxs, width, height) { // Your code here this.emit('track', { // Your code here }); };
  59. var MyTracker = function() { MyTracker.base(this, 'constructor'); }; tracking.inherits(MyTracker, tracking.Tracker);

    MyTracker.prototype.track = function(pxs, width, height) { // Your code here this.emit('track', { // Your code here }); };
  60. var tracker = new tracking.MyTracker(); tracking.track('#video', tracker, { camera: true

    }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
  61. var tracker = new tracking.MyTracker(); tracking.track('#video', tracker, { camera: true

    }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
  62. var tracker = new tracking.MyTracker(); tracking.track('#video', tracker, { camera: true

    }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
  63. who is using it?

  64. kdzwinel.github.io/JS-face-tracking-demo

  65. thedevilini.com/#masks

  66. lessons learned

  67. 24 fps 60 fps 1000ms / 24fps = 41ms per

    frame
  68. 24 fps 60 fps 1000ms / 24fps = 41ms per

    frame
  69. None
  70. None
  71. None
  72. Int32Array Uint16Array Float64Array Uint8ClampedArray [ ] Typed arrays

  73. jsperf.com/tracking-js-arrays

  74. jsperf.com/tracking-js-arithmetic

  75. Demo

  76. None
  77. None
  78. web components > > -

  79. how we create a component nowadays?

  80. 1. Never create! Just use a plugin

  81. 2. Copy & paste someone’s code

  82. 3. And hope it works

  83. 3. And hope it works

  84. web components

  85. web components Templates

  86. web components Templates Custom Elements

  87. web components Templates Custom Elements Shadow DOM

  88. web components HTML Imports Templates Custom Elements Shadow DOM

  89. None
  90. var tracker = new tracking.ColorTracker('magenta'); tracking.track('#video', tracker, { camera: true

    }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); }); Remember?
  91. Using Web Components… <video is="video-color-tracking" target="magenta" camera="true" ontrack="onResult" </video>

  92. more ?

  93. trackingjs.com

  94. Where the magic happens Your comfort zone

  95. Thanks! zenorocha.com @zenorocha

  96. Thanks! zenorocha.com @zenorocha