Crazy computer vision stuff in the browser

1. CRAZY COMPUTER vision stuff in the browser
2. .html .css .js
3. zenorocha
9. computer vision ? what is
10. Persistence of vision
11. Persistence of vision
12. Motion perception
13. Motion perception
14. vimeo.com/108331968
15. Face detection
16. Face recognition
17. Autonomous robots
18. Autonomous robots
19. Virtual reality
20. Augmented reality
21. Augmented reality
22. how to do that?
25. how to do that in the browser?
26. 1. Access the camera
28. Request user’s webcam navigator.getUserMedia({ video: true }, onSuccess, onFail);
29. Request user’s webcam navigator.getUserMedia({ video: true }, onSuccess, onFail);
30. Request user’s webcam navigator.getUserMedia({ video: true }, onSuccess, onFail);
31. 2. Reproduce in a video
32. <video>
33. Get User Media callback function onSuccess(stream) { var video = document.querySelector('video'); video.src = URL.createObjectURL(stream); video.onloadedmetadata = function(e) { // Done }; }
34. Get User Media callback function onSuccess(stream) { var video = document.querySelector('video'); video.src = URL.createObjectURL(stream); video.onloadedmetadata = function(e) { // Done }; }
35. Get User Media callback function onSuccess(stream) { var video = document.querySelector('video'); video.src = URL.createObjectURL(stream); video.onloadedmetadata = function(e) { // Done }; }
36. Get User Media callback function onSuccess(stream) { var video = document.querySelector('video'); video.src = URL.createObjectURL(stream); video.onloadedmetadata = function(e) { // Done }; }
37. 3. Obtain pixel matrix
38. <canvas>
39. 4. Tracking algorithms
40. Request accepted Request access to the camera Reproduce
 camera <video> Obtain pixel matrix <canvas> Tracking algorithms Result Summary
41. tracking.js
43. Maira Bello Team eduardo lundgren java pablo carvalho zeno rocha
44. 100% JavaScript! Open source No dependencies Simple and intuitive API Automatic setup Built-in tracking
 algorithms Easily extensible High performance Unit & performance
 regression tests ~ 7 Kb
45. tracker color
46. Demo
47. var tracker = new tracking.ColorTracker('magenta'); tracking.track('#video', tracker, { camera: true }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
48. var tracker = new tracking.ColorTracker('magenta'); tracking.track('#video', tracker, { camera: true }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
49. var tracker = new tracking.ColorTracker('magenta'); tracking.track('#video', tracker, { camera: true }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
50. tracker object
51. Demo
52. var tracker = new tracking.ObjectTracker('face'); tracking.track('#video', tracker, { camera: true }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
53. var tracker = new tracking.ObjectTracker('face'); tracking.track('#video', tracker, { camera: true }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
54. var tracker = new tracking.ObjectTracker('face'); tracking.track('#video', tracker, { camera: true }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
55. tracker custom
56. var MyTracker = function() { MyTracker.base(this, 'constructor'); }; tracking.inherits(MyTracker, tracking.Tracker); MyTracker.prototype.track = function(pxs, width, height) { // Your code here this.emit('track', { // Your code here }); };
57. var MyTracker = function() { MyTracker.base(this, 'constructor'); }; tracking.inherits(MyTracker, tracking.Tracker); MyTracker.prototype.track = function(pxs, width, height) { // Your code here this.emit('track', { // Your code here }); };
58. var MyTracker = function() { MyTracker.base(this, 'constructor'); }; tracking.inherits(MyTracker, tracking.Tracker); MyTracker.prototype.track = function(pxs, width, height) { // Your code here this.emit('track', { // Your code here }); };
59. var MyTracker = function() { MyTracker.base(this, 'constructor'); }; tracking.inherits(MyTracker, tracking.Tracker); MyTracker.prototype.track = function(pxs, width, height) { // Your code here this.emit('track', { // Your code here }); };
60. var tracker = new tracking.MyTracker(); tracking.track('#video', tracker, { camera: true }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
61. var tracker = new tracking.MyTracker(); tracking.track('#video', tracker, { camera: true }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
62. var tracker = new tracking.MyTracker(); tracking.track('#video', tracker, { camera: true }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); });
63. who is using it?
64. kdzwinel.github.io/JS-face-tracking-demo
65. thedevilini.com/#masks
66. lessons learned
67. 24 fps 60 fps 1000ms / 24fps = 41ms per frame
68. 24 fps 60 fps 1000ms / 24fps = 41ms per frame
72. Int32Array Uint16Array Float64Array Uint8ClampedArray [ ] Typed arrays
73. jsperf.com/tracking-js-arrays
74. jsperf.com/tracking-js-arithmetic
75. Demo
78. web components > > -
79. how we create a component nowadays?
80. 1. Never create! Just use a plugin
81. 2. Copy & paste someone’s code
82. 3. And hope it works
83. 3. And hope it works
84. web components
85. web components Templates
86. web components Templates Custom Elements
87. web components Templates Custom Elements Shadow DOM
88. web components HTML Imports Templates Custom Elements Shadow DOM
90. var tracker = new tracking.ColorTracker('magenta'); tracking.track('#video', tracker, { camera: true }); tracker.on('track', function(event) { event.data.forEach(function(rect) { // Your code goes here }); }); Remember?
91. Using Web Components… <video is="video-color-tracking" target="magenta" camera="true" ontrack="onResult" </video>
92. more ?
93. trackingjs.com
94. Where the magic happens Your comfort zone
95. Thanks! zenorocha.com @zenorocha
96. Thanks! zenorocha.com @zenorocha