Real-time transcription and sentiment analysis of audio streams; on the phone and in the browser

These are slightly updated slides for Geektime Code 2017, unfortunately, there is not a recording of that session. You can view a previous version of this talk with slightly different slides here: https://www.youtube.com/watch?v=v69dJbCfb-8

Discover how you can use Artifical Intelligence to perform sentiment analysis of an audio stream, in real-time! In this talk, we’re going to learn how to create a virtual rapporteur. A digital assistant who can join any conference call; record it, and then by using IBM Watson provide participants with a real-time transcript and insights into the overall tone of the call. All pushed to their browser via WebSockets.

Transcript

2. @aaronbassett
9. (function poll() { new Ajax.Request('/api/', { method:'get', onSuccess: function() { ... }, onFailure: function() { ... } }); setTimeout(poll, 1000); }());
10. (function poll() { new Ajax.Request('/api/', { method:'get', onSuccess: function() { ... }, onFailure: function() { ... } }); setTimeout(poll, 1000); }());
11. (function poll() { new Ajax.Request('/api/', { method:'get', onSuccess: function() { ... }, onFailure: function() { ... } }); setTimeout(poll, 1000); }());
12. (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
13. (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
14. (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
15. (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
16. (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
17. (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
18. –Google “SPDY: An experimental protocol for a faster web" “typical header sizes of 700-800 bytes is common”
19. 2 bytes. 0x00 UTF8 DATA 0xFF
20. 100,000
21. 700 Bytes
22. 560,000,000
23. 70MB/s
24. 0.2MB/s
26. https://git.io/ fNlS2
27. http://werise-tornado- server.ngrok.io/dashboard
29. on_message write_message
35. ish
42. var soundAllowed = function (stream) { window.persistAudioStream = stream; var audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);
43. var soundAllowed = function (stream) { window.persistAudioStream = stream; var audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);
44. var soundAllowed = function (stream) { window.persistAudioStream = stream; var audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);
45. var soundAllowed = function (stream) { window.persistAudioStream = stream; var audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);
53. def proxy(self): return [ { 'action': 'connect', 'eventUrl': [f'{self.base_url}/events'], 'from': os.environ['NEXMO_NUMBER'], 'endpoint': [ { 'type': 'websocket', 'uri': f'{os.environ["WEBSOCKET_SERVER_URL"]}/socket', 'content-type': 'audio/l16;rate=16000', 'headers': {} } ] } ]
54. def proxy(self): return [ { 'action': 'connect', 'eventUrl': [f'{self.base_url}/events'], 'from': os.environ['NEXMO_NUMBER'], 'endpoint': [ { 'type': 'websocket', 'uri': f'{os.environ["WEBSOCKET_SERVER_URL"]}/socket', 'content-type': 'audio/l16;rate=16000', 'headers': {} } ] } ]
55. def proxy(self): return [ { 'action': 'connect', 'eventUrl': [f'{self.base_url}/events'], 'from': os.environ['NEXMO_NUMBER'], 'endpoint': [ { 'type': 'websocket', 'uri': f'{os.environ["WEBSOCKET_SERVER_URL"]}/socket', 'content-type': 'audio/l16;rate=16000', 'headers': {} } ] } ]
61. def on_message(self, message): transcriber = yield self.transcriber if type(message) != str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)
62. def on_message(self, message): transcriber = yield self.transcriber if type(message) != str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)
63. def on_message(self, message): transcriber = yield self.transcriber if type(message) != str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)
64. def on_message(self, message): transcriber = yield self.transcriber if type(message) != str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)
65. self.transcriber = tornado.websocket.websocket_connect( 'wss://[url]?watson-token={token}&model={model}'.format( token=self.transcriber_token, model=os.environ['REALTIME_TRANSCRIBER_MODEL'] ), on_message_callback=self.on_transcriber_message )
66. self.transcriber = tornado.websocket.websocket_connect( 'wss://[url]?watson-token={token}&model={model}'.format( token=self.transcriber_token, model=os.environ['REALTIME_TRANSCRIBER_MODEL'] ), on_message_callback=self.on_transcriber_message )
73. def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results' in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))
74. def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results' in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))
75. def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results' in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))
76. def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results' in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))
77. def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results' in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))
81. class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self): DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass
82. class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self): DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass
83. class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self): DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass
84. class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self): DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass
87. var emotions = { anger: new TimeSeries(), disgust: new TimeSeries(), fear: new TimeSeries(), joy: new TimeSeries(), sadness: new TimeSeries() }
88. var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt) { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }
89. var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt) { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }
90. var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt) { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }
91. var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt) { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }
95. @aaronbassett @nexmodev developer.nexmo.com github.com/nexmo-community