Real-time transcription and sentiment analysis of audio streams; on the phone and in the browser

Real-time transcription and sentiment analysis of audio streams; on the phone and in the browser

These are slightly updated slides for Geektime Code 2017, unfortunately, there is not a recording of that session. You can view a previous version of this talk with slightly different slides here: https://www.youtube.com/watch?v=v69dJbCfb-8

Discover how you can use Artifical Intelligence to perform sentiment analysis of an audio stream, in real-time! In this talk, we’re going to learn how to create a virtual rapporteur. A digital assistant who can join any conference call; record it, and then by using IBM Watson provide participants with a real-time transcript and insights into the overall tone of the call. All pushed to their browser via WebSockets.

309287088ccfe196428a5dbe2b051c48?s=128

Aaron Bassett

June 06, 2018
Tweet

Transcript

  1. 1.
  2. 3.
  3. 4.
  4. 5.
  5. 6.
  6. 7.
  7. 8.
  8. 9.

    (function poll() { new Ajax.Request('/api/', { method:'get', onSuccess: function() {

    ... }, onFailure: function() { ... } }); setTimeout(poll, 1000); }());
  9. 10.

    (function poll() { new Ajax.Request('/api/', { method:'get', onSuccess: function() {

    ... }, onFailure: function() { ... } }); setTimeout(poll, 1000); }());
  10. 11.

    (function poll() { new Ajax.Request('/api/', { method:'get', onSuccess: function() {

    ... }, onFailure: function() { ... } }); setTimeout(poll, 1000); }());
  11. 12.

    (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000,

    onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
  12. 13.

    (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000,

    onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
  13. 14.

    (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000,

    onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
  14. 15.

    (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000,

    onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
  15. 16.

    (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000,

    onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
  16. 17.

    (function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000,

    onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());
  17. 18.
  18. 20.
  19. 21.
  20. 23.
  21. 24.
  22. 25.
  23. 28.
  24. 30.
  25. 31.
  26. 32.
  27. 33.
  28. 34.
  29. 35.

    ish

  30. 36.
  31. 37.
  32. 38.
  33. 39.
  34. 40.
  35. 41.
  36. 42.

    var soundAllowed = function (stream) { window.persistAudioStream = stream; var

    audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);
  37. 43.

    var soundAllowed = function (stream) { window.persistAudioStream = stream; var

    audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);
  38. 44.

    var soundAllowed = function (stream) { window.persistAudioStream = stream; var

    audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);
  39. 45.

    var soundAllowed = function (stream) { window.persistAudioStream = stream; var

    audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);
  40. 46.
  41. 47.
  42. 48.
  43. 49.
  44. 50.
  45. 51.
  46. 52.
  47. 53.

    def proxy(self): return [ { 'action': 'connect', 'eventUrl': [f'{self.base_url}/events'], 'from':

    os.environ['NEXMO_NUMBER'], 'endpoint': [ { 'type': 'websocket', 'uri': f'{os.environ["WEBSOCKET_SERVER_URL"]}/socket', 'content-type': 'audio/l16;rate=16000', 'headers': {} } ] } ]
  48. 54.

    def proxy(self): return [ { 'action': 'connect', 'eventUrl': [f'{self.base_url}/events'], 'from':

    os.environ['NEXMO_NUMBER'], 'endpoint': [ { 'type': 'websocket', 'uri': f'{os.environ["WEBSOCKET_SERVER_URL"]}/socket', 'content-type': 'audio/l16;rate=16000', 'headers': {} } ] } ]
  49. 55.

    def proxy(self): return [ { 'action': 'connect', 'eventUrl': [f'{self.base_url}/events'], 'from':

    os.environ['NEXMO_NUMBER'], 'endpoint': [ { 'type': 'websocket', 'uri': f'{os.environ["WEBSOCKET_SERVER_URL"]}/socket', 'content-type': 'audio/l16;rate=16000', 'headers': {} } ] } ]
  50. 56.
  51. 57.
  52. 58.
  53. 59.
  54. 60.
  55. 61.

    def on_message(self, message): transcriber = yield self.transcriber if type(message) !=

    str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)
  56. 62.

    def on_message(self, message): transcriber = yield self.transcriber if type(message) !=

    str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)
  57. 63.

    def on_message(self, message): transcriber = yield self.transcriber if type(message) !=

    str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)
  58. 64.

    def on_message(self, message): transcriber = yield self.transcriber if type(message) !=

    str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)
  59. 67.
  60. 68.
  61. 69.
  62. 70.
  63. 71.
  64. 72.
  65. 73.

    def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results'

    in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))
  66. 74.

    def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results'

    in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))
  67. 75.

    def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results'

    in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))
  68. 76.

    def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results'

    in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))
  69. 77.

    def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results'

    in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))
  70. 78.
  71. 79.
  72. 80.
  73. 81.

    class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self):

    DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass
  74. 82.

    class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self):

    DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass
  75. 83.

    class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self):

    DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass
  76. 84.

    class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self):

    DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass
  77. 85.
  78. 86.
  79. 87.

    var emotions = { anger: new TimeSeries(), disgust: new TimeSeries(),

    fear: new TimeSeries(), joy: new TimeSeries(), sadness: new TimeSeries() }
  80. 88.

    var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt)

    { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }
  81. 89.

    var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt)

    { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }
  82. 90.

    var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt)

    { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }
  83. 91.

    var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt)

    { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }
  84. 92.
  85. 93.
  86. 94.