Audio Capture

A talk about the details to create an app which records audio

Transcript

1. AVFoundation &
   
   
   Audio Capture
   
   
   Can you hear me?
   Ronan Rodrigo Nunes
   

   iOS Software Engineer @ Uber
   

   ronanrodrigo.dev
   

   View Slide

2. Straight forward implementation
   

   View Slide

3. Is simple right?
   let recordingSession = AVAudioSession.sharedInstance()
   
   
   audioSession.setCategory(.playAndRecord, mode: .default, options:
   [.mixWithOthers])
   
   
   recordingSession.setActive(true)
   
   
   recordingSession.requestRecordPermission { (allowed) in
   
   
   audioRecorder = AVAudioRecorder(url: filename, settings: settings)
   
   
   audioRecorder?.record()
   
   
   }
   

   View Slide

4. But…
   let recordingSession = AVAudioSession.sharedInstance()
   
   
   

   View Slide

5. I see AVAudioEngine…
   let audioEngine = AVAudioEngine()
   
   
   let inputNode = audioEngine.inputNode
   
   
   let inputFormat = inputNode.inputFormat(forBus: 0)
   
   
   inputNode.installTap(
   
   
   onBus: 0,
   
   
   bufferSize: size,
   
   
   format: inputFormat,
   
   
   block: handle(buffer:time:)
   
   
   )
   

   View Slide

6. ⚠ Terminating app due to uncaught
   exception com.apple.coreaudio.avfaudio
   reason: required condition is false:
   format.sampleRate == hwFormat.sampleRate
   

   View Slide

7. View Slide

8. I see AVAudioEngine…
   let audioEngine = AVAudioEngine()
   
   
   let inputNode = audioEngine.inputNode
   
   
   let inputFormat = inputNode.inputFormat(forBus: 0)
   
   
   inputNode.installTap(
   
   
   onBus: 0,
   
   
   bufferSize: size,
   
   
   format: inputFormat,
   
   
   block: handle(buffer:time:)
   
   
   )
   

   View Slide

9. A route change
   NotificationCenter.default.publisher(for: AVAudioSession.routeChangeNotification)
   
   
   .compactMap { notification -> AVAudioSession.RouteChangeReason? in //... }
   
   
   .sink(receiveValue: { [weak self] routeChangeReason in
   
   
   switch routeChangeReason {
   
   
   case .unknown: Logger.log("🏴☠ unknown")
   
   
   case .newDeviceAvailable: Logger.log("🏴☠ newDeviceAvailable")
   
   
   case .oldDeviceUnavailable: Logger.log("🏴☠ oldDeviceUnavailable")
   
   
   case .override: Logger.log("🏴☠ override")
   
   
   case .wakeFromSleep: Logger.log("🏴☠ wakeFromSleep")
   
   
   case .noSuitableRouteForCategory: Logger.log("🏴☠ noSuitableRouteForCategory")
   
   
   case .routeConfigurationChange: Logger.log("🏴☠ routeConfigurationChange")
   
   
   case .categoryChange: Logger.log("🏴☠ categoryChange")
   
   
   }
   
   
   })
   

   View Slide

10. Time to reset the capture
    audioEngine.inputNode.removeTap(onBus: 0)
    
    
    notificationCancellable?.cancel()
    
    
    notificationCancellable = nil
    
    
    audioEngine.stop()
    
    
    audioEngine = AVAudioEngine()
    
    
    setupAudioSession()
    
    
    startCapture()
    

    View Slide

11. Seems to be fixed…
    

    View Slide

12. A route change
    NotificationCenter.default.publisher(for: AVAudioSession.routeChangeNotification)
    
    
    .compactMap { notification -> AVAudioSession.RouteChangeReason? in //... }
    
    
    .sink(receiveValue: { [weak self] routeChangeReason in
    
    
    switch routeChangeReason {
    
    
    case .unknown: Logger.log("🏴☠ unknown")
    
    
    case .newDeviceAvailable: Logger.log("🏴☠ newDeviceAvailable")
    
    
    case .oldDeviceUnavailable: Logger.log("🏴☠ oldDeviceUnavailable")
    
    
    case .override: Logger.log("🏴☠ override")
    
    
    case .wakeFromSleep: Logger.log("🏴☠ wakeFromSleep")
    
    
    case .noSuitableRouteForCategory: Logger.log("🏴☠ noSuitableRouteForCategory")
    
    
    case .routeConfigurationChange: Logger.log("🏴☠ routeConfigurationChange")
    
    
    case .categoryChange: Logger.log("🏴☠ categoryChange")
    
    
    }
    
    
    })
    

    View Slide

13. Dumping the Audio Session
    private func dumpAudioSession() -> String {
    
    
    return """
    
    
    preferredSampleRate: \(audioSession.preferredSampleRate)
    
    
    preferredInputNumberOfChannels: \(audioSession.preferredInputNumberOfChannels)
    
    
    maximumInputNumberOfChannels: \(audioSession.maximumInputNumberOfChannels)
    
    
    inputNumberOfChannels: \(audioSession.inputNumberOfChannels)
    
    
    isInputAvailable: \(audioSession.isInputAvailable)
    
    
    inputDataSources: \((audioSession.inputDataSources?.map { "\($0.dataSourceName)" }))
    
    
    inputDataSource: \(audioSession.inputDataSource?.dataSourceName)
    
    
    inputSelectedDatasource: \(audioSession.currentRoute.inputs.compactMap { $0.selectedDataSource?.dataSourceName })
    
    
    sampleRate: \(audioSession.sampleRate)
    
    
    preferredOutputNumberOfChannels: \(audioSession.preferredOutputNumberOfChannels)
    
    
    outputSelectedDatasource: \(audioSession.currentRoute.outputs.compactMap { $0.selectedDataSource?.dataSourceName })
    
    
    outputDataSources: \(audioSession.outputDataSources)
    
    
    outputDataSource: \(audioSession.outputDataSource)
    
    
    """
    
    
    }
    

    View Slide

14. Now restart
    if dumpAudioSession() == requiredConfig { return }
    
    
    audioEngine.inputNode.removeTap(onBus: 0)
    
    
    notificationCancellable?.cancel()
    
    
    notificationCancellable = nil
    
    
    audioEngine.stop()
    
    
    audioEngine = AVAudioEngine()
    
    
    setupAudioSession()
    
    
    startCapture()
    

    View Slide

15. A new issue
    

    View Slide

16. A new issue
    
    
    AVAudioSession
    

    is a singleton!
    

    View Slide

17. Audio resource provider
    engineProvider
    
    
    .engine(for: consumer)
    
    
    .compactMap { availability -> AudioResourceAudioEngine? in
    
    
    switch availability {
    
    
    case let .available(engine): return engine
    
    
    //...
    
    
    }
    
    
    }
    
    
    .subscribe(onNext: { engine in
    
    
    try? engine.inputNode.installTap { buffer, time in
    
    
    //...
    
    
    }
    
    
    })
    

    View Slide