手抜き3D音響レシピ / iOSDC Japan 2023 talk mihara

Transcript

1. 手抜き3D音響レシピ
   2023.09.02
   三原亮介 @oinariman
   株式会社GENDA
   

   View Slide

2. © GENDA Inc.
   世界中の

   人々の人生を

   より楽しく

   会社紹介
   人が人らしく生きるために「
   楽しさ」は不可欠と考え、
   私たちは「世界中の人々の人生をより楽しく
   」という
   Aspiration（アスピレーション＝大志）を掲げています。
   

   View Slide

3. 話すこと
   2Dゲーム開発用の
   SpriteKitを使って
   簡単に3D音響効果を扱える。
   

   View Slide

4. SpriteKitの音声再生機能は便利
   SpriteKit
   AVFoundation
   iOS 7 - 8 iOS 9+
   SpriteKit
   AVFoundation
   SKAudioNode
   SKScene
   .audioEngine
   AVAudioEngine
   AVAudioEngine
   

   View Slide

5. AVAudioEngineのセットアップは大変
   /**
   ハードウェアをチェックして、3D音声再生用のフォーマットを作成して返す。
   - returns: audio format
   */
   func constructOutputConnectionFormatForEnvironment() -> AVAudioFormat {
   var environmentOutputConnectionFormat: AVAudioFormat? = nil
   var numHardwareOutputChannels: AVAudioChannelCount = engine.outputNode.outputFormat(forBus: 0).channelCount
   let hardwareSampleRate: Double = engine.outputNode.outputFormat(forBus: 0).sampleRate
   // if we're connected to multichannel hardware, create a compatible multichannel format for the environment node
   if numHardwareOutputChannels > 2 && numHardwareOutputChannels != 3 {
   if numHardwareOutputChannels > 8 {
   numHardwareOutputChannels = 8
   }
   var environmentOutputLayoutTag: AudioChannelLayoutTag = kAudioChannelLayoutTag_Stereo
   switch numHardwareOutputChannels {
   case 4:
   environmentOutputLayoutTag = kAudioChannelLayoutTag_AudioUnit_4
   case 5:
   environmentOutputLayoutTag = kAudioChannelLayoutTag_AudioUnit_5_0
   case 6:
   environmentOutputLayoutTag = kAudioChannelLayoutTag_AudioUnit_6_0
   case 7:
   environmentOutputLayoutTag = kAudioChannelLayoutTag_AudioUnit_7_0
   case 8:
   environmentOutputLayoutTag = kAudioChannelLayoutTag_AudioUnit_8
   default:
   break
   }
   let environmentOutputChannelLayout = AVAudioChannelLayout(layoutTag: environmentOutputLayoutTag)
   environmentOutputConnectionFormat = AVAudioFormat(standardFormatWithSampleRate: hardwareSampleRate, channelLayout: environmentOutputChannelLayout!)
   multichannelOutputEnabled = true
   } else {
   environmentOutputConnectionFormat = AVAudioFormat(standardFormatWithSampleRate: hardwareSampleRate, channels: 2)
   multichannelOutputEnabled = false
   }
   return environmentOutputConnectionFormat!
   }
   func loadSoundIntoBuffer(_ filename: String, ofType: String = "caf", seek: Double = 0.0) -> AVAudioPCMBuffer? {
   guard let path = Bundle.main.path(forResource: filename, ofType: ofType),
   let url = URL(string: path) else {
   return nil
   }
   do {
   let soundFile = try AVAudioFile(forReading: url, commonFormat: AVAudioCommonFormat.pcmFormatFloat32, interleaved: false)
   let outputBuffer = AVAudioPCMBuffer(pcmFormat: soundFile.processingFormat, frameCapacity: AVAudioFrameCount(soundFile.length))!
   if seek > 0.0 && seek <= 1.0 {
   soundFile.framePosition = AVAudioFramePosition(Double(soundFile.length) * seek)
   }
   try soundFile.read(into: outputBuffer)
   return outputBuffer
   } catch {
   return nil
   }
   }
   ハードウェアをチェックして3D音声再生用の
   AVFormatを作成
   /**
   wire everything up
   */
   func makeEngineConnections() {
   guard let environment: AVAudioEnvironmentNode = environment,
   let sound: Sound = buffers.keys.first,
   let buffer: AVAudioPCMBuffer = buffers[sound] else
   {
   return
   }
   // 効果音再生器 -> 3D環境
   for playerNode: AVAudioPlayerNode in playerNodes {
   engine.connect(playerNode, to: environment, format: buffer.format)
   }
   // 3D環境 -> 効果音用ミキサー
   let format: AVAudioFormat = constructOutputConnectionFormatForEnvironment()
   engine.connect(environment, to: sfxMixer, format: format)
   // BGM再生プレイヤー -> ピッチコントローラー -> BGM用ミキサー
   let stereoFormat: AVAudioFormat = AVAudioFormat(standardFormatWithSampleRate: 44100, channels: 2)!
   for (i, bgmPlayerNode) in bgmPlayerNodes.enumerated() {
   let varispeed = varispeeds[i]
   engine.connect(bgmPlayerNode, to: varispeed, format: stereoFormat)
   engine.connect(varispeed, to: bgmMixer, format: stereoFormat)
   varispeed.rate = 1.0
   }
   // 効果音用ミキサー -> リバーブ -> マスター卓
   engine.connect(sfxMixer, to: reverb, format: stereoFormat)
   engine.connect(reverb, to: engine.mainMixerNode, fromBus: 0, toBus: 0, format: stereoFormat)
   // BGMミキサー -> マスター卓
   engine.connect(bgmMixer, to: engine.mainMixerNode, fromBus: 0, toBus: 1, format: stereoFormat)
   // レンダリングアルゴリズム
   for playerNode in playerNodes {
   playerNode.renderingAlgorithm = multichannelOutputEnabled ? AVAudio3DMixingRenderingAlgorithm.soundField : AVAudio3DMixingRenderingAlgorithm.sphericalHead
   }
   for bgmPlayerNode in bgmPlayerNodes {
   bgmPlayerNode.renderingAlgorithm = multichannelOutputEnabled ? AVAudio3DMixingRenderingAlgorithm.soundField : AVAudio3DMixingRenderingAlgorithm.sphericalHead
   }
   }
   @discardableResult func startEngine() -> Bool {
   do {
   try engine.start()
   return true
   } catch {
   return false
   }
   }
   func setUp() -> Bool {
   // 他アプリの音楽などを再生できるようにする
   do {
   try AVAudioSession.sharedInstance().setCategory(.ambient)
   try AVAudioSession.sharedInstance().setActive(true)
   } catch {
   puts(error.localizedDescription)
   }
   AVAudioPlayerNodeやAVAudioMixerNodeを
   AVAudioEngineに接続
   guard !engine.isRunning else {
   return false
   }
   (0..let playerNode = AVAudioPlayerNode()
   playerNodes.append(playerNode)
   engine.attach(playerNode)
   let varispeed = AVAudioUnitVarispeed()
   varispeeds.append(varispeed)
   engine.attach(varispeed)
   let repeater = Repeater()
   repeaters.append(repeater)
   }
   bgmPlayerNodes = (0..let bgmPlayerNode = AVAudioPlayerNode()
   engine.attach(bgmPlayerNode)
   return bgmPlayerNode
   }
   environment = AVAudioEnvironmentNode()
   guard let environment: AVAudioEnvironmentNode = environment else {
   return false
   }
   engine.attach(sfxMixer)
   engine.attach(bgmMixer)
   engine.attach(environment)
   engine.attach(reverb)
   Sound.allCases.forEach { sound in
   if let buffer = loadSoundIntoBuffer(sound.rawValue) {
   buffers[sound] = buffer
   }
   }
   makeEngineConnections()
   let nc = NotificationCenter.default
   nc.addObserver(forName: NSNotification.Name.AVAudioEngineConfigurationChange, object: engine, queue: nil) {[weak self] (note: Notification) -> Void in
   guard let strongSelf = self else {
   return
   }
   /*
   strongSelf.playerNodes.forEach({ (playerNode) in
   playerNode.pause()
   })
   strongSelf.bgmPlayerNode.stop()
   */
   strongSelf.makeEngineConnections()
   strongSelf.startEngine()
   }
   // ミキサーボリューム初期値
   sfxMixer.outputVolume = Storage.sfxVolume
   bgmMixer.outputVolume = Storage.bgmVolume
   // リバーブ設定
   reverb.wetDryMix = 5
   reverb.loadFactoryPreset(.largeChamber)
   // 開始
   return startEngine()
   }
   オーディオファイルをバッファに読み込み
   音量等設定
   AVAudioEngine起動
   

   View Slide

6. コードとても減る
   let scene = SKScene(size: .zero)
   SpriteKitによるAVFoundationのセットアップに必
   要なコード:
   (実際はSKSceneの表示のためにもう少し行数必要ですが。 )
   

   View Slide

7. 手抜きポイント
   AVFoundationの知識がいらない
   初期セットアップコードがいらない
   

   View Slide

8. SKScene
   SKNode
   (-7, -6)
   SKAudioNodeを使った3D音響
   SKAudioNode
   (-7, -9)
   SKAudioNode
   (7, -8)
   listener
   (0, 0)
   X
   Y
   SKLabelNode
   (7, 3)
   SKSpriteNode
   (4, 10)
   

   View Slide

9. デモアプリ
   import SpriteKit
   final class Scene: SKScene {
   override func didMove(to view: SKView) {
   backgroundColor = .white
   let listener = SKSpriteNode(imageNamed: "listener")
   listener.verticalAlignmentMode = .center
   listener.position.x = view.bounds.midX
   listener.position.y = view.bounds.midY
   addChild(listener)
   self.listener = listener
   let speaker = SKSpriteNode(imageNamed: "speaker")
   speaker.addChild(SKAudioNode(fileNamed: "iosdc2023_01.mp3"))
   addChild(speaker)
   let radiusMax = CGFloat(320)
   let duration = TimeInterval(12)
   let revolove = SKAction.customAction(withDuration: duration, actionBlock: { node, t in
   let radius = radiusMax
   let angle = CGFloat.pi * 2 * (t / duration)
   let x = view.bounds.midX + radius * cos(angle)
   let y = view.bounds.midY + radius * sin(angle) * 0.5
   speaker.position = .init(x: x, y: y)
   })
   speaker.run(.repeatForever(revolove))
   }
   }
   

   View Slide

10. デモアプリ
    import SpriteKit
    final class Scene: SKScene {
    override func didMove(to view: SKView) {
    backgroundColor = .white
    let listener = SKSpriteNode(imageNamed: "listener")
    listener.verticalAlignmentMode = .center
    listener.position.x = view.bounds.midX
    listener.position.y = view.bounds.midY
    addChild(listener)
    self.listener = listener
    let speaker = SKSpriteNode(imageNamed: "speaker")
    speaker.addChild(SKAudioNode(fileNamed: "iosdc2023_01.mp3"))
    addChild(speaker)
    let radiusMax = CGFloat(320)
    let duration = TimeInterval(12)
    let revolove = SKAction.customAction(withDuration: duration, actionBlock: { node, t in
    let radius = radiusMax
    let angle = CGFloat.pi * 2 * (t / duration)
    let x = view.bounds.midX + radius * cos(angle)
    let y = view.bounds.midY + radius * sin(angle) * 0.5
    speaker.position = .init(x: x, y: y)
    })
    speaker.run(.repeatForever(revolove))
    }
    }
    

    View Slide

11. デモアプリ
    import SpriteKit
    final class Scene: SKScene {
    override func didMove(to view: SKView) {
    backgroundColor = .white
    let listener = SKSpriteNode(imageNamed: "listener")
    listener.verticalAlignmentMode = .center
    listener.position.x = view.bounds.midX
    listener.position.y = view.bounds.midY
    addChild(listener)
    self.listener = listener
    let speaker = SKSpriteNode(imageNamed: "speaker")
    speaker.addChild(SKAudioNode(fileNamed: "iosdc2023_01.mp3"))
    addChild(speaker)
    let radiusMax = CGFloat(320)
    let duration = TimeInterval(12)
    let revolove = SKAction.customAction(withDuration: duration, actionBlock: { node, t in
    let radius = radiusMax
    let angle = CGFloat.pi * 2 * (t / duration)
    let x = view.bounds.midX + radius * cos(angle)
    let y = view.bounds.midY + radius * sin(angle) * 0.5
    speaker.position = .init(x: x, y: y)
    })
    speaker.run(.repeatForever(revolove))
    }
    }
    

    View Slide

12. デモアプリ
    import SpriteKit
    final class Scene: SKScene {
    override func didMove(to view: SKView) {
    backgroundColor = .white
    let listener = SKSpriteNode(imageNamed: "listener")
    listener.verticalAlignmentMode = .center
    listener.position.x = view.bounds.midX
    listener.position.y = view.bounds.midY
    addChild(listener)
    self.listener = listener
    let speaker = SKSpriteNode(imageNamed: "speaker")
    speaker.addChild(SKAudioNode(fileNamed: "iosdc2023_01.mp3"))
    addChild(speaker)
    let radiusMax = CGFloat(320)
    let duration = TimeInterval(12)
    let revolove = SKAction.customAction(withDuration: duration, actionBlock: { node, t in
    let radius = radiusMax
    let angle = CGFloat.pi * 2 * (t / duration)
    let x = view.bounds.midX + radius * cos(angle)
    let y = view.bounds.midY + radius * sin(angle) * 0.5
    speaker.position = .init(x: x, y: y)
    })
    speaker.run(.repeatForever(revolove))
    }
    }
    

    View Slide

13. 実演
    

    View Slide

14. まとめ
    SpriteKitを使って3D音響を扱える
    オーディオ関連のセットアップのコードが不要
    

    View Slide

15. 補足1
    UIKit SwiftUI
    let skView = SKView(frame: .zero)
    view.addSubview(skView)
    let scene = Scene(size: .zero)
    skView.presentScene(scene)
    let scene = Scene(size: .zero)
    var body: some View {
    SpriteView(scene: scene)
    .frame(width: 0, height: 0)
    }
    SpriteKitの描画機能を使わなくてもよい
    SKSceneを画面に埋め込みつつ表示しなくても音声は再生される。
    

    View Slide

16. 補足2
    SceneKitを使えば音源配置に奥行きも
    

    View Slide

17. © GENDA Inc.
    募集中
    モバイルエンジニア、募集しています！
    他にもフロントやバックエンド、
    PDMやデザイナー、
    アナリスト、マーケティングまで幅広く募集しています。
    Now Hiring
    

    View Slide