Upgrade to Pro — share decks privately, control downloads, hide ads and more …

TwilioのCameraSource に任意のフレームを使用するすゝめ

22e056dcb79e14363e844f1e57986325?s=47 BOB
December 22, 2020

TwilioのCameraSource に任意のフレームを使用するすゝめ

potatotips 72

22e056dcb79e14363e844f1e57986325?s=128

BOB

December 22, 2020
Tweet

Transcript

  1. ʲJ04ʳ5XJMJPͷ$BNFSB4PVSDF ʹ೚ҙͷϑϨʔϜΛ࢖༻͢Δ͢ʍΊ "3"5":0,0:"#0#!QPUBUPUJQTJ04"OESPJE։ൃ5JQTڞ༗ձ

  2. !#0#(7J "SBUB:PLPZBNB#0#  # ळݩ߁ؔ࿈ʹ10೥Ҏ্අ΍͢ # ๺ւಓੜ·Ε # ͸ͩͯ͜ະདྷେଔ #

    2020೥αΠόʔΤʔδΣϯτ৽ଔೖࣾ # λοϓϧͷiOSΤϯδχΞ
  3. λοϓϧ  w ୅޲͚ͷϚονϯάΞϓϦ w ೥݄ϦϦʔε w ϑϦοΫ6* w ຊਓೝূɼը໘࿥ը๷ࢭ

    w 6*͕உঁͰେ͖͘ҧ͏ w 5XJMJPΛ࢖༻ͨ͠ϏσΦνϟοτ
  4. λοϓϧɿϏσΦνϟοτ  w ݄ϏσΦνϟοτϦϦʔε w 5XJMJPΛ࢖༻͓ͯ͠Γ·͢ w ݄ϑΟϧλʔػೳϦϦʔε w ʮ͓෦԰ͷย෇͚͕େมʯɼ

    ʮ਎ͩ͠ͳΈΛ੔͑Δͷʹ͕͔͔࣌ؒΔʯ ͱ͍ͬͨ੠Λड͚ͯ։ൃ͍ͨ͠·ͨ͠ w ϑΟϧλʔػೳ͸"3,JU΋࢖༻͓ͯ͠Γ·͢ ϚονϯάΞϓϦʮλοϓϧ஀ੜʯɺ͙͢ʹΦϯϥΠϯσʔτ͕Ͱ͖Δ ʮϏσΦνϟοτʯʹ৽ͨʹʮϑΟϧλʔػೳʯΛ௥Ճɿhttps://www.cyberagent.co.jp/news/detail/id=25083 5XJMJPհͯ͠೚ҙͷϑϨʔϜΛૹΔํ๏Λ͝঺հ
  5.  (0"- ΞδΣϯμ CameraSourceʹ೚ҙͷϑϨʔϜΛ࢖༻͢ΔϙΠϯτ͸˓˓  $BNFSB4PVSDFૹΔʹ͸˓˓͕େ੾  "7$BQUVSFɿϑϨʔϜͷऔΓํ  "3,JUɿϑϨʔϜͷऔΓํ

  6. $BNFSB4PVSDFʹૹΔʹ͸˓˓͕େ੾

  7. $BNFSB4PVSDFʹૹΔʹ͸˓˓͕େ੾  // TVIVideoFrame - (nullable instancetype)initWithTimestamp:(CMTime)timestamp buffer:(nonnull CVImageBufferRef)imageBuffer orientation:(TVIVideoOrientation)orientation;

    - (nullable instancetype)initWithTimeInterval:(CFTimeInterval)timeInterval buffer:(nonnull CVImageBufferRef)imageBuffer orientation:(TVIVideoOrientation)orientation; TVIVideoFrame Class Referenceɿhttps://twilio.github.io/twilio-video-ios/docs/latest/Classes/TVIVideoFrame.html#//api/name/initWithTimeInterval:buffer:orientation:
  8. $BNFSB4PVSDFʹૹΔʹ͸˓˓͕େ੾  // TVIVideoFrame - (nullable instancetype)initWithTimestamp:(CMTime)timestamp buffer:(nonnull CVImageBufferRef)imageBuffer orientation:(TVIVideoOrientation)orientation;

    - (nullable instancetype)initWithTimeInterval:(CFTimeInterval)timeInterval buffer:(nonnull CVImageBufferRef)imageBuffer orientation:(TVIVideoOrientation)orientation; CVImageBufferRef CVImageBufferRef TVIVideoFrame Class Referenceɿhttps://twilio.github.io/twilio-video-ios/docs/latest/Classes/TVIVideoFrame.html#//api/name/initWithTimeInterval:buffer:orientation:
  9. $BNFSB4PVSDFʹૹΔʹ͸˓˓͕େ੾  // TVIVideoFrame - (nullable instancetype)initWithTimestamp:(CMTime)timestamp buffer:(nonnull CVImageBufferRef)imageBuffer orientation:(TVIVideoOrientation)orientation;

    - (nullable instancetype)initWithTimeInterval:(CFTimeInterval)timeInterval buffer:(nonnull CVImageBufferRef)imageBuffer orientation:(TVIVideoOrientation)orientation; CMTime CFTimeInterval ϑϨʔϜΛऔಘ͢Δ Ճ޻͢Δ 7JEFP'SBNFΛ ࡞੒͢Δ ྲྀ͠ࠐΉ " # TVIVideoFrame Class Referenceɿhttps://twilio.github.io/twilio-video-ios/docs/latest/Classes/TVIVideoFrame.html#//api/name/initWithTimeInterval:buffer:orientation: $ %
  10. $BNFSB4PVSDFʹૹΔʹ͸4JOL͕େ੾  import TwilioVideo final class LocalTrackSourceManager: NSObject { private(set)

    var camera: CameraSource! func onVideoFrame(image: CIImage, time: TimeInterval) { guard let pixelBuffer = image.convertCVPixelBuffer() else { return } guard let videoFrame = VideoFrame(timeInterval: time, buffer: pixelBuffer, orientation: VideoOrientation.up) else { return } camera.sink?.onVideoFrame(videoFrame) } func onVideoFrame(image: CIImage, time: CMTime) { guard let pixelBuffer = image.convertCVPixelBuffer() else { return } guard let videoFrame = VideoFrame(timestamp: time, buffer: pixelBuffer, orientation: VideoOrientation.up) else { return } camera.sink?.onVideoFrame(videoFrame) } } # $ %
  11. import TwilioVideo final class LocalTrackSourceManager: NSObject { private(set) var camera:

    CameraSource! func onVideoFrame(image: CIImage, time: TimeInterval) { guard let pixelBuffer = image.convertCVPixelBuffer() else { return } guard let videoFrame = VideoFrame(timeInterval: time, buffer: pixelBuffer, orientation: VideoOrientation.up) else { return } camera.sink?.onVideoFrame(videoFrame) } func onVideoFrame(image: CIImage, time: CMTime) { guard let pixelBuffer = image.convertCVPixelBuffer() else { return } guard let videoFrame = VideoFrame(timestamp: time, buffer: pixelBuffer, orientation: VideoOrientation.up) else { return } camera.sink?.onVideoFrame(videoFrame) } } $BNFSB4PVSDFʹૹΔʹ͸4JOL͕େ੾  # $ % guard let pixelBuffer = image.convertCVPixelBuffer() else { return } guard let pixelBuffer = image.convertCVPixelBuffer() else { return }
  12. import TwilioVideo final class LocalTrackSourceManager: NSObject { private(set) var camera:

    CameraSource! func onVideoFrame(image: CIImage, time: TimeInterval) { guard let pixelBuffer = image.convertCVPixelBuffer() else { return } guard let videoFrame = VideoFrame(timeInterval: time, buffer: pixelBuffer, orientation: VideoOrientation.up) else { return } camera.sink?.onVideoFrame(videoFrame) } func onVideoFrame(image: CIImage, time: CMTime) { guard let pixelBuffer = image.convertCVPixelBuffer() else { return } guard let videoFrame = VideoFrame(timestamp: time, buffer: pixelBuffer, orientation: VideoOrientation.up) else { return } camera.sink?.onVideoFrame(videoFrame) } } $BNFSB4PVSDFʹૹΔʹ͸4JOL͕େ੾  # $ % VideoFrame(timeInterval: time, buffer: pixelBuffer, orientation: VideoOrientation.up) VideoFrame(timestamp: time, buffer: pixelBuffer, orientation: VideoOrientation.up)
  13. import TwilioVideo final class LocalTrackSourceManager: NSObject { private(set) var camera:

    CameraSource! func onVideoFrame(image: CIImage, time: TimeInterval) { guard let pixelBuffer = image.convertCVPixelBuffer() else { return } guard let videoFrame = VideoFrame(timeInterval: time, buffer: pixelBuffer, orientation: VideoOrientation.up) else { return } camera.sink?.onVideoFrame(videoFrame) } func onVideoFrame(image: CIImage, time: CMTime) { guard let pixelBuffer = image.convertCVPixelBuffer() else { return } guard let videoFrame = VideoFrame(timestamp: time, buffer: pixelBuffer, orientation: VideoOrientation.up) else { return } camera.sink?.onVideoFrame(videoFrame) } } $BNFSB4PVSDFʹૹΔʹ͸4JOL͕େ੾  camera.sink?.onVideoFrame(videoFrame) camera.sink?.onVideoFrame(videoFrame)
  14. $BNFSB4PVSDFʹૹΔʹ͸4JOL͕େ੾  import TwilioVideo final class LocalTrackSourceManager: NSObject { private(set)

    var camera: CameraSource! func onVideoFrame(image: CIImage, time: TimeInterval) { guard let pixelBuffer = image.convertCVPixelBuffer() else { return } guard let videoFrame = VideoFrame(timeInterval: time, buffer: pixelBuffer, orientation: VideoOrientation.up) else { return } camera.sink?.onVideoFrame(videoFrame) } func onVideoFrame(image: CIImage, time: CMTime) { guard let pixelBuffer = image.convertCVPixelBuffer() else { return } guard let videoFrame = VideoFrame(timestamp: time, buffer: pixelBuffer, orientation: VideoOrientation.up) else { return } camera.sink?.onVideoFrame(videoFrame) } } camera.sink?.onVideoFrame(videoFrame) camera.sink?.onVideoFrame(videoFrame) extension VideoChatAVCaptureView: VideoSource { weak var sink: VideoSink? var isScreencast: Bool { return false } func requestOutputFormat(_ outputFormat: VideoFormat) { guard let sink = sink else { return } sink.onVideoFormatRequest(outputFormat) } } w 7JEFP4PVSDFʹ४ڌͤͯ͞TJOLΛ࣋ͭ͜ͱ΋Մೳ w ͨͩ͠ɼৗʹ4JOL΁ϑϨʔϜΛૹΔ͜ͱ͕৚݅ ʢΫϥογϡ͠·͢ʣ w Ұํɼ$BNFSB4PVSDFͷ4JOLʹ͸ͦͷ৚͕݅ͳ͍ 5JQT var camera: CameraSource!
  15. "7$BQUVSFɿϑϨʔϜͷऔΓํ

  16. "7$BQUVSFɿϑϨʔϜͷऔΓํ  import AVFoundation import UIKit final class VideoChatAVCaptureView: UIView

    { private var session: AVCaptureSession? private func prepare() { let output: AVCaptureVideoDataOutput = AVCaptureVideoDataOutput() output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoChat")) } } extension VideoChatAVCaptureView: AVCaptureVideoDataOutputSampleBufferDelegate { func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { guard let ciImage = sampleBuffer.convertCIImage() else { return } let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) localTrackSourceManager.onVideoFrame(image: ciImage, time: timestamp) } } " #
  17. "7$BQUVSFɿϑϨʔϜͷऔΓํ  import AVFoundation import UIKit final class VideoChatAVCaptureView: UIView

    { private var session: AVCaptureSession? private func prepare() { let output: AVCaptureVideoDataOutput = AVCaptureVideoDataOutput() output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoChat")) } } extension VideoChatAVCaptureView: AVCaptureVideoDataOutputSampleBufferDelegate { func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { guard let ciImage = sampleBuffer.convertCIImage() else { return } let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) localTrackSourceManager.onVideoFrame(image: ciImage, time: timestamp) } } output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoChat")) EFMFHBUFΛࣗ਎ʹ͋ͯ·͢
  18. "7$BQUVSFɿϑϨʔϜͷऔΓํ  import AVFoundation import UIKit final class VideoChatAVCaptureView: UIView

    { private var session: AVCaptureSession? private func prepare() { let output: AVCaptureVideoDataOutput = AVCaptureVideoDataOutput() output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoChat")) } } extension VideoChatAVCaptureView: AVCaptureVideoDataOutputSampleBufferDelegate { func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { guard let ciImage = sampleBuffer.convertCIImage() else { return } let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) localTrackSourceManager.onVideoFrame(image: ciImage, time: timestamp) } } guard let ciImage = sampleBuffer.convertCIImage() else { return } $.4BNQMF#V⒎FSΛ$**NBHFʹม׵͠·͢
  19. "7$BQUVSFɿϑϨʔϜͷऔΓํ  import AVFoundation import UIKit final class VideoChatAVCaptureView: UIView

    { private var session: AVCaptureSession? private func prepare() { let output: AVCaptureVideoDataOutput = AVCaptureVideoDataOutput() output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoChat")) } } extension VideoChatAVCaptureView: AVCaptureVideoDataOutputSampleBufferDelegate { func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { guard let ciImage = sampleBuffer.convertCIImage() else { return } let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) localTrackSourceManager.onVideoFrame(image: ciImage, time: timestamp) } } let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) $.5JNFΛ࡞੒͠·͢
  20. "3,JUɿϑϨʔϜͷऔΓํ

  21. "3,JUɿϑϨʔϜͷऔΓํ  import ARKit import SceneKit import UIKit final class

    VideoChatARKitView: UIView { private lazy var sceneView: ARSCNView = {...}() private func prepare() { sceneView.delegate = self } } extension VideoChatARKitView: ARSCNViewDelegate { func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) { guard let sceneView = renderer as? ARSCNView else { return } guard let ciImage = CIImage(image: sceneView.snapshot()) else { return } localTrackSourceManager.onVideoFrame(image: ciImage, time: time) } } " #
  22. import ARKit import SceneKit import UIKit final class VideoChatARKitView: UIView

    { private lazy var sceneView: ARSCNView = {...}() private func prepare() { sceneView.delegate = self } } extension VideoChatARKitView: ARSCNViewDelegate { func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) { guard let sceneView = renderer as? ARSCNView else { return } guard let ciImage = CIImage(image: sceneView.snapshot()) else { return } localTrackSourceManager.onVideoFrame(image: ciImage, time: time) } } "3,JUɿϑϨʔϜͷऔΓํ  sceneView.delegate = self EFMFHBUFΛࣗ਎ʹ͋ͯ·͢
  23. "3,JUɿϑϨʔϜͷऔΓํ  import ARKit import SceneKit import UIKit final class

    VideoChatARKitView: UIView { private lazy var sceneView: ARSCNView = {...}() private func prepare() { sceneView.delegate = self } } extension VideoChatARKitView: ARSCNViewDelegate { func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) { guard let sceneView = renderer as? ARSCNView else { return } guard let ciImage = CIImage(image: sceneView.snapshot()) else { return } localTrackSourceManager.onVideoFrame(image: ciImage, time: time) } } let ciImage = CIImage(image: sceneView.snapshot()) εφοϓγϣοτΛऔಘ͠·͢
  24. "3,JUɿϑϨʔϜͷऔΓํ  import ARKit import SceneKit import UIKit final class

    VideoChatARKitView: UIView { private lazy var sceneView: ARSCNView = {...}() private func prepare() { sceneView.delegate = self } } extension VideoChatARKitView: ARSCNViewDelegate { func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) { guard let sceneView = renderer as? ARSCNView else { return } guard let ciImage = CIImage(image: sceneView.snapshot()) else { return } localTrackSourceManager.onVideoFrame(image: ciImage, time: time) } } time: TimeInterval ͦͷ··౉͢͜ͱ͕Ͱ͖·͢
  25. ·ͱΊ

  26. ·ͱΊ  (0"- εςοϓ CameraSourceʹ೚ҙͷϑϨʔϜΛ࢖༻͢ΔϙΠϯτ͸Sink ϑϨʔϜΛऔಘ͢Δ Ճ޻͢Δ 7JEFP'SBNFΛ ࡞੒͢Δ ྲྀ͠ࠐΉ

    " # $ %