Upgrade to Pro — share decks privately, control downloads, hide ads and more …

NDIとARKitを連動させた新しい映像表現

1a1d418bdf51cbf8fce1317f6c80a907?s=47 satoshi0212
September 18, 2021

 NDIとARKitを連動させた新しい映像表現

1a1d418bdf51cbf8fce1317f6c80a907?s=128

satoshi0212

September 18, 2021
Tweet

Transcript

  1. 4BUPTIJ)BUUPSJ /%*ͱ"3,JUΛ࿈ಈͤͨ͞ ৽͍͠ө૾දݱ J04%$ !TINEFWFMPQ

  2. None
  3. /FUXPSL%FWJDF*OUFSGBDF

  4. /%* ωοτϫʔΫɾσόΠεɾΠϯλʔϑΣʔε ͸ɺ *1ωοτϫʔΫΛܦ༝ͯ͠ϏσΦͱΦʔσΟΦετϦʔϜΛ ఻ૹ͢Δํࣜ *1఻ૹํࣜ ͷͻͱͭͰ͢ɻ 7J[SUάϧʔϓͷ/FX5FLࣾʹΑͬͯ։ൃ͞Ε·ͨ͠ɻ /%*͸ɺݱࡏੈքͰ࠷΋ར༻͞Ε͍ͯΔ*1఻ૹςΫϊϩδʔͰɺ *1Λར༻ͨ͠ϥΠϒө૾੍࡞ϫʔΫϑϩʔࢧԉΛ໨తͱ͍ͯ͠·͢

    IUUQTXXXEQTKDPKQUFDIBSUJDMFTOEJ
  5. ͳͥ/%*Λ࢖͍͍ͨͷ͔ʁ

  6. None
  7. None
  8. ιϑτ΢ΣΞϕʔεͷ*1ϏσΦςΫϊϩδʔ ૒ํ޲௨৴ ͢΂ͯͷιʔε͕ૹ৴ݩͱಉ࣌ʹѼઌʣ %$5ѹॖɾ௿஗Ԇ ѹॖ ల։ΛԿ౓܁Γฦͯ͠΋ΫΦϦςΟʔϩεແ͠ ղ૾౓ͱϑϨʔϜϨʔτʹґଘͤͣ ೚ҙͷΦʔσΟΦαϯϓϦϯάϨʔτ࢖༻Մ CJUΧϥʔܭࢉ ϏσΦ

    Ωʔ ΦʔσΟΦ ίϯτϩʔϧ λϦʔ ΧελϜϝλσʔλ ਖ਼֬ͳλΠϜελϯϓΛαϙʔτ N%/4Λ࢖༻ͨࣗ͠ಈσΟεΧόϦʔ IUUQTQBOEBUJNFTDPNOEJ
  9. None
  10. Ͳ͏΍ͬͯ࢖͏ͷ͔ʁ

  11. IUUQTXXXOEJUWTEL

  12. None
  13. IUUQTBQQTBQQMFDPNVTBQQOEJIYDBNFSBJE MT

  14. /%*Ͱͷૹ৴ػೳΛ࡞Δʂ

  15. IUUQTHJUIVCDPNTBUPTIJ/%*4FOEFS&YBNQMF

  16. None
  17. None
  18. None
  19. None
  20. None
  21. None
  22. #import <Foundation/Foundation.h> #import <AVFoundation/AVFoundation.h> @interface NDIWrapper : NSObject + (void)initialize;

    - (void)start:(NSString *)name; - (void)stop; - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata; @end </%*8SBQQFS>/%*8SBQQFSI
  23. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata { if (!my_ndi_send) { NSLog(@"ERROR: NDI

    instance is nil"); return; } CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); NDIlib_video_frame_v2_t video_frame; video_frame.frame_rate_N = 30000; video_frame.frame_rate_D = 1000; video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer); video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer); video_frame.FourCC = NDIlib_FourCC_type_BGRA; video_frame.frame_format_type = NDIlib_frame_format_type_progressive; video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres; video_frame.line_stride_in_bytes = video_frame.xres * 4; video_frame.p_metadata = metadata.UTF8String; CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer); CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame); } </%*8SBQQFS>/%*8SBQQFSNൈਮ
  24. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata { if (!my_ndi_send) { NSLog(@"ERROR: NDI

    instance is nil"); return; } CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); NDIlib_video_frame_v2_t video_frame; video_frame.frame_rate_N = 30000; video_frame.frame_rate_D = 1000; video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer); video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer); video_frame.FourCC = NDIlib_FourCC_type_BGRA; video_frame.frame_format_type = NDIlib_frame_format_type_progressive; video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres; video_frame.line_stride_in_bytes = video_frame.xres * 4; video_frame.p_metadata = metadata.UTF8String; CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer); CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame); } </%*8SBQQFS>/%*8SBQQFSNൈਮ
  25. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata { if (!my_ndi_send) { NSLog(@"ERROR: NDI

    instance is nil"); return; } CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); NDIlib_video_frame_v2_t video_frame; video_frame.frame_rate_N = 30000; video_frame.frame_rate_D = 1000; video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer); video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer); video_frame.FourCC = NDIlib_FourCC_type_BGRA; video_frame.frame_format_type = NDIlib_frame_format_type_progressive; video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres; video_frame.line_stride_in_bytes = video_frame.xres * 4; video_frame.p_metadata = metadata.UTF8String; CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer); CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame); } </%*8SBQQFS>/%*8SBQQFSNൈਮ
  26. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata { if (!my_ndi_send) { NSLog(@"ERROR: NDI

    instance is nil"); return; } CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); NDIlib_video_frame_v2_t video_frame; video_frame.frame_rate_N = 30000; video_frame.frame_rate_D = 1000; video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer); video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer); video_frame.FourCC = NDIlib_FourCC_type_BGRA; video_frame.frame_format_type = NDIlib_frame_format_type_progressive; video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres; video_frame.line_stride_in_bytes = video_frame.xres * 4; video_frame.p_metadata = metadata.UTF8String; CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer); CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame); } </%*8SBQQFS>/%*8SBQQFSNൈਮ
  27. None
  28. None
  29. None
  30. None
  31. class ViewController: UIViewController { private var ndiWrapper: NDIWrapper? override func

    viewDidLoad() { super.viewDidLoad() ndiWrapper = NDIWrapper() ... } } </%*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
  32. class ViewController: UIViewController { private var ndiWrapper: NDIWrapper? ... override

    func viewDidLoad() { super.viewDidLoad() ndiWrapper = NDIWrapper() ... } } </%*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
  33. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection)

    { guard let ndiWrapper = self.ndiWrapper, isSending else { return } ndiWrapper.send(sampleBuffer, metadata: makeMetadata().serialize()) } </%*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU"7$BQUVSF7JEFP%BUB0VUQVU4BNQMF#V ff FS%FMFHBUF
  34. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection)

    { guard let ndiWrapper = self.ndiWrapper, isSending else { return } ndiWrapper.send(sampleBuffer, metadata: makeMetadata().serialize()) } </%*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU"7$BQUVSF7JEFP%BUB0VUQVU4BNQMF#V ff FS%FMFHBUF
  35. public struct Metadata { public let cameraPosition: simd_float3 public let

    cameraRotation: CMQuaternion public let projectionMatrix: matrix_float4x4 public let depthRange: simd_float2 public let inputState: InputState } public struct InputState { public let buttons: [UInt8] // 2 public let toggles: [UInt8] // 2 public let knobs: [UInt8] // 32 } </%*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
  36. // Debug func makeMetadata() -> Metadata { var knobs =

    Array<UInt8>(repeating: 0, count: 32) knobs[1] = UInt8(slider.value) let inputState = InputState(buttons: [0, 0], toggles: [0, 0], knobs: knobs) let metadata = Metadata(cameraPosition: simd_float3(0, 0, 0), cameraRotation: CMQuaternion(x: 0, y: 0, z: 0, w: 0), projectionMatrix: matrix_float4x4(SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0)), depthRange: simd_float2(0, 1), inputState: inputState) return metadata } </%*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
  37. // Debug func makeMetadata() -> Metadata { var knobs =

    Array<UInt8>(repeating: 0, count: 32) knobs[1] = UInt8(slider.value) let inputState = InputState(buttons: [0, 0], toggles: [0, 0], knobs: knobs) let metadata = Metadata(cameraPosition: simd_float3(0, 0, 0), cameraRotation: CMQuaternion(x: 0, y: 0, z: 0, w: 0), projectionMatrix: matrix_float4x4(SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0)), depthRange: simd_float2(0, 1), inputState: inputState) return metadata } </%*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
  38. /%*7JEFP.POJUPS

  39. None
  40. None
  41. None
  42. None
  43. 🎉

  44. QFSTPO4FHNFOUBUJPO৘ใ෇Ճ

  45. override func viewWillAppear(_ animated: Bool) { super.viewWillAppear(animated) let configuration =

    ARWorldTrackingConfiguration() configuration.frameSemantics = .personSegmentationWithDepth session.run(configuration) } </%*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
  46. override func viewWillAppear(_ animated: Bool) { super.viewWillAppear(animated) let configuration =

    ARWorldTrackingConfiguration() configuration.frameSemantics = .personSegmentationWithDepth session.run(configuration) } </%*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
  47. class Renderer { let session: ARSession let matteGenerator: ARMatteGenerator let

    device: MTLDevice let inFlightSemaphore = DispatchSemaphore(value: kMaxBuffersInFlight) var renderDestination: RenderDestinationProvider ... } func updateMatteTextures(commandBuffer: MTLCommandBuffer) { guard let currentFrame = session.currentFrame else { return } alphaTexture = matteGenerator.generateMatte(from: currentFrame, commandBuffer: commandBuffer) dilatedDepthTexture = matteGenerator.generateDilatedDepth(from: currentFrame, commandBuffer: commandBuffer) } </%*4FOEFS&YBNQMF>3FOEFSFSTXJGU
  48. 3FDFJWFSͰड৴֨͠޷ྑ͘දࣔ

  49. IUUQTHJUIVCDPNLFJKJSP3DBN

  50. None
  51. None
  52. None
  53. void UpdateMetadata() { // Deserialization var xml = _ndiReceiver.metadata; if

    (xml == null || xml.Length == 0) return; _metadata = Metadata.Deserialize(xml); // Input state update with the metadata Singletons.InputHandle.InputState = _metadata.InputState; } <3DBN7JTVBMJ[FS>3DBN3FDFJWFSDT
  54. void UpdateMetadata() { // Deserialization var xml = _ndiReceiver.metadata; if

    (xml == null || xml.Length == 0) return; _metadata = Metadata.Deserialize(xml); // Input state update with the metadata Singletons.InputHandle.InputState = _metadata.InputState; } <3DBN7JTVBMJ[FS>3DBN3FDFJWFSDT
  55. void UpdateMetadata() { // Deserialization var xml = _ndiReceiver.metadata; if

    (xml == null || xml.Length == 0) return; _metadata = Metadata.Deserialize(xml); // Input state update with the metadata Singletons.InputHandle.InputState = _metadata.InputState; } <3DBN7JTVBMJ[FS>3DBN3FDFJWFSDT
  56. None
  57. None
  58. None
  59. &OKPZJ04%$ʂ🎉 5XJUUFSͰ࠷৽৘ใπΠʔτͯ͠·͢ʂϑΥϩʔ͓ئ͍͠·͢ʂ!TINEFWFMPQ