NDIとARKitを連動させた新しい映像表現

Transcript

1. 4BUPTIJ
   )BUUPSJ /%*ͱ"3,JUΛ࿈ಈͤͨ͞
   ৽͍͠ө૾දݱ J04%$
    !TINEFWFMPQ
3. /FUXPSL
   %FWJDF
   *OUFSGBDF
4. /%* ωοτϫʔΫɾσόΠεɾΠϯλʔϑΣʔε
   ͸ɺ
   *1
   ωοτϫʔΫΛܦ༝ͯ͠ϏσΦͱΦʔσΟΦετϦʔϜΛ
   ఻ૹ͢Δํࣜ *1
   ఻ૹํࣜ ͷͻͱͭͰ͢ɻ
   7J[SU
   άϧʔϓͷ/FX5FL
   ࣾʹΑͬͯ։ൃ͞Ε·ͨ͠ɻ
   /%*
   ͸ɺݱࡏੈքͰ࠷΋ར༻͞Ε͍ͯΔ
   *1
   ఻ૹςΫϊϩδʔͰɺ
   *1
   Λར༻ͨ͠ϥΠϒө૾੍࡞ϫʔΫϑϩʔࢧԉΛ໨తͱ͍ͯ͠·͢ IUUQTXXXEQTKDPKQUFDIBSUJDMFTOEJ
5. ͳͥ/%*Λ࢖͍͍ͨͷ͔ʁ
8. 
   ιϑτ΢ΣΞϕʔεͷ*1ϏσΦςΫϊϩδʔ
   
   ૒ํ޲௨৴
   ͢΂ͯͷιʔε͕ૹ৴ݩͱಉ࣌ʹѼઌʣ
   
   %$5ѹॖɾ௿஗Ԇ
   
   ѹॖ ల։ΛԿ౓܁Γฦͯ͠΋ΫΦϦςΟʔϩεແ͠
   
   ղ૾౓ͱϑϨʔϜϨʔτʹґଘͤͣ
   
   ೚ҙͷΦʔσΟΦαϯϓϦϯάϨʔτ࢖༻Մ
   
   CJU
   Χϥʔܭࢉ
   
   ϏσΦ
   Ωʔ
   ΦʔσΟΦ
   ίϯτϩʔϧ
   λϦʔ
   ΧελϜϝλσʔλ
   ਖ਼֬ͳλΠϜελϯϓΛαϙʔτ
   
   N%/4
   Λ࢖༻ͨࣗ͠ಈσΟεΧόϦʔ IUUQTQBOEBUJNFTDPNOEJ
10. Ͳ͏΍ͬͯ࢖͏ͷ͔ʁ
11. IUUQTXXXOEJUWTEL
13. IUUQTBQQTBQQMFDPNVTBQQOEJIYDBNFSBJE MT
14. /%*Ͱͷૹ৴ػೳΛ࡞Δʂ
15. IUUQTHJUIVCDPNTBUPTIJ/%*4FOEFS&YBNQMF
22. #import <Foundation/Foundation.h> #import <AVFoundation/AVFoundation.h> @interface NDIWrapper : NSObject + (void)initialize; - (void)start:(NSString *)name; - (void)stop; - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata; @end </%*8SBQQFS>
    /%*8SBQQFSI
23. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata { if (!my_ndi_send) { NSLog(@"ERROR: NDI instance is nil"); return; } CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); NDIlib_video_frame_v2_t video_frame; video_frame.frame_rate_N = 30000; video_frame.frame_rate_D = 1000; video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer); video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer); video_frame.FourCC = NDIlib_FourCC_type_BGRA; video_frame.frame_format_type = NDIlib_frame_format_type_progressive; video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres; video_frame.line_stride_in_bytes = video_frame.xres * 4; video_frame.p_metadata = metadata.UTF8String; CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer); CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame); } </%*8SBQQFS>
    /%*8SBQQFSN
    ൈਮ
24. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata { if (!my_ndi_send) { NSLog(@"ERROR: NDI instance is nil"); return; } CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); NDIlib_video_frame_v2_t video_frame; video_frame.frame_rate_N = 30000; video_frame.frame_rate_D = 1000; video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer); video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer); video_frame.FourCC = NDIlib_FourCC_type_BGRA; video_frame.frame_format_type = NDIlib_frame_format_type_progressive; video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres; video_frame.line_stride_in_bytes = video_frame.xres * 4; video_frame.p_metadata = metadata.UTF8String; CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer); CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame); } </%*8SBQQFS>
    /%*8SBQQFSN
    ൈਮ
25. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata { if (!my_ndi_send) { NSLog(@"ERROR: NDI instance is nil"); return; } CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); NDIlib_video_frame_v2_t video_frame; video_frame.frame_rate_N = 30000; video_frame.frame_rate_D = 1000; video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer); video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer); video_frame.FourCC = NDIlib_FourCC_type_BGRA; video_frame.frame_format_type = NDIlib_frame_format_type_progressive; video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres; video_frame.line_stride_in_bytes = video_frame.xres * 4; video_frame.p_metadata = metadata.UTF8String; CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer); CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame); } </%*8SBQQFS>
    /%*8SBQQFSN
    ൈਮ
26. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata { if (!my_ndi_send) { NSLog(@"ERROR: NDI instance is nil"); return; } CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); NDIlib_video_frame_v2_t video_frame; video_frame.frame_rate_N = 30000; video_frame.frame_rate_D = 1000; video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer); video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer); video_frame.FourCC = NDIlib_FourCC_type_BGRA; video_frame.frame_format_type = NDIlib_frame_format_type_progressive; video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres; video_frame.line_stride_in_bytes = video_frame.xres * 4; video_frame.p_metadata = metadata.UTF8String; CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer); CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly); NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame); } </%*8SBQQFS>
    /%*8SBQQFSN
    ൈਮ
31. class ViewController: UIViewController { private var ndiWrapper: NDIWrapper? override func viewDidLoad() { super.viewDidLoad() ndiWrapper = NDIWrapper() ... } } </%*4FOEFS&YBNQMF>
    7JFX$POUSPMMFSTXJGU
32. class ViewController: UIViewController { private var ndiWrapper: NDIWrapper? ... override func viewDidLoad() { super.viewDidLoad() ndiWrapper = NDIWrapper() ... } } </%*4FOEFS&YBNQMF>
    7JFX$POUSPMMFSTXJGU
33. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { guard let ndiWrapper = self.ndiWrapper, isSending else { return } ndiWrapper.send(sampleBuffer, metadata: makeMetadata().serialize()) } </%*4FOEFS&YBNQMF>
    7JFX$POUSPMMFSTXJGU
    "7$BQUVSF7JEFP%BUB0VUQVU4BNQMF#V ff FS%FMFHBUF
34. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { guard let ndiWrapper = self.ndiWrapper, isSending else { return } ndiWrapper.send(sampleBuffer, metadata: makeMetadata().serialize()) } </%*4FOEFS&YBNQMF>
    7JFX$POUSPMMFSTXJGU
    "7$BQUVSF7JEFP%BUB0VUQVU4BNQMF#V ff FS%FMFHBUF
35. public struct Metadata { public let cameraPosition: simd_float3 public let cameraRotation: CMQuaternion public let projectionMatrix: matrix_float4x4 public let depthRange: simd_float2 public let inputState: InputState } public struct InputState { public let buttons: [UInt8] // 2 public let toggles: [UInt8] // 2 public let knobs: [UInt8] // 32 } </%*4FOEFS&YBNQMF>
    7JFX$POUSPMMFSTXJGU
36. // Debug func makeMetadata() -> Metadata { var knobs = Array<UInt8>(repeating: 0, count: 32) knobs[1] = UInt8(slider.value) let inputState = InputState(buttons: [0, 0], toggles: [0, 0], knobs: knobs) let metadata = Metadata(cameraPosition: simd_float3(0, 0, 0), cameraRotation: CMQuaternion(x: 0, y: 0, z: 0, w: 0), projectionMatrix: matrix_float4x4(SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0)), depthRange: simd_float2(0, 1), inputState: inputState) return metadata } </%*4FOEFS&YBNQMF>
    7JFX$POUSPMMFSTXJGU
37. // Debug func makeMetadata() -> Metadata { var knobs = Array<UInt8>(repeating: 0, count: 32) knobs[1] = UInt8(slider.value) let inputState = InputState(buttons: [0, 0], toggles: [0, 0], knobs: knobs) let metadata = Metadata(cameraPosition: simd_float3(0, 0, 0), cameraRotation: CMQuaternion(x: 0, y: 0, z: 0, w: 0), projectionMatrix: matrix_float4x4(SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0), SIMD4<Float>(0, 0, 0, 0)), depthRange: simd_float2(0, 1), inputState: inputState) return metadata } </%*4FOEFS&YBNQMF>
    7JFX$POUSPMMFSTXJGU
38. /%*
    7JEFP
    .POJUPS
43. 🎉
44. QFSTPO4FHNFOUBUJPO৘ใ෇Ճ
45. override func viewWillAppear(_ animated: Bool) { super.viewWillAppear(animated) let configuration = ARWorldTrackingConfiguration() configuration.frameSemantics = .personSegmentationWithDepth session.run(configuration) } </%*4FOEFS&YBNQMF>
    7JFX$POUSPMMFSTXJGU
46. override func viewWillAppear(_ animated: Bool) { super.viewWillAppear(animated) let configuration = ARWorldTrackingConfiguration() configuration.frameSemantics = .personSegmentationWithDepth session.run(configuration) } </%*4FOEFS&YBNQMF>
    7JFX$POUSPMMFSTXJGU
47. class Renderer { let session: ARSession let matteGenerator: ARMatteGenerator let device: MTLDevice let inFlightSemaphore = DispatchSemaphore(value: kMaxBuffersInFlight) var renderDestination: RenderDestinationProvider ... } func updateMatteTextures(commandBuffer: MTLCommandBuffer) { guard let currentFrame = session.currentFrame else { return } alphaTexture = matteGenerator.generateMatte(from: currentFrame, commandBuffer: commandBuffer) dilatedDepthTexture = matteGenerator.generateDilatedDepth(from: currentFrame, commandBuffer: commandBuffer) } </%*4FOEFS&YBNQMF>
    3FOEFSFSTXJGU
48. 3FDFJWFSͰड৴֨͠޷ྑ͘දࣔ
49. IUUQTHJUIVCDPNLFJKJSP3DBN
53. void UpdateMetadata() { // Deserialization var xml = _ndiReceiver.metadata; if (xml == null || xml.Length == 0) return; _metadata = Metadata.Deserialize(xml); // Input state update with the metadata Singletons.InputHandle.InputState = _metadata.InputState; } <3DBN7JTVBMJ[FS>
    3DBN3FDFJWFSDT
54. void UpdateMetadata() { // Deserialization var xml = _ndiReceiver.metadata; if (xml == null || xml.Length == 0) return; _metadata = Metadata.Deserialize(xml); // Input state update with the metadata Singletons.InputHandle.InputState = _metadata.InputState; } <3DBN7JTVBMJ[FS>
    3DBN3FDFJWFSDT
55. void UpdateMetadata() { // Deserialization var xml = _ndiReceiver.metadata; if (xml == null || xml.Length == 0) return; _metadata = Metadata.Deserialize(xml); // Input state update with the metadata Singletons.InputHandle.InputState = _metadata.InputState; } <3DBN7JTVBMJ[FS>
    3DBN3FDFJWFSDT
59. &OKPZ
    J04%$ʂ🎉
    5XJUUFSͰ࠷৽৘ใπΠʔτͯ͠·͢ʂϑΥϩʔ͓ئ͍͠·͢ʂ
    !TINEFWFMPQ