NDIとARKitを連動させた新しい映像表現

Transcript

1. 4BUPTIJ)BUUPSJ
   /%*ͱ"3,JUΛ࿈ಈͤͨ͞
   ৽͍͠ө૾දݱ
   J04%$
   !TINEFWFMPQ
   

   View Slide

2. View Slide

3. /FUXPSL%FWJDF*OUFSGBDF
   

   View Slide

4. /%* ωοτϫʔΫɾσόΠεɾΠϯλʔϑΣʔε
   ͸ɺ
   *1ωοτϫʔΫΛܦ༝ͯ͠ϏσΦͱΦʔσΟΦετϦʔϜΛ
   ఻ૹ͢Δํࣜ *1఻ૹํࣜ
   ͷͻͱͭͰ͢ɻ
   7J[SUάϧʔϓͷ/FX5FLࣾʹΑͬͯ։ൃ͞Ε·ͨ͠ɻ
   /%*͸ɺݱࡏੈքͰ࠷΋ར༻͞Ε͍ͯΔ*1఻ૹςΫϊϩδʔͰɺ
   *1Λར༻ͨ͠ϥΠϒө૾੍࡞ϫʔΫϑϩʔࢧԉΛ໨తͱ͍ͯ͠·͢
   IUUQTXXXEQTKDPKQUFDIBSUJDMFTOEJ
   

   View Slide

5. ͳͥ/%*Λ࢖͍͍ͨͷ͔ʁ
   

   View Slide

6. View Slide

7. View Slide

8. ιϑτ΢ΣΞϕʔεͷ*1ϏσΦςΫϊϩδʔ
   ૒ํ޲௨৴ ͢΂ͯͷιʔε͕ૹ৴ݩͱಉ࣌ʹѼઌʣ
   %$5ѹॖɾ௿஗Ԇ
   ѹॖ ల։ΛԿ౓܁Γฦͯ͠΋ΫΦϦςΟʔϩεແ͠
   ղ૾౓ͱϑϨʔϜϨʔτʹґଘͤͣ
   ೚ҙͷΦʔσΟΦαϯϓϦϯάϨʔτ࢖༻Մ
   CJUΧϥʔܭࢉ
   ϏσΦ Ωʔ ΦʔσΟΦ ίϯτϩʔϧ λϦʔ ΧελϜϝλσʔλ ਖ਼֬ͳλΠϜελϯϓΛαϙʔτ
   N%/4Λ࢖༻ͨࣗ͠ಈσΟεΧόϦʔ
   IUUQTQBOEBUJNFTDPNOEJ
   

   View Slide

9. View Slide

10. Ͳ͏΍ͬͯ࢖͏ͷ͔ʁ
    

    View Slide

11. IUUQTXXXOEJUWTEL
    

    View Slide

12. View Slide

13. IUUQTBQQTBQQMFDPNVTBQQOEJIYDBNFSBJE MT
    

    View Slide

14. /%*Ͱͷૹ৴ػೳΛ࡞Δʂ
    

    View Slide

15. IUUQTHJUIVCDPNTBUPTIJ/%*4FOEFS&YBNQMF
    

    View Slide

16. View Slide

17. View Slide

18. View Slide

19. View Slide

20. View Slide

21. View Slide

22. #import
    
    
    #import
    
    
    @interface NDIWrapper : NSObject
    
    
    + (void)initialize;
    
    
    - (void)start:(NSString *)name;
    
    
    - (void)stop;
    
    
    - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata;
    
    
    @end
    %*8SBQQFS>/%*8SBQQFSI
    

    View Slide

23. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata {
    
    
    if (!my_ndi_send) {
    
    
    NSLog(@"ERROR: NDI instance is nil");
    
    
    return;
    
    
    }
    
    
    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    
    
    NDIlib_video_frame_v2_t video_frame;
    
    
    video_frame.frame_rate_N = 30000;
    
    
    video_frame.frame_rate_D = 1000;
    
    
    video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer);
    
    
    video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer);
    
    
    video_frame.FourCC = NDIlib_FourCC_type_BGRA;
    
    
    video_frame.frame_format_type = NDIlib_frame_format_type_progressive;
    
    
    video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres;
    
    
    video_frame.line_stride_in_bytes = video_frame.xres * 4;
    
    
    video_frame.p_metadata = metadata.UTF8String;
    
    
    CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
    
    
    video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);
    
    
    CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
    
    
    NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);
    
    
    }
    %*8SBQQFS>/%*8SBQQFSNൈਮ
    

    View Slide

24. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata {
    
    
    if (!my_ndi_send) {
    
    
    NSLog(@"ERROR: NDI instance is nil");
    
    
    return;
    
    
    }
    
    
    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    
    
    NDIlib_video_frame_v2_t video_frame;
    
    
    video_frame.frame_rate_N = 30000;
    
    
    video_frame.frame_rate_D = 1000;
    
    
    video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer);
    
    
    video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer);
    
    
    video_frame.FourCC = NDIlib_FourCC_type_BGRA;
    
    
    video_frame.frame_format_type = NDIlib_frame_format_type_progressive;
    
    
    video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres;
    
    
    video_frame.line_stride_in_bytes = video_frame.xres * 4;
    
    
    video_frame.p_metadata = metadata.UTF8String;
    
    
    CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
    
    
    video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);
    
    
    CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
    
    
    NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);
    
    
    }
    %*8SBQQFS>/%*8SBQQFSNൈਮ
    

    View Slide

25. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata {
    
    
    if (!my_ndi_send) {
    
    
    NSLog(@"ERROR: NDI instance is nil");
    
    
    return;
    
    
    }
    
    
    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    
    
    NDIlib_video_frame_v2_t video_frame;
    
    
    video_frame.frame_rate_N = 30000;
    
    
    video_frame.frame_rate_D = 1000;
    
    
    video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer);
    
    
    video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer);
    
    
    video_frame.FourCC = NDIlib_FourCC_type_BGRA;
    
    
    video_frame.frame_format_type = NDIlib_frame_format_type_progressive;
    
    
    video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres;
    
    
    video_frame.line_stride_in_bytes = video_frame.xres * 4;
    
    
    video_frame.p_metadata = metadata.UTF8String;
    
    
    CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
    
    
    video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);
    
    
    CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
    
    
    NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);
    
    
    }
    %*8SBQQFS>/%*8SBQQFSNൈਮ
    

    View Slide

26. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata {
    
    
    if (!my_ndi_send) {
    
    
    NSLog(@"ERROR: NDI instance is nil");
    
    
    return;
    
    
    }
    
    
    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    
    
    NDIlib_video_frame_v2_t video_frame;
    
    
    video_frame.frame_rate_N = 30000;
    
    
    video_frame.frame_rate_D = 1000;
    
    
    video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer);
    
    
    video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer);
    
    
    video_frame.FourCC = NDIlib_FourCC_type_BGRA;
    
    
    video_frame.frame_format_type = NDIlib_frame_format_type_progressive;
    
    
    video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres;
    
    
    video_frame.line_stride_in_bytes = video_frame.xres * 4;
    
    
    video_frame.p_metadata = metadata.UTF8String;
    
    
    CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
    
    
    video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);
    
    
    CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
    
    
    NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);
    
    
    }
    %*8SBQQFS>/%*8SBQQFSNൈਮ
    

    View Slide

27. View Slide

28. View Slide

29. View Slide

30. View Slide

31. class ViewController: UIViewController {
    
    
    private var ndiWrapper: NDIWrapper?
    
    
    override func viewDidLoad() {
    
    
    super.viewDidLoad()
    
    
    ndiWrapper = NDIWrapper()
    
    
    ...
    
    
    }
    
    
    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
    

    View Slide

32. class ViewController: UIViewController {
    
    
    private var ndiWrapper: NDIWrapper?
    
    
    ...
    
    
    override func viewDidLoad() {
    
    
    super.viewDidLoad()
    
    
    ndiWrapper = NDIWrapper()
    
    
    ...
    
    
    }
    
    
    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
    

    View Slide

33. func captureOutput(_ output: AVCaptureOutput,
    
    
    didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
    
    
    guard let ndiWrapper = self.ndiWrapper, isSending else { return }
    
    
    ndiWrapper.send(sampleBuffer, metadata: makeMetadata().serialize())
    
    
    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU"7$BQUVSF7JEFP%BUB0VUQVU4BNQMF#V
    ff
    FS%FMFHBUF
    

    View Slide

34. func captureOutput(_ output: AVCaptureOutput,
    
    
    didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
    
    
    guard let ndiWrapper = self.ndiWrapper, isSending else { return }
    
    
    ndiWrapper.send(sampleBuffer, metadata: makeMetadata().serialize())
    
    
    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU"7$BQUVSF7JEFP%BUB0VUQVU4BNQMF#V
    ff
    FS%FMFHBUF
    

    View Slide

35. public struct Metadata {
    
    
    public let cameraPosition: simd_float3
    
    
    public let cameraRotation: CMQuaternion
    
    
    public let projectionMatrix: matrix_float4x4
    
    
    public let depthRange: simd_float2
    
    
    public let inputState: InputState
    
    
    }
    
    
    public struct InputState {
    
    
    public let buttons: [UInt8] // 2
    
    
    public let toggles: [UInt8] // 2
    
    
    public let knobs: [UInt8] // 32
    
    
    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
    

    View Slide

36. // Debug
    
    
    func makeMetadata() -> Metadata {
    
    
    var knobs = Array(repeating: 0, count: 32)
    
    
    knobs[1] = UInt8(slider.value)
    
    
    let inputState = InputState(buttons: [0, 0],
    
    
    toggles: [0, 0],
    
    
    knobs: knobs)
    
    
    let metadata = Metadata(cameraPosition: simd_float3(0, 0, 0),
    
    
    cameraRotation: CMQuaternion(x: 0, y: 0, z: 0, w: 0),
    
    
    projectionMatrix: matrix_float4x4(SIMD4(0, 0, 0, 0),
    
    
    SIMD4(0, 0, 0, 0),
    
    
    SIMD4(0, 0, 0, 0),
    
    
    SIMD4(0, 0, 0, 0)),
    
    
    depthRange: simd_float2(0, 1),
    
    
    inputState: inputState)
    
    
    return metadata
    
    
    }
    
    
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
    

    View Slide

37. // Debug
    
    
    func makeMetadata() -> Metadata {
    
    
    var knobs = Array(repeating: 0, count: 32)
    
    
    knobs[1] = UInt8(slider.value)
    
    
    let inputState = InputState(buttons: [0, 0],
    
    
    toggles: [0, 0],
    
    
    knobs: knobs)
    
    
    let metadata = Metadata(cameraPosition: simd_float3(0, 0, 0),
    
    
    cameraRotation: CMQuaternion(x: 0, y: 0, z: 0, w: 0),
    
    
    projectionMatrix: matrix_float4x4(SIMD4(0, 0, 0, 0),
    
    
    SIMD4(0, 0, 0, 0),
    
    
    SIMD4(0, 0, 0, 0),
    
    
    SIMD4(0, 0, 0, 0)),
    
    
    depthRange: simd_float2(0, 1),
    
    
    inputState: inputState)
    
    
    return metadata
    
    
    }
    
    
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
    

    View Slide

38. /%*7JEFP.POJUPS
    

    View Slide

39. View Slide

40. View Slide

41. View Slide

42. View Slide

43. 🎉
    

    View Slide

44. QFSTPO4FHNFOUBUJPO৘ใ෇Ճ
    

    View Slide

45. override func viewWillAppear(_ animated: Bool) {
    
    
    super.viewWillAppear(animated)
    
    
    let configuration = ARWorldTrackingConfiguration()
    
    
    configuration.frameSemantics = .personSegmentationWithDepth
    
    
    session.run(configuration)
    
    
    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
    

    View Slide

46. override func viewWillAppear(_ animated: Bool) {
    
    
    super.viewWillAppear(animated)
    
    
    let configuration = ARWorldTrackingConfiguration()
    
    
    configuration.frameSemantics = .personSegmentationWithDepth
    
    
    session.run(configuration)
    
    
    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU
    

    View Slide

47. class Renderer {
    
    
    let session: ARSession
    
    
    let matteGenerator: ARMatteGenerator
    
    
    let device: MTLDevice
    
    
    let inFlightSemaphore = DispatchSemaphore(value: kMaxBuffersInFlight)
    
    
    var renderDestination: RenderDestinationProvider
    
    
    ...
    
    
    }
    
    
    func updateMatteTextures(commandBuffer: MTLCommandBuffer) {
    
    
    guard let currentFrame = session.currentFrame else {
    
    
    return
    
    
    }
    
    
    alphaTexture = matteGenerator.generateMatte(from: currentFrame, commandBuffer: commandBuffer)
    
    
    dilatedDepthTexture = matteGenerator.generateDilatedDepth(from: currentFrame, commandBuffer: commandBuffer)
    
    
    }
    
    
    %*4FOEFS&YBNQMF>3FOEFSFSTXJGU
    

    View Slide

48. 3FDFJWFSͰड৴֨͠޷ྑ͘දࣔ
    

    View Slide

49. IUUQTHJUIVCDPNLFJKJSP3DBN
    

    View Slide

50. View Slide

51. View Slide

52. View Slide

53. void UpdateMetadata()
    
    
    {
    
    
    // Deserialization
    
    
    var xml = _ndiReceiver.metadata;
    
    
    if (xml == null || xml.Length == 0) return;
    
    
    _metadata = Metadata.Deserialize(xml);
    
    
    // Input state update with the metadata
    
    
    Singletons.InputHandle.InputState = _metadata.InputState;
    
    
    }
    <3DBN7JTVBMJ[FS>3DBN3FDFJWFSDT
    

    View Slide

54. void UpdateMetadata()
    
    
    {
    
    
    // Deserialization
    
    
    var xml = _ndiReceiver.metadata;
    
    
    if (xml == null || xml.Length == 0) return;
    
    
    _metadata = Metadata.Deserialize(xml);
    
    
    // Input state update with the metadata
    
    
    Singletons.InputHandle.InputState = _metadata.InputState;
    
    
    }
    <3DBN7JTVBMJ[FS>3DBN3FDFJWFSDT
    

    View Slide

55. void UpdateMetadata()
    
    
    {
    
    
    // Deserialization
    
    
    var xml = _ndiReceiver.metadata;
    
    
    if (xml == null || xml.Length == 0) return;
    
    
    _metadata = Metadata.Deserialize(xml);
    
    
    // Input state update with the metadata
    
    
    Singletons.InputHandle.InputState = _metadata.InputState;
    
    
    }
    <3DBN7JTVBMJ[FS>3DBN3FDFJWFSDT
    

    View Slide

56. View Slide

57. View Slide

58. View Slide

59. &OKPZJ04%$ʂ🎉
    5XJUUFSͰ࠷৽৘ใπΠʔτͯ͠·͢ʂϑΥϩʔ͓ئ͍͠·͢ʂ!TINEFWFMPQ
    

    View Slide