Upgrade to Pro — share decks privately, control downloads, hide ads and more …

NDIとARKitを連動させた新しい映像表現

satoshi0212
September 18, 2021

 NDIとARKitを連動させた新しい映像表現

satoshi0212

September 18, 2021
Tweet

More Decks by satoshi0212

Other Decks in Programming

Transcript

  1. 4BUPTIJ)BUUPSJ
    /%*ͱ"3,JUΛ࿈ಈͤͨ͞
    ৽͍͠ө૾දݱ
    J04%$
    !TINEFWFMPQ

    View Slide

  2. View Slide

  3. /FUXPSL%FWJDF*OUFSGBDF

    View Slide

  4. /%* ωοτϫʔΫɾσόΠεɾΠϯλʔϑΣʔε
    ͸ɺ
    *1ωοτϫʔΫΛܦ༝ͯ͠ϏσΦͱΦʔσΟΦετϦʔϜΛ
    ఻ૹ͢Δํࣜ *1఻ૹํࣜ
    ͷͻͱͭͰ͢ɻ
    7J[SUάϧʔϓͷ/FX5FLࣾʹΑͬͯ։ൃ͞Ε·ͨ͠ɻ
    /%*͸ɺݱࡏੈքͰ࠷΋ར༻͞Ε͍ͯΔ*1఻ૹςΫϊϩδʔͰɺ
    *1Λར༻ͨ͠ϥΠϒө૾੍࡞ϫʔΫϑϩʔࢧԉΛ໨తͱ͍ͯ͠·͢
    IUUQTXXXEQTKDPKQUFDIBSUJDMFTOEJ

    View Slide

  5. ͳͥ/%*Λ࢖͍͍ͨͷ͔ʁ

    View Slide

  6. View Slide

  7. View Slide

  8. ιϑτ΢ΣΞϕʔεͷ*1ϏσΦςΫϊϩδʔ
    ૒ํ޲௨৴ ͢΂ͯͷιʔε͕ૹ৴ݩͱಉ࣌ʹѼઌʣ
    %$5ѹॖɾ௿஗Ԇ
    ѹॖ ల։ΛԿ౓܁Γฦͯ͠΋ΫΦϦςΟʔϩεແ͠
    ղ૾౓ͱϑϨʔϜϨʔτʹґଘͤͣ
    ೚ҙͷΦʔσΟΦαϯϓϦϯάϨʔτ࢖༻Մ
    CJUΧϥʔܭࢉ
    ϏσΦ Ωʔ ΦʔσΟΦ ίϯτϩʔϧ λϦʔ ΧελϜϝλσʔλ ਖ਼֬ͳλΠϜελϯϓΛαϙʔτ
    N%/4Λ࢖༻ͨࣗ͠ಈσΟεΧόϦʔ
    IUUQTQBOEBUJNFTDPNOEJ

    View Slide

  9. View Slide

  10. Ͳ͏΍ͬͯ࢖͏ͷ͔ʁ

    View Slide

  11. IUUQTXXXOEJUWTEL

    View Slide

  12. View Slide

  13. IUUQTBQQTBQQMFDPNVTBQQOEJIYDBNFSBJE MT

    View Slide

  14. /%*Ͱͷૹ৴ػೳΛ࡞Δʂ

    View Slide

  15. IUUQTHJUIVCDPNTBUPTIJ/%*4FOEFS&YBNQMF

    View Slide

  16. View Slide

  17. View Slide

  18. View Slide

  19. View Slide

  20. View Slide

  21. View Slide

  22. #import


    #import


    @interface NDIWrapper : NSObject


    + (void)initialize;


    - (void)start:(NSString *)name;


    - (void)stop;


    - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata;


    @end
    %*8SBQQFS>/%*8SBQQFSI

    View Slide

  23. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata {


    if (!my_ndi_send) {


    NSLog(@"ERROR: NDI instance is nil");


    return;


    }


    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);


    NDIlib_video_frame_v2_t video_frame;


    video_frame.frame_rate_N = 30000;


    video_frame.frame_rate_D = 1000;


    video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer);


    video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer);


    video_frame.FourCC = NDIlib_FourCC_type_BGRA;


    video_frame.frame_format_type = NDIlib_frame_format_type_progressive;


    video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres;


    video_frame.line_stride_in_bytes = video_frame.xres * 4;


    video_frame.p_metadata = metadata.UTF8String;


    CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);


    video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);


    CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);


    NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);


    }
    %*8SBQQFS>/%*8SBQQFSNൈਮ

    View Slide

  24. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata {


    if (!my_ndi_send) {


    NSLog(@"ERROR: NDI instance is nil");


    return;


    }


    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);


    NDIlib_video_frame_v2_t video_frame;


    video_frame.frame_rate_N = 30000;


    video_frame.frame_rate_D = 1000;


    video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer);


    video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer);


    video_frame.FourCC = NDIlib_FourCC_type_BGRA;


    video_frame.frame_format_type = NDIlib_frame_format_type_progressive;


    video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres;


    video_frame.line_stride_in_bytes = video_frame.xres * 4;


    video_frame.p_metadata = metadata.UTF8String;


    CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);


    video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);


    CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);


    NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);


    }
    %*8SBQQFS>/%*8SBQQFSNൈਮ

    View Slide

  25. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata {


    if (!my_ndi_send) {


    NSLog(@"ERROR: NDI instance is nil");


    return;


    }


    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);


    NDIlib_video_frame_v2_t video_frame;


    video_frame.frame_rate_N = 30000;


    video_frame.frame_rate_D = 1000;


    video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer);


    video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer);


    video_frame.FourCC = NDIlib_FourCC_type_BGRA;


    video_frame.frame_format_type = NDIlib_frame_format_type_progressive;


    video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres;


    video_frame.line_stride_in_bytes = video_frame.xres * 4;


    video_frame.p_metadata = metadata.UTF8String;


    CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);


    video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);


    CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);


    NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);


    }
    %*8SBQQFS>/%*8SBQQFSNൈਮ

    View Slide

  26. - (void)send:(CMSampleBufferRef)sampleBuffer metadata:(NSString *)metadata {


    if (!my_ndi_send) {


    NSLog(@"ERROR: NDI instance is nil");


    return;


    }


    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);


    NDIlib_video_frame_v2_t video_frame;


    video_frame.frame_rate_N = 30000;


    video_frame.frame_rate_D = 1000;


    video_frame.xres = (int)CVPixelBufferGetWidth(imageBuffer);


    video_frame.yres = (int)CVPixelBufferGetHeight(imageBuffer);


    video_frame.FourCC = NDIlib_FourCC_type_BGRA;


    video_frame.frame_format_type = NDIlib_frame_format_type_progressive;


    video_frame.picture_aspect_ratio = video_frame.yres / video_frame.xres;


    video_frame.line_stride_in_bytes = video_frame.xres * 4;


    video_frame.p_metadata = metadata.UTF8String;


    CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);


    video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);


    CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);


    NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);


    }
    %*8SBQQFS>/%*8SBQQFSNൈਮ

    View Slide

  27. View Slide

  28. View Slide

  29. View Slide

  30. View Slide

  31. class ViewController: UIViewController {


    private var ndiWrapper: NDIWrapper?


    override func viewDidLoad() {


    super.viewDidLoad()


    ndiWrapper = NDIWrapper()


    ...


    }


    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU

    View Slide

  32. class ViewController: UIViewController {


    private var ndiWrapper: NDIWrapper?


    ...


    override func viewDidLoad() {


    super.viewDidLoad()


    ndiWrapper = NDIWrapper()


    ...


    }


    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU

    View Slide

  33. func captureOutput(_ output: AVCaptureOutput,


    didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {


    guard let ndiWrapper = self.ndiWrapper, isSending else { return }


    ndiWrapper.send(sampleBuffer, metadata: makeMetadata().serialize())


    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU"7$BQUVSF7JEFP%BUB0VUQVU4BNQMF#V
    ff
    FS%FMFHBUF

    View Slide

  34. func captureOutput(_ output: AVCaptureOutput,


    didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {


    guard let ndiWrapper = self.ndiWrapper, isSending else { return }


    ndiWrapper.send(sampleBuffer, metadata: makeMetadata().serialize())


    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU"7$BQUVSF7JEFP%BUB0VUQVU4BNQMF#V
    ff
    FS%FMFHBUF

    View Slide

  35. public struct Metadata {


    public let cameraPosition: simd_float3


    public let cameraRotation: CMQuaternion


    public let projectionMatrix: matrix_float4x4


    public let depthRange: simd_float2


    public let inputState: InputState


    }


    public struct InputState {


    public let buttons: [UInt8] // 2


    public let toggles: [UInt8] // 2


    public let knobs: [UInt8] // 32


    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU

    View Slide

  36. // Debug


    func makeMetadata() -> Metadata {


    var knobs = Array(repeating: 0, count: 32)


    knobs[1] = UInt8(slider.value)


    let inputState = InputState(buttons: [0, 0],


    toggles: [0, 0],


    knobs: knobs)


    let metadata = Metadata(cameraPosition: simd_float3(0, 0, 0),


    cameraRotation: CMQuaternion(x: 0, y: 0, z: 0, w: 0),


    projectionMatrix: matrix_float4x4(SIMD4(0, 0, 0, 0),


    SIMD4(0, 0, 0, 0),


    SIMD4(0, 0, 0, 0),


    SIMD4(0, 0, 0, 0)),


    depthRange: simd_float2(0, 1),


    inputState: inputState)


    return metadata


    }


    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU

    View Slide

  37. // Debug


    func makeMetadata() -> Metadata {


    var knobs = Array(repeating: 0, count: 32)


    knobs[1] = UInt8(slider.value)


    let inputState = InputState(buttons: [0, 0],


    toggles: [0, 0],


    knobs: knobs)


    let metadata = Metadata(cameraPosition: simd_float3(0, 0, 0),


    cameraRotation: CMQuaternion(x: 0, y: 0, z: 0, w: 0),


    projectionMatrix: matrix_float4x4(SIMD4(0, 0, 0, 0),


    SIMD4(0, 0, 0, 0),


    SIMD4(0, 0, 0, 0),


    SIMD4(0, 0, 0, 0)),


    depthRange: simd_float2(0, 1),


    inputState: inputState)


    return metadata


    }


    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU

    View Slide

  38. /%*7JEFP.POJUPS

    View Slide

  39. View Slide

  40. View Slide

  41. View Slide

  42. View Slide

  43. 🎉

    View Slide

  44. QFSTPO4FHNFOUBUJPO৘ใ෇Ճ

    View Slide

  45. override func viewWillAppear(_ animated: Bool) {


    super.viewWillAppear(animated)


    let configuration = ARWorldTrackingConfiguration()


    configuration.frameSemantics = .personSegmentationWithDepth


    session.run(configuration)


    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU

    View Slide

  46. override func viewWillAppear(_ animated: Bool) {


    super.viewWillAppear(animated)


    let configuration = ARWorldTrackingConfiguration()


    configuration.frameSemantics = .personSegmentationWithDepth


    session.run(configuration)


    }
    %*4FOEFS&YBNQMF>7JFX$POUSPMMFSTXJGU

    View Slide

  47. class Renderer {


    let session: ARSession


    let matteGenerator: ARMatteGenerator


    let device: MTLDevice


    let inFlightSemaphore = DispatchSemaphore(value: kMaxBuffersInFlight)


    var renderDestination: RenderDestinationProvider


    ...


    }


    func updateMatteTextures(commandBuffer: MTLCommandBuffer) {


    guard let currentFrame = session.currentFrame else {


    return


    }


    alphaTexture = matteGenerator.generateMatte(from: currentFrame, commandBuffer: commandBuffer)


    dilatedDepthTexture = matteGenerator.generateDilatedDepth(from: currentFrame, commandBuffer: commandBuffer)


    }


    %*4FOEFS&YBNQMF>3FOEFSFSTXJGU

    View Slide

  48. 3FDFJWFSͰड৴֨͠޷ྑ͘දࣔ

    View Slide

  49. IUUQTHJUIVCDPNLFJKJSP3DBN

    View Slide

  50. View Slide

  51. View Slide

  52. View Slide

  53. void UpdateMetadata()


    {


    // Deserialization


    var xml = _ndiReceiver.metadata;


    if (xml == null || xml.Length == 0) return;


    _metadata = Metadata.Deserialize(xml);


    // Input state update with the metadata


    Singletons.InputHandle.InputState = _metadata.InputState;


    }
    <3DBN7JTVBMJ[FS>3DBN3FDFJWFSDT

    View Slide

  54. void UpdateMetadata()


    {


    // Deserialization


    var xml = _ndiReceiver.metadata;


    if (xml == null || xml.Length == 0) return;


    _metadata = Metadata.Deserialize(xml);


    // Input state update with the metadata


    Singletons.InputHandle.InputState = _metadata.InputState;


    }
    <3DBN7JTVBMJ[FS>3DBN3FDFJWFSDT

    View Slide

  55. void UpdateMetadata()


    {


    // Deserialization


    var xml = _ndiReceiver.metadata;


    if (xml == null || xml.Length == 0) return;


    _metadata = Metadata.Deserialize(xml);


    // Input state update with the metadata


    Singletons.InputHandle.InputState = _metadata.InputState;


    }
    <3DBN7JTVBMJ[FS>3DBN3FDFJWFSDT

    View Slide

  56. View Slide

  57. View Slide

  58. View Slide

  59. &OKPZJ04%$ʂ🎉
    5XJUUFSͰ࠷৽৘ใπΠʔτͯ͠·͢ʂϑΥϩʔ͓ئ͍͠·͢ʂ!TINEFWFMPQ

    View Slide