仮想カメラで切り開く拡張現実の世界

Transcript

1. Ծ૝ΧϝϥͰ੾Γ։֦͘ுݱ࣮ͷੈք
   93,BJHJ
   
   ෰෦ஐ5XJUUFS!TINEFWFMPQ
   

   View Slide

2. Ծ૝Χϝϥʹڵຯ͕͋ΔΤϯδχΞʹಧ͚
   େܕελδΦͰ͸ͳ͘σεΫτοϓ΍εϚʔτϑΥϯ
   ૝ఆλʔήοτ
   

   View Slide

3. όʔνϟϧΧϝϥ࡞ΕΔʂָͦ͠͏ʂ
   ͱ͍͏஍ฏΛ։͍ͯ΄͍͠ʂ
   

   View Slide

4. ࣮૷ํ๏
   ݱ࣮֦ுͱͯ͠ͷར༻
   

   View Slide

5. Ծ૝Χϝϥ
   4OBQ$BNFSB NNINN
   

   View Slide

6. View Slide

7. View Slide

8. View Slide

9. View Slide

10. View Slide

11. View Slide

12. View Slide

13. ࣮૷ํ๏
    ݱ࣮֦ுͱͯ͠ͷར༻
    

    View Slide

14. 5XJUUFSͰ(8όʔνϟϧΧϝϥ࡞੒νϟϨϯδͰ࡞੒աఔπΠʔτ͍ͯ͠·͢
    

    View Slide

15. ࡞ͬͨࡍͷۤ࿑ΛৼΓฦΓ
    
    

    View Slide

16. ֎෦ϥΠϒϥϦͳ͠ɺ4XJGUͷΈͰ࣮૷Մೳ
    

    View Slide

17. Ծ૝Χϝϥ࣮૷αϯϓϧ
    IUUQTHJUIVCDPNTBUPTIJ7JSUVBM$BNFSB4BNQMF
    

    View Slide

18. ߏ੒ཁૉ
    

    View Slide

19. ߏ੒ཁૉ
    ɹ$PSF.FEJB*0%"-1MVHJO
    ɹίϯτϩʔϥΞϓϦ
    

    View Slide

20. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ
    

    View Slide

21. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾ͷऔಘͱग़ྗ
    $PSF.FEJB*0%"-1MVHJO
    

    View Slide

22. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾ͷऔಘͱग़ྗ
    $PSF.FEJB*0%"-1MVHJO
    

    View Slide

23. View Slide

24. View Slide

25. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾Λग़ྗઃఆ
    $PSF.FEJB*0%"-1MVHJO
    

    View Slide

26. 1MVHJOGBDUPSZJOUFSGBDFT
    1MVHJOUZQFT
    Λ௥Ճ
    

    View Slide

27. View Slide

28. ࣗ෼Ͱࢦఆ͢Δ*%஋ɻ66*%ͳͲઃఆ
    

    View Slide

29. ΤϯτϦʔϙΠϯτGVODUJPO໊
    

    View Slide

30. Ծ૝Χϝϥ1MVH*Oݻఆ஋
    ઌड़ͷ66*%஋
    

    View Slide

31. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾Λग़ྗઃఆ
    $PSF.FEJB*0%"-1MVHJO
    

    View Slide

32. import Foundation
    import CoreMediaIO
    @_cdecl("VirtualCameraSampleMain")
    func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef {
    return pluginRef
    }
    Main.swift
    

    View Slide

33. import Foundation
    import CoreMediaIO
    @_cdecl("VirtualCameraSampleMain")
    func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef {
    return pluginRef
    }
    Main.swift
    ΤϯτϦʔϙΠϯτ
    ΠϯλʔϑΣΠεͷࢀরΛฦ٫
    

    View Slide

34. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾Λग़ྗઃఆ
    $PSF.FEJB*0%"-1MVHJO
    

    View Slide

35. private func createPluginInterface() -> CMIOHardwarePlugInInterface {
    return CMIOHardwarePlugInInterface(
    _reserved: nil,
    QueryInterface: QueryInterface,
    AddRef: AddRef,
    Release: Release,
    Initialize: Initialize,
    InitializeWithObjectID: InitializeWithObjectID,
    Teardown: Teardown,
    ObjectShow: ObjectShow,
    ObjectHasProperty: ObjectHasProperty,
    ObjectIsPropertySettable: ObjectIsPropertySettable,
    ObjectGetPropertyDataSize: ObjectGetPropertyDataSize,
    ObjectGetPropertyData: ObjectGetPropertyData,
    ObjectSetPropertyData: ObjectSetPropertyData,
    DeviceSuspend: DeviceSuspend,
    DeviceResume: DeviceResume,
    DeviceStartStream: DeviceStartStream,
    DeviceStopStream: DeviceStopStream,
    DeviceProcessAVCCommand: DeviceProcessAVCCommand,
    DeviceProcessRS422Command: DeviceProcessRS422Command,
    StreamCopyBufferQueue: StreamCopyBufferQueue,
    StreamDeckPlay: StreamDeckPlay,
    StreamDeckStop: StreamDeckStop,
    StreamDeckJog: StreamDeckJog,
    StreamDeckCueTo: StreamDeckCueTo)
    }
    PluginInterface.swift (ൈਮ)
    

    View Slide

36. private func createPluginInterface() -> CMIOHardwarePlugInInterface {
    return CMIOHardwarePlugInInterface(
    _reserved: nil,
    QueryInterface: QueryInterface,
    AddRef: AddRef,
    Release: Release,
    Initialize: Initialize,
    InitializeWithObjectID: InitializeWithObjectID,
    Teardown: Teardown,
    ObjectShow: ObjectShow,
    ObjectHasProperty: ObjectHasProperty,
    ObjectIsPropertySettable: ObjectIsPropertySettable,
    ObjectGetPropertyDataSize: ObjectGetPropertyDataSize,
    ObjectGetPropertyData: ObjectGetPropertyData,
    ObjectSetPropertyData: ObjectSetPropertyData,
    DeviceSuspend: DeviceSuspend,
    DeviceResume: DeviceResume,
    DeviceStartStream: DeviceStartStream,
    DeviceStopStream: DeviceStopStream,
    DeviceProcessAVCCommand: DeviceProcessAVCCommand,
    DeviceProcessRS422Command: DeviceProcessRS422Command,
    StreamCopyBufferQueue: StreamCopyBufferQueue,
    StreamDeckPlay: StreamDeckPlay,
    StreamDeckStop: StreamDeckStop,
    StreamDeckJog: StreamDeckJog,
    StreamDeckCueTo: StreamDeckCueTo)
    }
    PluginInterface.swift (ൈਮ)
    ΠϯλʔϑΣΠε͕ظ଴͢Δ΋ͷΛฦ٫
    

    View Slide

37. PluginInterface.swift (ൈਮ)
    ৄࡉ͸ιʔεࢀর
    

    View Slide

38. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾औಘͱग़ྗ
    $PSF.FEJB*0%"-1MVHJO
    

    View Slide

39. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
    ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection: AVCaptureConnection) {
    if output == cameraCapture.output {
    guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
    let cameraImage = CIImage(cvImageBuffer: imageBuffer)
    let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage)
    var pixelBuffer: CVPixelBuffer?
    _ = CVPixelBufferCreate(
    kCFAllocatorDefault,
    Int(compositedImage.extent.size.width),
    Int(compositedImage.extent.height),
    kCVPixelFormatType_32BGRA,
    self.CVPixelBufferCreateOptions as CFDictionary,
    &pixelBuffer
    )
    if let pixelBuffer = pixelBuffer {
    context.render(compositedImage, to: pixelBuffer)
    delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer)
    }
    }
    }
    VideoComposer.swift (ൈਮ)
    

    View Slide

40. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
    ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection: AVCaptureConnection) {
    if output == cameraCapture.output {
    guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
    let cameraImage = CIImage(cvImageBuffer: imageBuffer)
    let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage)
    var pixelBuffer: CVPixelBuffer?
    _ = CVPixelBufferCreate(
    kCFAllocatorDefault,
    Int(compositedImage.extent.size.width),
    Int(compositedImage.extent.height),
    kCVPixelFormatType_32BGRA,
    self.CVPixelBufferCreateOptions as CFDictionary,
    &pixelBuffer
    )
    if let pixelBuffer = pixelBuffer {
    context.render(compositedImage, to: pixelBuffer)
    delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer)
    }
    }
    }
    VideoComposer.swift (ൈਮ)
    ผ్ੜ੒ͨ͠ΦʔόʔϨΠςΩετը૾ͱ߹੒
    EFMFHBUFܦ༝ͰQJYFM#V⒎FSΛ౉͢
    

    View Slide

41. private lazy var timer: DispatchSourceTimer = {
    let interval = 1.0 / Double(frameRate)
    let timer = DispatchSource.makeTimerSource()
    timer.schedule(deadline: .now() + interval, repeating: interval)
    timer.setEventHandler(handler: { [weak self] in
    self?.enqueueBuffer()
    })
    return timer
    }()
    Stream.swift (ൈਮ)
    

    View Slide

42. private lazy var timer: DispatchSourceTimer = {
    let interval = 1.0 / Double(frameRate)
    let timer = DispatchSource.makeTimerSource()
    timer.schedule(deadline: .now() + interval, repeating: interval)
    timer.setEventHandler(handler: { [weak self] in
    self?.enqueueBuffer()
    })
    return timer
    }()
    Stream.swift (ൈਮ)
    GSBNF3BUFࢦఆ͠ߋ৽ॲཧݺͼग़͠
    

    View Slide

43. private func enqueueBuffer() {
    (தུ)
    var sampleBufferUnmanaged: Unmanaged? = nil
    error = CMIOSampleBufferCreateForImageBuffer(
    kCFAllocatorDefault,
    pixelBuffer,
    formatDescription,
    &timing,
    sequenceNumber,
    UInt32(kCMIOSampleBufferNoDiscontinuities),
    &sampleBufferUnmanaged
    )
    guard error == noErr else {
    log("CMIOSampleBufferCreateForImageBuffer Error: \(error)")
    return
    }
    CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque())
    queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon)
    sequenceNumber += 1
    }
    Stream.swift (ൈਮ)
    

    View Slide

44. private func enqueueBuffer() {
    (தུ)
    var sampleBufferUnmanaged: Unmanaged? = nil
    error = CMIOSampleBufferCreateForImageBuffer(
    kCFAllocatorDefault,
    pixelBuffer,
    formatDescription,
    &timing,
    sequenceNumber,
    UInt32(kCMIOSampleBufferNoDiscontinuities),
    &sampleBufferUnmanaged
    )
    guard error == noErr else {
    log("CMIOSampleBufferCreateForImageBuffer Error: \(error)")
    return
    }
    CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque())
    queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon)
    sequenceNumber += 1
    }
    Stream.swift (ൈਮ)
    QJYFM#V⒎FS͔Βੜ੒ͨ͠
    $.4BNQMF#V⒎FSΛRVFVFʹ௥Ճ
    

    View Slide

45. @IBAction func sendButton_action(_ sender: Any) {
    SettingsPasteboard.shared.settings["text1"] = mainTextField.stringValue
    SettingsPasteboard.shared.update()
    }
    ViewController.swift (ൈਮ)
    

    View Slide

46. View Slide

47. QMVHJOϑΝΠϧΛ഑ஔ
    

    View Slide

48. -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    

    View Slide

49. දࣔ͞Εͨʂ
    

    View Slide

50. View Slide

51. ߏ੒ཁૉ
    ɹ$PSF.FEJB*0%"-1MVHJO
    ɹίϯτϩʔϥΞϓϦ
    

    View Slide

52. ίϯτϩʔϥΞϓϦ
    

    View Slide

53. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ
    

    View Slide

54. 1BTUFCPBSEʹ஋Λஔ͘໾ׂ
    

    View Slide

55. ϓϩδΣΫτ࡞੒
    ίϯτϩʔϧ഑ஔ
    /41BTUFCPBSEͰσʔλڞ༗
    ίϯτϩʔϥΞϓϦ
    

    View Slide

56. View Slide

57. View Slide

58. ϓϩδΣΫτ࡞੒
    ίϯτϩʔϧ഑ஔ
    /41BTUFCPBSEͰσʔλڞ༗
    ίϯτϩʔϥΞϓϦ
    

    View Slide

59. View Slide

60. View Slide

61. ϓϩδΣΫτ࡞੒
    ίϯτϩʔϧ഑ஔ
    /41BTUFCPBSEͰσʔλڞ༗
    ίϯτϩʔϥΞϓϦ
    

    View Slide

62. extension NSPasteboard.Name {
    static let main = NSPasteboard.Name(Config.mainAppBundleIdentifier)
    }
    extension NSPasteboard.PasteboardType {
    static let plain = NSPasteboard.PasteboardType(rawValue: "public.utf8-plain-text")
    }
    class SettingsPasteboard {
    static let shared = SettingsPasteboard()
    open var settings = [String: Any]()
    open func current() -> [String: Any] {
    let pasteboard = NSPasteboard(name: .main)
    if let element = pasteboard.pasteboardItems?.last, let str = element.string(forType: .plain), let data = str.data(using: .utf8) {
    do {
    let json = try JSONSerialization.jsonObject(with: data, options : .allowFragments) as! [String: Any]
    settings = json
    } catch {
    print("can't convert json")
    }
    }
    return settings
    }
    open func update() {
    let jsonStr = stringify(json: settings)
    let pasteboard = NSPasteboard(name: .main)
    pasteboard.declareTypes([.string], owner: nil)
    pasteboard.setString(jsonStr, forType: .string)
    }
    ...
    }
    SettingsPasteboard.swift (ൈਮ)
    

    View Slide

63. View Slide

64. ׬੒
    

    View Slide

65. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ
    

    View Slide

66. ίϯτϩʔϥΞϓϦ͔Β஋Λૹ৴
    

    View Slide

67. View Slide

68. View Slide

69. View Slide

70. ࣮૷ํ๏
    ݱ࣮֦ுͱͯ͠ͷར༻
    

    View Slide

71. 'BDFUJNFΧϝϥө૾
    Ի੠ೖྗ
    ը૾Ճ޻
    "1*Ϩεϙϯε
    ΩʔϘʔυೖྗ
    ,FZOPUFը໘
    FUD
    ֎෦ Ծ૝Χϝϥ
    ग़ྗՃ޻ػߏ
    ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ
    ֤छೖྗ
    

    View Slide

72. NNINNϏοάϋϯυϞʔυ
    

    View Slide

73. Ή͜͏ͷ͘ʹϑΟϧλɺϚΠϖʔδɺϝλόʔε
    

    View Slide

74. ϑΟϧλҟٞ͋Γɺ଴ͬͨ
    

    View Slide

75. ϑΟϧλΠϩϞωΞత
    

    View Slide

76. UPOBSJԕִϦΞϧλΠϜө૾
    

    View Slide

77. ϚΠϯυϚοϓ
    

    View Slide

78. View Slide

79. δΣωϨΠςΟϒදݱ
    

    View Slide

80. ελϯϓϦΞΫγϣϯ
    

    View Slide

81. Ի੠ೝࣝͱ຋༁
    

    View Slide

82. View Slide

83. View Slide

84. ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ
    

    View Slide

85. ࢀߟ
    IUUQTEFWFMPQFSBQQMFDPNMJCSBSZBSDIJWFTBNQMFDPEF$PSF.FEJB*0*OUSPEVDUJPO*OUSPIUNM
    IUUQTHJUIVCDPNKPIOCPJMFTDPSFNFEJBJPEBMNJOJNBMFYBNQMF
    IUUQTHJUIVCDPNTFBODIBT4JNQMF%"-1MVHJO
    IUUQTTQFBLFSEFDLDPNLJTIJLBXBLBUTVNJWJSUVBMXFCDBNFSBXP[VPSPV
    

    View Slide

86. 5XJUUFSͰ࠷৽৘ใൃ৴த
    !TINEFWFMPQ
    Ծ૝Χϝϥ࣮૷αϯϓϧ
    IUUQTHJUIVCDPNTBUPTIJ7JSUVBM$BNFSB4BNQMF
    

    View Slide