仮想カメラで切り開く拡張現実の世界

Transcript

1. Ծ૝ΧϝϥͰ੾Γ։֦͘ுݱ࣮ͷੈք 93,BJHJ
     ෰෦
   ஐ
   5XJUUFS
   !TINEFWFMPQ

2. Ծ૝Χϝϥʹڵຯ͕͋ΔΤϯδχΞʹಧ͚
   େܕελδΦͰ͸ͳ͘σεΫτοϓ΍εϚʔτϑΥϯ ૝ఆλʔήοτ

3. όʔνϟϧΧϝϥ࡞ΕΔʂָͦ͠͏ʂ
   ͱ͍͏஍ฏΛ։͍ͯ΄͍͠ʂ

4. ࣮૷ํ๏
   ݱ࣮֦ுͱͯ͠ͷར༻

5. Ծ૝Χϝϥ 4OBQ
   $BNFSB NNINN

6. None
7. None
8. None
9. None
10. None
11. None
12. None

13. ࣮૷ํ๏
    ݱ࣮֦ுͱͯ͠ͷར༻

14. 5XJUUFSͰ
    (8όʔνϟϧΧϝϥ࡞੒νϟϨϯδ
    Ͱ࡞੒աఔπΠʔτ͍ͯ͠·͢

15. ࡞ͬͨࡍͷۤ࿑ΛৼΓฦΓ

16. ֎෦ϥΠϒϥϦͳ͠ɺ4XJGUͷΈͰ࣮૷Մೳ

17. Ծ૝Χϝϥ࣮૷αϯϓϧ
    IUUQTHJUIVCDPNTBUPTIJ7JSUVBM$BNFSB4BNQMF

18. ߏ੒ཁૉ
    

19. ߏ੒ཁૉ
    ɹ$PSF.FEJB*0
    %"-
    1MVHJO
    ɹίϯτϩʔϥΞϓϦ
    

20. $PSF.FEJB*0
    %"-
    1MVHJO ίϯτϩʔϥΞϓϦ /41BTUFCPBSE -JCSBSZ$PSF.FEJB*01MVH*OT%"- Ծ૝Χϝϥͷ࣮ମ 6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ

21. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾ͷऔಘͱग़ྗ $PSF.FEJB*0
    %"-
    1MVHJO

22. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾ͷऔಘͱग़ྗ $PSF.FEJB*0
    %"-
    1MVHJO

23. None
24. None

25. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾Λग़ྗઃఆ $PSF.FEJB*0
    %"-
    1MVHJO

26. 1MVHJO
    GBDUPSZ
    JOUFSGBDFT
    1MVHJO
    UZQFT
    Λ௥Ճ

27. None

28. ࣗ෼Ͱࢦఆ͢Δ*%஋ɻ66*%ͳͲઃఆ

29. ΤϯτϦʔϙΠϯτ
    GVODUJPO໊

30. Ծ૝Χϝϥ1MVH*O
    ݻఆ஋ ઌड़ͷ66*%஋

31. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾Λग़ྗઃఆ $PSF.FEJB*0
    %"-
    1MVHJO

32. import Foundation import CoreMediaIO @_cdecl("VirtualCameraSampleMain") func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID)

    -> CMIOHardwarePlugInRef { return pluginRef } Main.swift

33. import Foundation import CoreMediaIO @_cdecl("VirtualCameraSampleMain") func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID)

    -> CMIOHardwarePlugInRef { return pluginRef } Main.swift ΤϯτϦʔϙΠϯτ
    ΠϯλʔϑΣΠεͷࢀরΛฦ٫

34. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾Λग़ྗઃఆ $PSF.FEJB*0
    %"-
    1MVHJO

35. private func createPluginInterface() -> CMIOHardwarePlugInInterface { return CMIOHardwarePlugInInterface( _reserved: nil,

    QueryInterface: QueryInterface, AddRef: AddRef, Release: Release, Initialize: Initialize, InitializeWithObjectID: InitializeWithObjectID, Teardown: Teardown, ObjectShow: ObjectShow, ObjectHasProperty: ObjectHasProperty, ObjectIsPropertySettable: ObjectIsPropertySettable, ObjectGetPropertyDataSize: ObjectGetPropertyDataSize, ObjectGetPropertyData: ObjectGetPropertyData, ObjectSetPropertyData: ObjectSetPropertyData, DeviceSuspend: DeviceSuspend, DeviceResume: DeviceResume, DeviceStartStream: DeviceStartStream, DeviceStopStream: DeviceStopStream, DeviceProcessAVCCommand: DeviceProcessAVCCommand, DeviceProcessRS422Command: DeviceProcessRS422Command, StreamCopyBufferQueue: StreamCopyBufferQueue, StreamDeckPlay: StreamDeckPlay, StreamDeckStop: StreamDeckStop, StreamDeckJog: StreamDeckJog, StreamDeckCueTo: StreamDeckCueTo) } PluginInterface.swift (ൈਮ)

36. private func createPluginInterface() -> CMIOHardwarePlugInInterface { return CMIOHardwarePlugInInterface( _reserved: nil,

    QueryInterface: QueryInterface, AddRef: AddRef, Release: Release, Initialize: Initialize, InitializeWithObjectID: InitializeWithObjectID, Teardown: Teardown, ObjectShow: ObjectShow, ObjectHasProperty: ObjectHasProperty, ObjectIsPropertySettable: ObjectIsPropertySettable, ObjectGetPropertyDataSize: ObjectGetPropertyDataSize, ObjectGetPropertyData: ObjectGetPropertyData, ObjectSetPropertyData: ObjectSetPropertyData, DeviceSuspend: DeviceSuspend, DeviceResume: DeviceResume, DeviceStartStream: DeviceStartStream, DeviceStopStream: DeviceStopStream, DeviceProcessAVCCommand: DeviceProcessAVCCommand, DeviceProcessRS422Command: DeviceProcessRS422Command, StreamCopyBufferQueue: StreamCopyBufferQueue, StreamDeckPlay: StreamDeckPlay, StreamDeckStop: StreamDeckStop, StreamDeckJog: StreamDeckJog, StreamDeckCueTo: StreamDeckCueTo) } PluginInterface.swift (ൈਮ) ΠϯλʔϑΣΠε͕ظ଴͢Δ΋ͷΛฦ٫

37. PluginInterface.swift (ൈਮ) ৄࡉ͸ιʔεࢀর

38. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾औಘͱग़ྗ $PSF.FEJB*0
    %"-
    1MVHJO

39. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection:

    AVCaptureConnection) { if output == cameraCapture.output { guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } let cameraImage = CIImage(cvImageBuffer: imageBuffer) let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage) var pixelBuffer: CVPixelBuffer? _ = CVPixelBufferCreate( kCFAllocatorDefault, Int(compositedImage.extent.size.width), Int(compositedImage.extent.height), kCVPixelFormatType_32BGRA, self.CVPixelBufferCreateOptions as CFDictionary, &pixelBuffer ) if let pixelBuffer = pixelBuffer { context.render(compositedImage, to: pixelBuffer) delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer) } } } VideoComposer.swift (ൈਮ)

40. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection:

    AVCaptureConnection) { if output == cameraCapture.output { guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } let cameraImage = CIImage(cvImageBuffer: imageBuffer) let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage) var pixelBuffer: CVPixelBuffer? _ = CVPixelBufferCreate( kCFAllocatorDefault, Int(compositedImage.extent.size.width), Int(compositedImage.extent.height), kCVPixelFormatType_32BGRA, self.CVPixelBufferCreateOptions as CFDictionary, &pixelBuffer ) if let pixelBuffer = pixelBuffer { context.render(compositedImage, to: pixelBuffer) delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer) } } } VideoComposer.swift (ൈਮ) ผ్ੜ੒ͨ͠ΦʔόʔϨΠςΩετը૾ͱ߹੒ EFMFHBUFܦ༝ͰQJYFM#V⒎FSΛ౉͢

41. private lazy var timer: DispatchSourceTimer = { let interval =

    1.0 / Double(frameRate) let timer = DispatchSource.makeTimerSource() timer.schedule(deadline: .now() + interval, repeating: interval) timer.setEventHandler(handler: { [weak self] in self?.enqueueBuffer() }) return timer }() Stream.swift (ൈਮ)

42. private lazy var timer: DispatchSourceTimer = { let interval =

    1.0 / Double(frameRate) let timer = DispatchSource.makeTimerSource() timer.schedule(deadline: .now() + interval, repeating: interval) timer.setEventHandler(handler: { [weak self] in self?.enqueueBuffer() }) return timer }() Stream.swift (ൈਮ) GSBNF3BUFࢦఆ͠ߋ৽ॲཧݺͼग़͠

43. private func enqueueBuffer() { (தུ) var sampleBufferUnmanaged: Unmanaged<CMSampleBuffer>? = nil

    error = CMIOSampleBufferCreateForImageBuffer( kCFAllocatorDefault, pixelBuffer, formatDescription, &timing, sequenceNumber, UInt32(kCMIOSampleBufferNoDiscontinuities), &sampleBufferUnmanaged ) guard error == noErr else { log("CMIOSampleBufferCreateForImageBuffer Error: \(error)") return } CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque()) queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon) sequenceNumber += 1 } Stream.swift (ൈਮ)

44. private func enqueueBuffer() { (தུ) var sampleBufferUnmanaged: Unmanaged<CMSampleBuffer>? = nil

    error = CMIOSampleBufferCreateForImageBuffer( kCFAllocatorDefault, pixelBuffer, formatDescription, &timing, sequenceNumber, UInt32(kCMIOSampleBufferNoDiscontinuities), &sampleBufferUnmanaged ) guard error == noErr else { log("CMIOSampleBufferCreateForImageBuffer Error: \(error)") return } CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque()) queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon) sequenceNumber += 1 } Stream.swift (ൈਮ) QJYFM#V⒎FS͔Βੜ੒ͨ͠
    $.4BNQMF#V⒎FSΛRVFVFʹ௥Ճ

45. @IBAction func sendButton_action(_ sender: Any) { SettingsPasteboard.shared.settings["text1"] = mainTextField.stringValue SettingsPasteboard.shared.update()

    } ViewController.swift (ൈਮ)
46. None

47. QMVHJOϑΝΠϧΛ഑ஔ

48. -JCSBSZ$PSF.FEJB*01MVH*OT%"-

49. දࣔ͞Εͨʂ

50. None

51. ߏ੒ཁૉ
    ɹ$PSF.FEJB*0
    %"-
    1MVHJO
    ɹίϯτϩʔϥΞϓϦ
    

52. ίϯτϩʔϥΞϓϦ

53. $PSF.FEJB*0
    %"-
    1MVHJO ίϯτϩʔϥΞϓϦ /41BTUFCPBSE -JCSBSZ$PSF.FEJB*01MVH*OT%"- Ծ૝Χϝϥͷ࣮ମ 6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ

54. 1BTUFCPBSEʹ஋Λஔ͘໾ׂ

55. ϓϩδΣΫτ࡞੒
    ίϯτϩʔϧ഑ஔ
    /41BTUFCPBSEͰσʔλڞ༗ ίϯτϩʔϥΞϓϦ

56. None
57. None

58. ϓϩδΣΫτ࡞੒
    ίϯτϩʔϧ഑ஔ
    /41BTUFCPBSEͰσʔλڞ༗ ίϯτϩʔϥΞϓϦ

59. None
60. None

61. ϓϩδΣΫτ࡞੒
    ίϯτϩʔϧ഑ஔ
    /41BTUFCPBSEͰσʔλڞ༗ ίϯτϩʔϥΞϓϦ

62. extension NSPasteboard.Name { static let main = NSPasteboard.Name(Config.mainAppBundleIdentifier) } extension

    NSPasteboard.PasteboardType { static let plain = NSPasteboard.PasteboardType(rawValue: "public.utf8-plain-text") } class SettingsPasteboard { static let shared = SettingsPasteboard() open var settings = [String: Any]() open func current() -> [String: Any] { let pasteboard = NSPasteboard(name: .main) if let element = pasteboard.pasteboardItems?.last, let str = element.string(forType: .plain), let data = str.data(using: .utf8) { do { let json = try JSONSerialization.jsonObject(with: data, options : .allowFragments) as! [String: Any] settings = json } catch { print("can't convert json") } } return settings } open func update() { let jsonStr = stringify(json: settings) let pasteboard = NSPasteboard(name: .main) pasteboard.declareTypes([.string], owner: nil) pasteboard.setString(jsonStr, forType: .string) } ... } SettingsPasteboard.swift (ൈਮ)
63. None

64. ׬੒

65. $PSF.FEJB*0
    %"-
    1MVHJO ίϯτϩʔϥΞϓϦ /41BTUFCPBSE -JCSBSZ$PSF.FEJB*01MVH*OT%"- Ծ૝Χϝϥͷ࣮ମ 6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ

66. ίϯτϩʔϥΞϓϦ͔Β஋Λૹ৴

67. None
68. None
69. None

70. ࣮૷ํ๏
    ݱ࣮֦ுͱͯ͠ͷར༻

71. 'BDFUJNFΧϝϥө૾ Ի੠ೖྗ ը૾Ճ޻ "1*Ϩεϙϯε ΩʔϘʔυೖྗ ,FZOPUFը໘ FUD ֎෦ Ծ૝Χϝϥ
    
    ग़ྗՃ޻
    ػߏ

    ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ ֤छೖྗ

72. NNINN
    ϏοάϋϯυϞʔυ

73. Ή͜͏ͷ͘ʹ
    ϑΟϧλɺϚΠϖʔδɺϝλόʔε

74. ϑΟϧλ
    ҟٞ͋Γɺ଴ͬͨ

75. ϑΟϧλ
    ΠϩϞωΞత

76. UPOBSJ
    ԕִϦΞϧλΠϜө૾

77. ϚΠϯυϚοϓ

78. None

79. δΣωϨΠςΟϒදݱ

80. ελϯϓϦΞΫγϣϯ

81. Ի੠ೝࣝͱ຋༁

82. None
83. None

84. ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ

85. ࢀߟ IUUQTEFWFMPQFSBQQMFDPNMJCSBSZBSDIJWFTBNQMFDPEF$PSF.FEJB*0*OUSPEVDUJPO*OUSPIUNM
    IUUQTHJUIVCDPNKPIOCPJMFTDPSFNFEJBJPEBMNJOJNBMFYBNQMF
    IUUQTHJUIVCDPNTFBODIBT4JNQMF%"-1MVHJO
    IUUQTTQFBLFSEFDLDPNLJTIJLBXBLBUTVNJWJSUVBMXFCDBNFSBXP[VPSPV

86. 5XJUUFSͰ࠷৽৘ใൃ৴த
    !TINEFWFMPQ
    Ծ૝Χϝϥ࣮૷αϯϓϧ
    IUUQTHJUIVCDPNTBUPTIJ7JSUVBM$BNFSB4BNQMF