macOS仮想カメラ「テロップカム」 実装方法とその先

Transcript

1. NBD04Ծ૝ΧϝϥʮςϩοϓΧϜʯ
   ࣮૷ํ๏ͱͦͷઌ
   J04%$+BQBO
   
   ෰෦ஐ!TINEFWFMPQ
   

   View Slide

2. Ծ૝Χϝϥ
   4OBQ$BNFSB NNINN
   

   View Slide

3. ͜Ε͸ָ͍͠ʂ
   

   View Slide

4. ϓϩάϥϚͳΒҰ౓͸
   

   View Slide

5. ࣗ࡞ͨ͘͠ͳΓ·͢ΑͶʂ
   

   View Slide

6. ྑ͍͓஌Βͤ
   

   View Slide

7. ͳΜͱ
   

   View Slide

8. 4XJGUͷΈͰ࣮૷Մೳʂ
   

   View Slide

9. 5XJUUFSͰ(8όʔνϟϧΧϝϥ࡞੒νϟϨϯδͰ࡞੒աఔπΠʔτ͍ͯ͠·͢
   

   View Slide

10. View Slide

11. View Slide

12. ߏ੒ཁૉ
    

    View Slide

13. ߏ੒ཁૉ
    ɹ$PSF.FEJB*0%"-1MVHJO
    ɹίϯτϩʔϥΞϓϦ
    

    View Slide

14. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ
    

    View Slide

15. ϓϩδΣΫτ࡞੒ͱΠϯλʔϑΣΠε࣮૷
    $PSF.FEJB*0%"-1MVHJO
    

    View Slide

16. View Slide

17. View Slide

18. View Slide

19. ࣗ෼Ͱࢦఆ͢Δ*%஋ɻ66*%ͳͲઃఆ
    

    View Slide

20. ΤϯτϦʔϙΠϯτGVODUJPO໊
    

    View Slide

21. Ծ૝Χϝϥ1MVH*Oݻఆ஋
    ઌड़ͷ66*%஋
    

    View Slide

22. import Foundation
    import CoreMediaIO
    @_cdecl("VirtualCameraSampleMain")
    func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef {
    return pluginRef
    }
    Main.swift
    

    View Slide

23. import Foundation
    import CoreMediaIO
    @_cdecl("VirtualCameraSampleMain")
    func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef {
    return pluginRef
    }
    Main.swift
    ΤϯτϦʔϙΠϯτ
    ΠϯλʔϑΣΠεͷࢀরΛฦ٫
    

    View Slide

24. let pluginRef: CMIOHardwarePlugInRef = {
    let interfacePtr = UnsafeMutablePointer.allocate(capacity: 1)
    interfacePtr.pointee = createPluginInterface()
    let pluginRef = CMIOHardwarePlugInRef.allocate(capacity: 1)
    pluginRef.pointee = interfacePtr
    return pluginRef
    }()
    PluginInterface.swift (ൈਮ)
    

    View Slide

25. let pluginRef: CMIOHardwarePlugInRef = {
    let interfacePtr = UnsafeMutablePointer.allocate(capacity: 1)
    interfacePtr.pointee = createPluginInterface()
    let pluginRef = CMIOHardwarePlugInRef.allocate(capacity: 1)
    pluginRef.pointee = interfacePtr
    return pluginRef
    }()
    PluginInterface.swift (ൈਮ)
    ΠϯλʔϑΣΠεࢀরΛฦ٫
    

    View Slide

26. private func createPluginInterface() -> CMIOHardwarePlugInInterface {
    return CMIOHardwarePlugInInterface(
    _reserved: nil,
    QueryInterface: QueryInterface,
    AddRef: AddRef,
    Release: Release,
    Initialize: Initialize,
    InitializeWithObjectID: InitializeWithObjectID,
    Teardown: Teardown,
    ObjectShow: ObjectShow,
    ObjectHasProperty: ObjectHasProperty,
    ObjectIsPropertySettable: ObjectIsPropertySettable,
    ObjectGetPropertyDataSize: ObjectGetPropertyDataSize,
    ObjectGetPropertyData: ObjectGetPropertyData,
    ObjectSetPropertyData: ObjectSetPropertyData,
    DeviceSuspend: DeviceSuspend,
    DeviceResume: DeviceResume,
    DeviceStartStream: DeviceStartStream,
    DeviceStopStream: DeviceStopStream,
    DeviceProcessAVCCommand: DeviceProcessAVCCommand,
    DeviceProcessRS422Command: DeviceProcessRS422Command,
    StreamCopyBufferQueue: StreamCopyBufferQueue,
    StreamDeckPlay: StreamDeckPlay,
    StreamDeckStop: StreamDeckStop,
    StreamDeckJog: StreamDeckJog,
    StreamDeckCueTo: StreamDeckCueTo)
    }
    PluginInterface.swift (ൈਮ)
    

    View Slide

27. private func createPluginInterface() -> CMIOHardwarePlugInInterface {
    return CMIOHardwarePlugInInterface(
    _reserved: nil,
    QueryInterface: QueryInterface,
    AddRef: AddRef,
    Release: Release,
    Initialize: Initialize,
    InitializeWithObjectID: InitializeWithObjectID,
    Teardown: Teardown,
    ObjectShow: ObjectShow,
    ObjectHasProperty: ObjectHasProperty,
    ObjectIsPropertySettable: ObjectIsPropertySettable,
    ObjectGetPropertyDataSize: ObjectGetPropertyDataSize,
    ObjectGetPropertyData: ObjectGetPropertyData,
    ObjectSetPropertyData: ObjectSetPropertyData,
    DeviceSuspend: DeviceSuspend,
    DeviceResume: DeviceResume,
    DeviceStartStream: DeviceStartStream,
    DeviceStopStream: DeviceStopStream,
    DeviceProcessAVCCommand: DeviceProcessAVCCommand,
    DeviceProcessRS422Command: DeviceProcessRS422Command,
    StreamCopyBufferQueue: StreamCopyBufferQueue,
    StreamDeckPlay: StreamDeckPlay,
    StreamDeckStop: StreamDeckStop,
    StreamDeckJog: StreamDeckJog,
    StreamDeckCueTo: StreamDeckCueTo)
    }
    PluginInterface.swift (ൈਮ)
    ΠϯλʔϑΣΠε͕ظ଴͢Δ΋ͷΛฦ٫
    

    View Slide

28. PluginInterface.swift (ൈਮ)
    ৄࡉ͸ιʔεࢀর
    

    View Slide

29. Χϝϥө૾औಘͱग़ྗ
    

    View Slide

30. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
    ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection: AVCaptureConnection) {
    if output == cameraCapture.output {
    guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
    let cameraImage = CIImage(cvImageBuffer: imageBuffer)
    let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage)
    var pixelBuffer: CVPixelBuffer?
    _ = CVPixelBufferCreate(
    kCFAllocatorDefault,
    Int(compositedImage.extent.size.width),
    Int(compositedImage.extent.height),
    kCVPixelFormatType_32BGRA,
    self.CVPixelBufferCreateOptions as CFDictionary,
    &pixelBuffer
    )
    if let pixelBuffer = pixelBuffer {
    context.render(compositedImage, to: pixelBuffer)
    delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer)
    }
    }
    }
    VideoComposer.swift (ൈਮ)
    

    View Slide

31. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
    ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection: AVCaptureConnection) {
    if output == cameraCapture.output {
    guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
    let cameraImage = CIImage(cvImageBuffer: imageBuffer)
    let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage)
    var pixelBuffer: CVPixelBuffer?
    _ = CVPixelBufferCreate(
    kCFAllocatorDefault,
    Int(compositedImage.extent.size.width),
    Int(compositedImage.extent.height),
    kCVPixelFormatType_32BGRA,
    self.CVPixelBufferCreateOptions as CFDictionary,
    &pixelBuffer
    )
    if let pixelBuffer = pixelBuffer {
    context.render(compositedImage, to: pixelBuffer)
    delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer)
    }
    }
    }
    VideoComposer.swift (ൈਮ)
    ผ్ੜ੒ͨ͠ΦʔόʔϨΠςΩετը૾ͱ߹੒
    EFMFHBUFܦ༝ͰQJYFM#V⒎FSΛ౉͢
    

    View Slide

32. private func enqueueBuffer() {
    (தུ)
    var sampleBufferUnmanaged: Unmanaged? = nil
    error = CMIOSampleBufferCreateForImageBuffer(
    kCFAllocatorDefault,
    pixelBuffer,
    formatDescription,
    &timing,
    sequenceNumber,
    UInt32(kCMIOSampleBufferNoDiscontinuities),
    &sampleBufferUnmanaged
    )
    guard error == noErr else {
    log("CMIOSampleBufferCreateForImageBuffer Error: \(error)")
    return
    }
    CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque())
    queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon)
    sequenceNumber += 1
    }
    Stream.swift (ൈਮ)
    

    View Slide

33. private func enqueueBuffer() {
    (தུ)
    var sampleBufferUnmanaged: Unmanaged? = nil
    error = CMIOSampleBufferCreateForImageBuffer(
    kCFAllocatorDefault,
    pixelBuffer,
    formatDescription,
    &timing,
    sequenceNumber,
    UInt32(kCMIOSampleBufferNoDiscontinuities),
    &sampleBufferUnmanaged
    )
    guard error == noErr else {
    log("CMIOSampleBufferCreateForImageBuffer Error: \(error)")
    return
    }
    CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque())
    queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon)
    sequenceNumber += 1
    }
    Stream.swift (ൈਮ)
    QJYFM#V⒎FS͔Βੜ੒ͨ͠
    $.4BNQMF#V⒎FSΛRVFVFʹ௥Ճ
    

    View Slide

34. QMVHJOϑΝΠϧͷ഑ஔ
    

    View Slide

35. View Slide

36. -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    

    View Slide

37. දࣔ͞Εͨʂ
    

    View Slide

38. ίϯτϩʔϥΞϓϦ
    

    View Slide

39. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ
    

    View Slide

40. 1BTUFCPBSEʹ஋Λஔ͘໾ׂ
    

    View Slide

41. View Slide

42. /41BTUFCPBSEͰσʔλڞ༗
    

    View Slide

43. extension NSPasteboard.Name {
    static let main = NSPasteboard.Name(Config.mainAppBundleIdentifier)
    }
    extension NSPasteboard.PasteboardType {
    static let plain = NSPasteboard.PasteboardType(rawValue: "public.utf8-plain-text")
    }
    class SettingsPasteboard {
    static let shared = SettingsPasteboard()
    open var settings = [String: Any]()
    open func current() -> [String: Any] {
    let pasteboard = NSPasteboard(name: .main)
    if let element = pasteboard.pasteboardItems?.last, let str = element.string(forType: .plain), let data = str.data(using: .utf8) {
    do {
    let json = try JSONSerialization.jsonObject(with: data, options : .allowFragments) as! [String: Any]
    settings = json
    } catch {
    print("can't convert json")
    }
    }
    return settings
    }
    open func update() {
    let jsonStr = stringify(json: settings)
    let pasteboard = NSPasteboard(name: .main)
    pasteboard.declareTypes([.string], owner: nil)
    pasteboard.setString(jsonStr, forType: .string)
    }
    ...
    }
    SettingsPasteboard.swift (ൈਮ)
    

    View Slide

44. extension NSPasteboard.Name {
    static let main = NSPasteboard.Name(Config.mainAppBundleIdentifier)
    }
    extension NSPasteboard.PasteboardType {
    static let plain = NSPasteboard.PasteboardType(rawValue: "public.utf8-plain-text")
    }
    class SettingsPasteboard {
    static let shared = SettingsPasteboard()
    open var settings = [String: Any]()
    open func current() -> [String: Any] {
    let pasteboard = NSPasteboard(name: .main)
    if let element = pasteboard.pasteboardItems?.last, let str = element.string(forType: .plain), let data = str.data(using: .utf8) {
    do {
    let json = try JSONSerialization.jsonObject(with: data, options : .allowFragments) as! [String: Any]
    settings = json
    } catch {
    print("can't convert json")
    }
    }
    return settings
    }
    open func update() {
    let jsonStr = stringify(json: settings)
    let pasteboard = NSPasteboard(name: .main)
    pasteboard.declareTypes([.string], owner: nil)
    pasteboard.setString(jsonStr, forType: .string)
    }
    ...
    }
    SettingsPasteboard.swift (ൈਮ)
    

    View Slide

45. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ
    

    View Slide

46. ׬੒
    

    View Slide

47. ίϯτϩʔϥΞϓϦ͔Β஋Λૹ৴
    

    View Slide

48. View Slide

49. View Slide

50. View Slide

51. ࣮૷ํ๏ͱʮͦͷઌʯ
    

    View Slide

52. 'BDFUJNFDBNFSB
    BVEJPJOQVU
    TIBEFSF⒎FDU
    "1*
    LFZCPBSEJOQVU
    ,FZOPUF
    FUD
    ֎෦ Ծ૝Χϝϥ
    ೖग़ྗՃ޻ػߏ
    ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ
    ֤छೖྗ
    

    View Slide

53. ϚΠϯυϚοϓ
    

    View Slide

54. δΣωϨΠςΟϒදݱ
    

    View Slide

55. ελϯϓϦΞΫγϣϯ
    

    View Slide

56. Ի੠ೝࣝͱ຋༁
    

    View Slide

57. ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ
    

    View Slide

58. ࢀߟ
    IUUQTEFWFMPQFSBQQMFDPNMJCSBSZBSDIJWFTBNQMFDPEF$PSF.FEJB*0*OUSPEVDUJPO*OUSPIUNM
    IUUQTHJUIVCDPNKPIOCPJMFTDPSFNFEJBJPEBMNJOJNBMFYBNQMF
    IUUQTHJUIVCDPNTFBODIBT4JNQMF%"-1MVHJO
    IUUQTTQFBLFSEFDLDPNLJTIJLBXBLBUTVNJWJSUVBMXFCDBNFSBXP[VPSPV
    

    View Slide

59. 5XJUUFSͰ࠷৽৘ใൃ৴த
    !TINEFWFMPQ
    Ծ૝Χϝϥ࣮૷αϯϓϧ
    IUUQTHJUIVCDPNTBUPTIJ7JSUVBM$BNFSB4BNQMF
    

    View Slide