Upgrade to Pro — share decks privately, control downloads, hide ads and more …

macOS仮想カメラ「テロップカム」 実装方法とその先

satoshi0212
September 20, 2020

macOS仮想カメラ「テロップカム」 実装方法とその先

satoshi0212

September 20, 2020
Tweet

More Decks by satoshi0212

Other Decks in Programming

Transcript

  1. NBD04Ծ૝ΧϝϥʮςϩοϓΧϜʯ
    ࣮૷ํ๏ͱͦͷઌ
    J04%$+BQBO

    ෰෦ஐ!TINEFWFMPQ

    View Slide

  2. Ծ૝Χϝϥ
    4OBQ$BNFSB NNINN

    View Slide

  3. ͜Ε͸ָ͍͠ʂ

    View Slide

  4. ϓϩάϥϚͳΒҰ౓͸

    View Slide

  5. ࣗ࡞ͨ͘͠ͳΓ·͢ΑͶʂ

    View Slide

  6. ྑ͍͓஌Βͤ

    View Slide

  7. ͳΜͱ

    View Slide

  8. 4XJGUͷΈͰ࣮૷Մೳʂ

    View Slide

  9. 5XJUUFSͰ(8όʔνϟϧΧϝϥ࡞੒νϟϨϯδͰ࡞੒աఔπΠʔτ͍ͯ͠·͢

    View Slide

  10. View Slide

  11. View Slide

  12. ߏ੒ཁૉ

    View Slide

  13. ߏ੒ཁૉ
    ɹ$PSF.FEJB*0%"-1MVHJO
    ɹίϯτϩʔϥΞϓϦ

    View Slide

  14. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ

    View Slide

  15. ϓϩδΣΫτ࡞੒ͱΠϯλʔϑΣΠε࣮૷
    $PSF.FEJB*0%"-1MVHJO

    View Slide

  16. View Slide

  17. View Slide

  18. View Slide

  19. ࣗ෼Ͱࢦఆ͢Δ*%஋ɻ66*%ͳͲઃఆ

    View Slide

  20. ΤϯτϦʔϙΠϯτGVODUJPO໊

    View Slide

  21. Ծ૝Χϝϥ1MVH*Oݻఆ஋
    ઌड़ͷ66*%஋

    View Slide

  22. import Foundation
    import CoreMediaIO
    @_cdecl("VirtualCameraSampleMain")
    func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef {
    return pluginRef
    }
    Main.swift

    View Slide

  23. import Foundation
    import CoreMediaIO
    @_cdecl("VirtualCameraSampleMain")
    func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef {
    return pluginRef
    }
    Main.swift
    ΤϯτϦʔϙΠϯτ
    ΠϯλʔϑΣΠεͷࢀরΛฦ٫

    View Slide

  24. let pluginRef: CMIOHardwarePlugInRef = {
    let interfacePtr = UnsafeMutablePointer.allocate(capacity: 1)
    interfacePtr.pointee = createPluginInterface()
    let pluginRef = CMIOHardwarePlugInRef.allocate(capacity: 1)
    pluginRef.pointee = interfacePtr
    return pluginRef
    }()
    PluginInterface.swift (ൈਮ)

    View Slide

  25. let pluginRef: CMIOHardwarePlugInRef = {
    let interfacePtr = UnsafeMutablePointer.allocate(capacity: 1)
    interfacePtr.pointee = createPluginInterface()
    let pluginRef = CMIOHardwarePlugInRef.allocate(capacity: 1)
    pluginRef.pointee = interfacePtr
    return pluginRef
    }()
    PluginInterface.swift (ൈਮ)
    ΠϯλʔϑΣΠεࢀরΛฦ٫

    View Slide

  26. private func createPluginInterface() -> CMIOHardwarePlugInInterface {
    return CMIOHardwarePlugInInterface(
    _reserved: nil,
    QueryInterface: QueryInterface,
    AddRef: AddRef,
    Release: Release,
    Initialize: Initialize,
    InitializeWithObjectID: InitializeWithObjectID,
    Teardown: Teardown,
    ObjectShow: ObjectShow,
    ObjectHasProperty: ObjectHasProperty,
    ObjectIsPropertySettable: ObjectIsPropertySettable,
    ObjectGetPropertyDataSize: ObjectGetPropertyDataSize,
    ObjectGetPropertyData: ObjectGetPropertyData,
    ObjectSetPropertyData: ObjectSetPropertyData,
    DeviceSuspend: DeviceSuspend,
    DeviceResume: DeviceResume,
    DeviceStartStream: DeviceStartStream,
    DeviceStopStream: DeviceStopStream,
    DeviceProcessAVCCommand: DeviceProcessAVCCommand,
    DeviceProcessRS422Command: DeviceProcessRS422Command,
    StreamCopyBufferQueue: StreamCopyBufferQueue,
    StreamDeckPlay: StreamDeckPlay,
    StreamDeckStop: StreamDeckStop,
    StreamDeckJog: StreamDeckJog,
    StreamDeckCueTo: StreamDeckCueTo)
    }
    PluginInterface.swift (ൈਮ)

    View Slide

  27. private func createPluginInterface() -> CMIOHardwarePlugInInterface {
    return CMIOHardwarePlugInInterface(
    _reserved: nil,
    QueryInterface: QueryInterface,
    AddRef: AddRef,
    Release: Release,
    Initialize: Initialize,
    InitializeWithObjectID: InitializeWithObjectID,
    Teardown: Teardown,
    ObjectShow: ObjectShow,
    ObjectHasProperty: ObjectHasProperty,
    ObjectIsPropertySettable: ObjectIsPropertySettable,
    ObjectGetPropertyDataSize: ObjectGetPropertyDataSize,
    ObjectGetPropertyData: ObjectGetPropertyData,
    ObjectSetPropertyData: ObjectSetPropertyData,
    DeviceSuspend: DeviceSuspend,
    DeviceResume: DeviceResume,
    DeviceStartStream: DeviceStartStream,
    DeviceStopStream: DeviceStopStream,
    DeviceProcessAVCCommand: DeviceProcessAVCCommand,
    DeviceProcessRS422Command: DeviceProcessRS422Command,
    StreamCopyBufferQueue: StreamCopyBufferQueue,
    StreamDeckPlay: StreamDeckPlay,
    StreamDeckStop: StreamDeckStop,
    StreamDeckJog: StreamDeckJog,
    StreamDeckCueTo: StreamDeckCueTo)
    }
    PluginInterface.swift (ൈਮ)
    ΠϯλʔϑΣΠε͕ظ଴͢Δ΋ͷΛฦ٫

    View Slide

  28. PluginInterface.swift (ൈਮ)
    ৄࡉ͸ιʔεࢀর

    View Slide

  29. Χϝϥө૾औಘͱग़ྗ

    View Slide

  30. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
    ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection: AVCaptureConnection) {
    if output == cameraCapture.output {
    guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
    let cameraImage = CIImage(cvImageBuffer: imageBuffer)
    let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage)
    var pixelBuffer: CVPixelBuffer?
    _ = CVPixelBufferCreate(
    kCFAllocatorDefault,
    Int(compositedImage.extent.size.width),
    Int(compositedImage.extent.height),
    kCVPixelFormatType_32BGRA,
    self.CVPixelBufferCreateOptions as CFDictionary,
    &pixelBuffer
    )
    if let pixelBuffer = pixelBuffer {
    context.render(compositedImage, to: pixelBuffer)
    delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer)
    }
    }
    }
    VideoComposer.swift (ൈਮ)

    View Slide

  31. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
    ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection: AVCaptureConnection) {
    if output == cameraCapture.output {
    guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
    let cameraImage = CIImage(cvImageBuffer: imageBuffer)
    let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage)
    var pixelBuffer: CVPixelBuffer?
    _ = CVPixelBufferCreate(
    kCFAllocatorDefault,
    Int(compositedImage.extent.size.width),
    Int(compositedImage.extent.height),
    kCVPixelFormatType_32BGRA,
    self.CVPixelBufferCreateOptions as CFDictionary,
    &pixelBuffer
    )
    if let pixelBuffer = pixelBuffer {
    context.render(compositedImage, to: pixelBuffer)
    delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer)
    }
    }
    }
    VideoComposer.swift (ൈਮ)
    ผ్ੜ੒ͨ͠ΦʔόʔϨΠςΩετը૾ͱ߹੒
    EFMFHBUFܦ༝ͰQJYFM#V⒎FSΛ౉͢

    View Slide

  32. private func enqueueBuffer() {
    (தུ)
    var sampleBufferUnmanaged: Unmanaged? = nil
    error = CMIOSampleBufferCreateForImageBuffer(
    kCFAllocatorDefault,
    pixelBuffer,
    formatDescription,
    &timing,
    sequenceNumber,
    UInt32(kCMIOSampleBufferNoDiscontinuities),
    &sampleBufferUnmanaged
    )
    guard error == noErr else {
    log("CMIOSampleBufferCreateForImageBuffer Error: \(error)")
    return
    }
    CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque())
    queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon)
    sequenceNumber += 1
    }
    Stream.swift (ൈਮ)

    View Slide

  33. private func enqueueBuffer() {
    (தུ)
    var sampleBufferUnmanaged: Unmanaged? = nil
    error = CMIOSampleBufferCreateForImageBuffer(
    kCFAllocatorDefault,
    pixelBuffer,
    formatDescription,
    &timing,
    sequenceNumber,
    UInt32(kCMIOSampleBufferNoDiscontinuities),
    &sampleBufferUnmanaged
    )
    guard error == noErr else {
    log("CMIOSampleBufferCreateForImageBuffer Error: \(error)")
    return
    }
    CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque())
    queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon)
    sequenceNumber += 1
    }
    Stream.swift (ൈਮ)
    QJYFM#V⒎FS͔Βੜ੒ͨ͠
    $.4BNQMF#V⒎FSΛRVFVFʹ௥Ճ

    View Slide

  34. QMVHJOϑΝΠϧͷ഑ஔ

    View Slide

  35. View Slide

  36. -JCSBSZ$PSF.FEJB*01MVH*OT%"-

    View Slide

  37. දࣔ͞Εͨʂ

    View Slide

  38. ίϯτϩʔϥΞϓϦ

    View Slide

  39. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ

    View Slide

  40. 1BTUFCPBSEʹ஋Λஔ͘໾ׂ

    View Slide

  41. View Slide

  42. /41BTUFCPBSEͰσʔλڞ༗

    View Slide

  43. extension NSPasteboard.Name {
    static let main = NSPasteboard.Name(Config.mainAppBundleIdentifier)
    }
    extension NSPasteboard.PasteboardType {
    static let plain = NSPasteboard.PasteboardType(rawValue: "public.utf8-plain-text")
    }
    class SettingsPasteboard {
    static let shared = SettingsPasteboard()
    open var settings = [String: Any]()
    open func current() -> [String: Any] {
    let pasteboard = NSPasteboard(name: .main)
    if let element = pasteboard.pasteboardItems?.last, let str = element.string(forType: .plain), let data = str.data(using: .utf8) {
    do {
    let json = try JSONSerialization.jsonObject(with: data, options : .allowFragments) as! [String: Any]
    settings = json
    } catch {
    print("can't convert json")
    }
    }
    return settings
    }
    open func update() {
    let jsonStr = stringify(json: settings)
    let pasteboard = NSPasteboard(name: .main)
    pasteboard.declareTypes([.string], owner: nil)
    pasteboard.setString(jsonStr, forType: .string)
    }
    ...
    }
    SettingsPasteboard.swift (ൈਮ)

    View Slide

  44. extension NSPasteboard.Name {
    static let main = NSPasteboard.Name(Config.mainAppBundleIdentifier)
    }
    extension NSPasteboard.PasteboardType {
    static let plain = NSPasteboard.PasteboardType(rawValue: "public.utf8-plain-text")
    }
    class SettingsPasteboard {
    static let shared = SettingsPasteboard()
    open var settings = [String: Any]()
    open func current() -> [String: Any] {
    let pasteboard = NSPasteboard(name: .main)
    if let element = pasteboard.pasteboardItems?.last, let str = element.string(forType: .plain), let data = str.data(using: .utf8) {
    do {
    let json = try JSONSerialization.jsonObject(with: data, options : .allowFragments) as! [String: Any]
    settings = json
    } catch {
    print("can't convert json")
    }
    }
    return settings
    }
    open func update() {
    let jsonStr = stringify(json: settings)
    let pasteboard = NSPasteboard(name: .main)
    pasteboard.declareTypes([.string], owner: nil)
    pasteboard.setString(jsonStr, forType: .string)
    }
    ...
    }
    SettingsPasteboard.swift (ൈਮ)

    View Slide

  45. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ

    View Slide

  46. ׬੒

    View Slide

  47. ίϯτϩʔϥΞϓϦ͔Β஋Λૹ৴

    View Slide

  48. View Slide

  49. View Slide

  50. View Slide

  51. ࣮૷ํ๏ͱʮͦͷઌʯ

    View Slide

  52. 'BDFUJNFDBNFSB
    BVEJPJOQVU
    TIBEFSF⒎FDU
    "1*
    LFZCPBSEJOQVU
    ,FZOPUF
    FUD
    ֎෦ Ծ૝Χϝϥ
    ೖग़ྗՃ޻ػߏ
    ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ
    ֤छೖྗ

    View Slide

  53. ϚΠϯυϚοϓ

    View Slide

  54. δΣωϨΠςΟϒදݱ

    View Slide

  55. ελϯϓϦΞΫγϣϯ

    View Slide

  56. Ի੠ೝࣝͱ຋༁

    View Slide

  57. ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ

    View Slide

  58. ࢀߟ
    IUUQTEFWFMPQFSBQQMFDPNMJCSBSZBSDIJWFTBNQMFDPEF$PSF.FEJB*0*OUSPEVDUJPO*OUSPIUNM
    IUUQTHJUIVCDPNKPIOCPJMFTDPSFNFEJBJPEBMNJOJNBMFYBNQMF
    IUUQTHJUIVCDPNTFBODIBT4JNQMF%"-1MVHJO
    IUUQTTQFBLFSEFDLDPNLJTIJLBXBLBUTVNJWJSUVBMXFCDBNFSBXP[VPSPV

    View Slide

  59. 5XJUUFSͰ࠷৽৘ใൃ৴த
    !TINEFWFMPQ
    Ծ૝Χϝϥ࣮૷αϯϓϧ
    IUUQTHJUIVCDPNTBUPTIJ7JSUVBM$BNFSB4BNQMF

    View Slide