Upgrade to Pro — share decks privately, control downloads, hide ads and more …

macOS仮想カメラ「テロップカム」 実装方法とその先

satoshi0212
September 20, 2020

macOS仮想カメラ「テロップカム」 実装方法とその先

satoshi0212

September 20, 2020
Tweet

More Decks by satoshi0212

Other Decks in Programming

Transcript

  1. NBD04Ծ૝ΧϝϥʮςϩοϓΧϜʯ
    ࣮૷ํ๏ͱͦͷઌ
    J04%$+BQBO

    ෰෦ஐ!TINEFWFMPQ

    View full-size slide

  2. Ծ૝Χϝϥ
    4OBQ$BNFSB NNINN

    View full-size slide

  3. ͜Ε͸ָ͍͠ʂ

    View full-size slide

  4. ϓϩάϥϚͳΒҰ౓͸

    View full-size slide

  5. ࣗ࡞ͨ͘͠ͳΓ·͢ΑͶʂ

    View full-size slide

  6. ྑ͍͓஌Βͤ

    View full-size slide

  7. 4XJGUͷΈͰ࣮૷Մೳʂ

    View full-size slide

  8. 5XJUUFSͰ(8όʔνϟϧΧϝϥ࡞੒νϟϨϯδͰ࡞੒աఔπΠʔτ͍ͯ͠·͢

    View full-size slide

  9. ߏ੒ཁૉ
    ɹ$PSF.FEJB*0%"-1MVHJO
    ɹίϯτϩʔϥΞϓϦ

    View full-size slide

  10. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ

    View full-size slide

  11. ϓϩδΣΫτ࡞੒ͱΠϯλʔϑΣΠε࣮૷
    $PSF.FEJB*0%"-1MVHJO

    View full-size slide

  12. ࣗ෼Ͱࢦఆ͢Δ*%஋ɻ66*%ͳͲઃఆ

    View full-size slide

  13. ΤϯτϦʔϙΠϯτGVODUJPO໊

    View full-size slide

  14. Ծ૝Χϝϥ1MVH*Oݻఆ஋
    ઌड़ͷ66*%஋

    View full-size slide

  15. import Foundation
    import CoreMediaIO
    @_cdecl("VirtualCameraSampleMain")
    func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef {
    return pluginRef
    }
    Main.swift

    View full-size slide

  16. import Foundation
    import CoreMediaIO
    @_cdecl("VirtualCameraSampleMain")
    func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef {
    return pluginRef
    }
    Main.swift
    ΤϯτϦʔϙΠϯτ
    ΠϯλʔϑΣΠεͷࢀরΛฦ٫

    View full-size slide

  17. let pluginRef: CMIOHardwarePlugInRef = {
    let interfacePtr = UnsafeMutablePointer.allocate(capacity: 1)
    interfacePtr.pointee = createPluginInterface()
    let pluginRef = CMIOHardwarePlugInRef.allocate(capacity: 1)
    pluginRef.pointee = interfacePtr
    return pluginRef
    }()
    PluginInterface.swift (ൈਮ)

    View full-size slide

  18. let pluginRef: CMIOHardwarePlugInRef = {
    let interfacePtr = UnsafeMutablePointer.allocate(capacity: 1)
    interfacePtr.pointee = createPluginInterface()
    let pluginRef = CMIOHardwarePlugInRef.allocate(capacity: 1)
    pluginRef.pointee = interfacePtr
    return pluginRef
    }()
    PluginInterface.swift (ൈਮ)
    ΠϯλʔϑΣΠεࢀরΛฦ٫

    View full-size slide

  19. private func createPluginInterface() -> CMIOHardwarePlugInInterface {
    return CMIOHardwarePlugInInterface(
    _reserved: nil,
    QueryInterface: QueryInterface,
    AddRef: AddRef,
    Release: Release,
    Initialize: Initialize,
    InitializeWithObjectID: InitializeWithObjectID,
    Teardown: Teardown,
    ObjectShow: ObjectShow,
    ObjectHasProperty: ObjectHasProperty,
    ObjectIsPropertySettable: ObjectIsPropertySettable,
    ObjectGetPropertyDataSize: ObjectGetPropertyDataSize,
    ObjectGetPropertyData: ObjectGetPropertyData,
    ObjectSetPropertyData: ObjectSetPropertyData,
    DeviceSuspend: DeviceSuspend,
    DeviceResume: DeviceResume,
    DeviceStartStream: DeviceStartStream,
    DeviceStopStream: DeviceStopStream,
    DeviceProcessAVCCommand: DeviceProcessAVCCommand,
    DeviceProcessRS422Command: DeviceProcessRS422Command,
    StreamCopyBufferQueue: StreamCopyBufferQueue,
    StreamDeckPlay: StreamDeckPlay,
    StreamDeckStop: StreamDeckStop,
    StreamDeckJog: StreamDeckJog,
    StreamDeckCueTo: StreamDeckCueTo)
    }
    PluginInterface.swift (ൈਮ)

    View full-size slide

  20. private func createPluginInterface() -> CMIOHardwarePlugInInterface {
    return CMIOHardwarePlugInInterface(
    _reserved: nil,
    QueryInterface: QueryInterface,
    AddRef: AddRef,
    Release: Release,
    Initialize: Initialize,
    InitializeWithObjectID: InitializeWithObjectID,
    Teardown: Teardown,
    ObjectShow: ObjectShow,
    ObjectHasProperty: ObjectHasProperty,
    ObjectIsPropertySettable: ObjectIsPropertySettable,
    ObjectGetPropertyDataSize: ObjectGetPropertyDataSize,
    ObjectGetPropertyData: ObjectGetPropertyData,
    ObjectSetPropertyData: ObjectSetPropertyData,
    DeviceSuspend: DeviceSuspend,
    DeviceResume: DeviceResume,
    DeviceStartStream: DeviceStartStream,
    DeviceStopStream: DeviceStopStream,
    DeviceProcessAVCCommand: DeviceProcessAVCCommand,
    DeviceProcessRS422Command: DeviceProcessRS422Command,
    StreamCopyBufferQueue: StreamCopyBufferQueue,
    StreamDeckPlay: StreamDeckPlay,
    StreamDeckStop: StreamDeckStop,
    StreamDeckJog: StreamDeckJog,
    StreamDeckCueTo: StreamDeckCueTo)
    }
    PluginInterface.swift (ൈਮ)
    ΠϯλʔϑΣΠε͕ظ଴͢Δ΋ͷΛฦ٫

    View full-size slide

  21. PluginInterface.swift (ൈਮ)
    ৄࡉ͸ιʔεࢀর

    View full-size slide

  22. Χϝϥө૾औಘͱग़ྗ

    View full-size slide

  23. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
    ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection: AVCaptureConnection) {
    if output == cameraCapture.output {
    guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
    let cameraImage = CIImage(cvImageBuffer: imageBuffer)
    let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage)
    var pixelBuffer: CVPixelBuffer?
    _ = CVPixelBufferCreate(
    kCFAllocatorDefault,
    Int(compositedImage.extent.size.width),
    Int(compositedImage.extent.height),
    kCVPixelFormatType_32BGRA,
    self.CVPixelBufferCreateOptions as CFDictionary,
    &pixelBuffer
    )
    if let pixelBuffer = pixelBuffer {
    context.render(compositedImage, to: pixelBuffer)
    delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer)
    }
    }
    }
    VideoComposer.swift (ൈਮ)

    View full-size slide

  24. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
    ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection: AVCaptureConnection) {
    if output == cameraCapture.output {
    guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
    let cameraImage = CIImage(cvImageBuffer: imageBuffer)
    let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage)
    var pixelBuffer: CVPixelBuffer?
    _ = CVPixelBufferCreate(
    kCFAllocatorDefault,
    Int(compositedImage.extent.size.width),
    Int(compositedImage.extent.height),
    kCVPixelFormatType_32BGRA,
    self.CVPixelBufferCreateOptions as CFDictionary,
    &pixelBuffer
    )
    if let pixelBuffer = pixelBuffer {
    context.render(compositedImage, to: pixelBuffer)
    delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer)
    }
    }
    }
    VideoComposer.swift (ൈਮ)
    ผ్ੜ੒ͨ͠ΦʔόʔϨΠςΩετը૾ͱ߹੒
    EFMFHBUFܦ༝ͰQJYFM#V⒎FSΛ౉͢

    View full-size slide

  25. private func enqueueBuffer() {
    (தུ)
    var sampleBufferUnmanaged: Unmanaged? = nil
    error = CMIOSampleBufferCreateForImageBuffer(
    kCFAllocatorDefault,
    pixelBuffer,
    formatDescription,
    &timing,
    sequenceNumber,
    UInt32(kCMIOSampleBufferNoDiscontinuities),
    &sampleBufferUnmanaged
    )
    guard error == noErr else {
    log("CMIOSampleBufferCreateForImageBuffer Error: \(error)")
    return
    }
    CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque())
    queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon)
    sequenceNumber += 1
    }
    Stream.swift (ൈਮ)

    View full-size slide

  26. private func enqueueBuffer() {
    (தུ)
    var sampleBufferUnmanaged: Unmanaged? = nil
    error = CMIOSampleBufferCreateForImageBuffer(
    kCFAllocatorDefault,
    pixelBuffer,
    formatDescription,
    &timing,
    sequenceNumber,
    UInt32(kCMIOSampleBufferNoDiscontinuities),
    &sampleBufferUnmanaged
    )
    guard error == noErr else {
    log("CMIOSampleBufferCreateForImageBuffer Error: \(error)")
    return
    }
    CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque())
    queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon)
    sequenceNumber += 1
    }
    Stream.swift (ൈਮ)
    QJYFM#V⒎FS͔Βੜ੒ͨ͠
    $.4BNQMF#V⒎FSΛRVFVFʹ௥Ճ

    View full-size slide

  27. QMVHJOϑΝΠϧͷ഑ஔ

    View full-size slide

  28. -JCSBSZ$PSF.FEJB*01MVH*OT%"-

    View full-size slide

  29. දࣔ͞Εͨʂ

    View full-size slide

  30. ίϯτϩʔϥΞϓϦ

    View full-size slide

  31. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ

    View full-size slide

  32. 1BTUFCPBSEʹ஋Λஔ͘໾ׂ

    View full-size slide

  33. /41BTUFCPBSEͰσʔλڞ༗

    View full-size slide

  34. extension NSPasteboard.Name {
    static let main = NSPasteboard.Name(Config.mainAppBundleIdentifier)
    }
    extension NSPasteboard.PasteboardType {
    static let plain = NSPasteboard.PasteboardType(rawValue: "public.utf8-plain-text")
    }
    class SettingsPasteboard {
    static let shared = SettingsPasteboard()
    open var settings = [String: Any]()
    open func current() -> [String: Any] {
    let pasteboard = NSPasteboard(name: .main)
    if let element = pasteboard.pasteboardItems?.last, let str = element.string(forType: .plain), let data = str.data(using: .utf8) {
    do {
    let json = try JSONSerialization.jsonObject(with: data, options : .allowFragments) as! [String: Any]
    settings = json
    } catch {
    print("can't convert json")
    }
    }
    return settings
    }
    open func update() {
    let jsonStr = stringify(json: settings)
    let pasteboard = NSPasteboard(name: .main)
    pasteboard.declareTypes([.string], owner: nil)
    pasteboard.setString(jsonStr, forType: .string)
    }
    ...
    }
    SettingsPasteboard.swift (ൈਮ)

    View full-size slide

  35. extension NSPasteboard.Name {
    static let main = NSPasteboard.Name(Config.mainAppBundleIdentifier)
    }
    extension NSPasteboard.PasteboardType {
    static let plain = NSPasteboard.PasteboardType(rawValue: "public.utf8-plain-text")
    }
    class SettingsPasteboard {
    static let shared = SettingsPasteboard()
    open var settings = [String: Any]()
    open func current() -> [String: Any] {
    let pasteboard = NSPasteboard(name: .main)
    if let element = pasteboard.pasteboardItems?.last, let str = element.string(forType: .plain), let data = str.data(using: .utf8) {
    do {
    let json = try JSONSerialization.jsonObject(with: data, options : .allowFragments) as! [String: Any]
    settings = json
    } catch {
    print("can't convert json")
    }
    }
    return settings
    }
    open func update() {
    let jsonStr = stringify(json: settings)
    let pasteboard = NSPasteboard(name: .main)
    pasteboard.declareTypes([.string], owner: nil)
    pasteboard.setString(jsonStr, forType: .string)
    }
    ...
    }
    SettingsPasteboard.swift (ൈਮ)

    View full-size slide

  36. $PSF.FEJB*0%"-
    1MVHJO
    ίϯτϩʔϥΞϓϦ
    /41BTUFCPBSE
    -JCSBSZ$PSF.FEJB*01MVH*OT%"-
    Ծ૝Χϝϥͷ࣮ମ
    6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ

    View full-size slide

  37. ίϯτϩʔϥΞϓϦ͔Β஋Λૹ৴

    View full-size slide

  38. ࣮૷ํ๏ͱʮͦͷઌʯ

    View full-size slide

  39. 'BDFUJNFDBNFSB
    BVEJPJOQVU
    TIBEFSF⒎FDU
    "1*
    LFZCPBSEJOQVU
    ,FZOPUF
    FUD
    ֎෦ Ծ૝Χϝϥ
    ೖग़ྗՃ޻ػߏ
    ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ
    ֤छೖྗ

    View full-size slide

  40. ϚΠϯυϚοϓ

    View full-size slide

  41. δΣωϨΠςΟϒදݱ

    View full-size slide

  42. ελϯϓϦΞΫγϣϯ

    View full-size slide

  43. Ի੠ೝࣝͱ຋༁

    View full-size slide

  44. ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ

    View full-size slide

  45. ࢀߟ
    IUUQTEFWFMPQFSBQQMFDPNMJCSBSZBSDIJWFTBNQMFDPEF$PSF.FEJB*0*OUSPEVDUJPO*OUSPIUNM
    IUUQTHJUIVCDPNKPIOCPJMFTDPSFNFEJBJPEBMNJOJNBMFYBNQMF
    IUUQTHJUIVCDPNTFBODIBT4JNQMF%"-1MVHJO
    IUUQTTQFBLFSEFDLDPNLJTIJLBXBLBUTVNJWJSUVBMXFCDBNFSBXP[VPSPV

    View full-size slide

  46. 5XJUUFSͰ࠷৽৘ใൃ৴த
    !TINEFWFMPQ
    Ծ૝Χϝϥ࣮૷αϯϓϧ
    IUUQTHJUIVCDPNTBUPTIJ7JSUVBM$BNFSB4BNQMF

    View full-size slide