Virtual Web Cameraを作ろう

Virtual Web Cameraを作ろう

Virtual Web Cameraを作ろう/Let's Build Virtual Webcam for macOS
CoreMediaIO Device Abstraction Layer Plug-In

9bf923e39671cde83584e3e926296c13?s=128

Kishikawa Katsumi

April 27, 2020
Tweet

Transcript

  1. 3.
  2. 7.
  3. 8.
  4. 10.
  5. 13.
  6. 14.
  7. 23.

    CoreMediaIO DAL Plug-In #import <CoreMediaIO/CMIOHardwarePlugin.h> #import "PlugInInterface.h" #import "Logging.h" //!

    PlugInMain is the entrypoint for the plugin extern "C" { void* PlugInMain(CFAllocatorRef allocator, CFUUIDRef requestedTypeUUID) { DLogFunc(@""); if (!CFEqual(requestedTypeUUID, kCMIOHardwarePlugInTypeID)) { return 0; } return PlugInRef(); } }
  8. 24.

    CoreMediaIO DAL Plug-In #import <CoreMediaIO/CMIOHardwarePlugin.h> #import "PlugInInterface.h" #import "Logging.h" //!

    PlugInMain is the entrypoint for the plugin extern "C" { void* PlugInMain(CFAllocatorRef allocator, CFUUIDRef requestedTypeUUID) { DLogFunc(@""); if (!CFEqual(requestedTypeUUID, kCMIOHardwarePlugInTypeID)) { return 0; } return PlugInRef(); } } C++Ͱॻ͘ͳΒϚϯάϦϯά͞Εͳ͍Α͏ʹ͢Δɻ
  9. 25.

    CoreMediaIO DAL Plug-In import Foundation import CoreMediaIO @_cdecl("simpleDALPluginMain") func simpleDALPluginMain(allocator:

    CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef { NSLog("simpleDALPluginMain") return pluginRef } SwiftͰ΋ಉ͡ɻ
  10. 26.

    CoreMediaIO DAL Plug-In PlugIn *plugIn = [PlugIn SharedPlugIn]; plugIn.objectId =

    objectID; Device *device = [[Device alloc] init]; CMIOObjectID deviceId; error = CMIOObjectCreate(PlugInRef(), kCMIOObjectSystemObject, kCMIODeviceClassID, &deviceId); ... Stream *stream = [[Stream alloc] init]; CMIOObjectID streamId; error = CMIOObjectCreate(PlugInRef(), deviceId, kCMIOStreamClassID, &streamId); ... stream.objectId = streamId; [[ObjectStore SharedObjectStore] setObject:stream forObjectId:streamId]; device.streamId = streamId; // Tell the system about the Device error = CMIOObjectsPublishedAndDied(PlugInRef(), kCMIOObjectSystemObject, 1, &deviceId, 0, 0); ... // Tell the system about the Stream error = CMIOObjectsPublishedAndDied(PlugInRef(), deviceId, 1, &streamId, 0, 0); ... ΠχγϟϥΠζ͕ݺͼग़͞ΕͨΒ σόΠε΍ετϦʔϜΛొ࿥͢Δɻ ͜ΕͰΧϝϥσόΠεͱͯ͠બ୒ Ͱ͖ΔΑ͏ʹͳΔɻ
  11. 27.

    CoreMediaIO DAL Plug-In CMSampleBufferRef buffer; err = CMIOSampleBufferCreateForImageBuffer( kCFAllocatorDefault, pixelBuffer,

    format, &timing, self.sequenceNumber, kCMIOSampleBufferNoDiscontinuities, &buffer ); if (err != noErr) { DLog(@"CMIOSampleBufferCreateForImageBuffer err %d", err); } CMSimpleQueueEnqueue(self.queue, buffer); // Inform the clients that the queue has been altered if (self.alteredProc != NULL) { (self.alteredProc)(self.objectId, buffer, self.alteredRefCon); } ͋ͱ͸αϯϓϧόοϑΝΛ Ͳ͏ʹ͔ͯ͠࡞ͬͯ Ωϡʔʹ௥Ճ͍͚ͯͩ͘͠ɻ
  12. 29.

    Development Tips UnknownͷAVCaptureDeviceͱͯ͠ݟ͑Δ import Cocoa import AVFoundation class ViewController: NSViewController,

    AVCaptureVideoDataOutputSampleBufferDelegate { private let session = AVCaptureSession() override func viewDidLoad() { super.viewDidLoad() view.wantsLayer = true AVCaptureDevice.DiscoverySession(deviceTypes: [.externalUnknown], mediaType: .video, position: .unspecified).devices.forEach { do { let input = try AVCaptureDeviceInput(device: $0) // ଞʹ΋ϓϥάΠϯ͕͋Δ৔߹͸ద౰ʹௐ੔͢Δ if session.canAddInput(input) { session.addInput(input) } } catch { print(error) } } let previewLayer = AVCaptureVideoPreviewLayer() previewLayer.autoresizingMask = [.layerWidthSizable, .layerHeightSizable] previewLayer.session = session if let layer = view.layer { previewLayer.frame = layer.bounds layer.addSublayer(previewLayer) } session.startRunning() } } σόοά΍ಈ࡞֬ೝΛ؆୯ʹ͢Δʹ͸ɾɾɾ
  13. 30.

    Development Tips UnknownͷAVCaptureDeviceͱͯ͠ݟ͑Δ import Cocoa import AVFoundation class ViewController: NSViewController,

    AVCaptureVideoDataOutputSampleBufferDelegate { private let session = AVCaptureSession() override func viewDidLoad() { super.viewDidLoad() view.wantsLayer = true AVCaptureDevice.DiscoverySession(deviceTypes: [.externalUnknown], mediaType: .video, position: .unspecified).devices.forEach { do { let input = try AVCaptureDeviceInput(device: $0) // ଞʹ΋ϓϥάΠϯ͕͋Δ৔߹͸ద౰ʹௐ੔͢Δ if session.canAddInput(input) { session.addInput(input) } } catch { print(error) } } let previewLayer = AVCaptureVideoPreviewLayer() previewLayer.autoresizingMask = [.layerWidthSizable, .layerHeightSizable] previewLayer.session = session if let layer = view.layer { previewLayer.frame = layer.bounds layer.addSublayer(previewLayer) } session.startRunning() } }
  14. 31.

    Development Tips UnknownͷAVCaptureDeviceͱͯ͠ݟ͑Δ import Cocoa import AVFoundation class ViewController: NSViewController,

    AVCaptureVideoDataOutputSampleBufferDelegate { private let session = AVCaptureSession() override func viewDidLoad() { super.viewDidLoad() view.wantsLayer = true AVCaptureDevice.DiscoverySession(deviceTypes: [.externalUnknown], mediaType: .video, position: .unspecified).devices.forEach { do { let input = try AVCaptureDeviceInput(device: $0) // ଞʹ΋ϓϥάΠϯ͕͋Δ৔߹͸ద౰ʹௐ੔͢Δ if session.canAddInput(input) { session.addInput(input) } } catch { print(error) } } let previewLayer = AVCaptureVideoPreviewLayer() previewLayer.autoresizingMask = [.layerWidthSizable, .layerHeightSizable] previewLayer.session = session if let layer = view.layer { previewLayer.frame = layer.bounds layer.addSublayer(previewLayer) } session.startRunning() } } ϓϥάΠϯ΍ΤΫεςϯγϣϯͱ͍ͬͨιϑτ΢ΣΞ͸ ͱʹ͔͘σόοά͕େม͕ͩɺ͜Ε͚ͩͰಉ͡ϓϩηεͰಈ͘ͷͰ ϩάͳͲ͕શ෦ݟ͑ΔɻPreviewLayerͰಈ࡞΋֬ೝͰ͖Δɻ ͨͩผͷΞϓϦͰಈ͔͢৔߹ͱڍಈ͕ҟͳΔ͜ͱ͕͋ΔͷͰద౓ʹ֬ೝ͢Δ
  15. 32.

    Development Tips αϯϓϧόοϑΝΛCIImageʹͯ͠ҐஔΛม͑ͯ߹੒ func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,

    from connection: AVCaptureConnection) { if output == self.cameraCapture.output { guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } let cameraImage = CIImage(cvImageBuffer: imageBuffer) guard let screenImageBuffer = lastScreenImageBuffer else { return } let screenImage = CIImage(cvImageBuffer: screenImageBuffer) let translatedCameraImage = cameraImage.transformed(by: CGAffineTransform(translationX: screenImage.extent.width - cameraImage.extent.width, y: 0)) let compositedImage = translatedCameraImage.composited(over: screenImage) ... αϯϓϧόοϑΝ͸ͱʹ͔͘CIImageʹͯ͠͠·͏ͱɺૢ࡞͕؆୯ɻ ߹੒΍֦େॖখɾҠಈ΋CIImageͷϝιουΛݺͿ͚ͩɻ ͜Ε͸΢Πϯυ΢ͷΩϟϓνϟͱΧϝϥͷө૾Λ߹੒͍ͯ͠Δɻ
  16. 33.

    Development Tips αϯϓϧόοϑΝΛCIImageʹͯ͠ςΩετΛΦʔόʔϨΠ let windowImage = CGWindowListCreateImage(.null, .optionIncludingWindow, CGWindowID(windowID), [.bestResolution,

    .boundsIgnoreFraming]) { let ciImage = CIImage(cgImage: windowImage) if let text = self.settings["text"] as? String, !text.isEmpty { let bitmapImageRep = NSBitmapImageRep(ciImage: ciImage) let g = NSGraphicsContext(bitmapImageRep: bitmapImageRep) NSGraphicsContext.saveGraphicsState() NSGraphicsContext.current = g (text as NSString).draw(at: NSPoint(x: 200, y: ciImage.extent.height - 400), withAttributes: [.fon NSFont.boldSystemFont(ofSize: 400), .foregroundColor: NSColor.black]) NSGraphicsContext.restoreGraphicsState() αϯϓϧόοϑΝʹςΩετΛࡌͤΔͷ΋ CIImageʹͯ͠draw͢Ε͹OKɻ
  17. 34.

    Development Tips εΫϦʔϯશମͷΩϟϓνϟ class ScreenCapture: NSObject { private let session

    = AVCaptureSession() let output = AVCaptureVideoDataOutput() override init() { session.sessionPreset = .high if let input = AVCaptureScreenInput(displayID: CGMainDisplayID()) { if session.canAddInput(input) { session.addInput(input) if session.canAddOutput(output) { session.addOutput(output) } } } } func startRunning() { session.startRunning() } func stopRunning() { session.stopRunning() } } εΫϦʔϯશମͷΩϟϓνϟʹ͸ AVCaptureScreenInput͕༻ҙ͞Ε͍ͯΔɻ
  18. 35.

    Development Tips ΢Πϯυ΢ͷҰཡ if let windowList = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)

    { arrayController.content = (windowList as NSArray).filter{ (entry) -> Bool in if let entry = entry as? NSDictionary, let sharingState = entry[kCGWindowSharingState] as? Int, sharingState != CGWindowSharingType.none.rawValue { return true } return false } ΢Πϯυ΢͝ͱͷεΫϦʔϯγϣοτΛࡱΔʹ͸ ·ͣ΢Πϯυ΢ͷҰཡ͔ΒIDΛऔಘͯ͠ɺ
  19. 36.

    Development Tips ΢Πϯυ΢ͷΩϟϓνϟ if let windowID = self.settings["windowID"] as? Int,

    let windowImage = CGWindowListCreateImage(.null, .optionIncludingWindow, CGWindowID(windowID), [.bestResolution, .boundsIgnoreFraming]) { let ciImage = CIImage(cgImage: windowImage) ઐ༻ͷؔ਺ʹ΢Πϯυ΢IDΛ౉͢ɻ
  20. 38.

    Development Tips σʔλͷड͚౉͠ʢϓϩηεؒ௨৴ʣ • File I/O • Distributed Notification •

    XPC Service • HTTPS (Maybe) • Unix IPC (Maybe) ઃఆΛม͑ͨΓ֎͔ΒσʔλΛ౉͍ͨ͠ͱ͍͏ͱ͖ɺ ϓϩηεؒ௨৴͕࢖͑Δʢ͸ͣʣɻ ͨͩFileܦ༝΍XPC Service͸ಈ͖·ͤΜͰͨ͠ɻ ΋ͱ΋ͱͷAppleͷαϯϓϧͰ͸Mach PortΛ࢖ͬͨ ϓϩηεؒ௨৴Λ͍ͯ͠ΔͷͰɺগͳ͘ͱ΋ ͦΕ͸࢖͑Δʢ͸ͣʣ