Share
Embed
Virtual Web Cameraを作ろう
Virtual Web Cameraを作ろう/Let's Build Virtual Webcam for macOS
CoreMediaIO Device Abstraction Layer Plug-In
Kishikawa Katsumi
April 27, 2020
Transcript
38 slides
-
Sample Code • https://developer.apple.com/library/archive/samplecode/CoreMediaIO/Introduction/Intro.html • https://github.com/lvsti/CoreMediaIO-DAL-Example • https://github.com/johnboiles/obs-mac-virtualcam • https://github.com/johnboiles/coremediaio-dal-minimal-example
• https://github.com/seanchas116/SimpleDALPlugin • https://github.com/kishikawakatsumi/VirtualCameraComposer-Example -
CoreMediaIO DAL Plug-In #import <CoreMediaIO/CMIOHardwarePlugin.h> #import "PlugInInterface.h" #import "Logging.h" //!
PlugInMain is the entrypoint for the plugin extern "C" { void* PlugInMain(CFAllocatorRef allocator, CFUUIDRef requestedTypeUUID) { DLogFunc(@""); if (!CFEqual(requestedTypeUUID, kCMIOHardwarePlugInTypeID)) { return 0; } return PlugInRef(); } } -
CoreMediaIO DAL Plug-In #import <CoreMediaIO/CMIOHardwarePlugin.h> #import "PlugInInterface.h" #import "Logging.h" //!
PlugInMain is the entrypoint for the plugin extern "C" { void* PlugInMain(CFAllocatorRef allocator, CFUUIDRef requestedTypeUUID) { DLogFunc(@""); if (!CFEqual(requestedTypeUUID, kCMIOHardwarePlugInTypeID)) { return 0; } return PlugInRef(); } } C++Ͱॻ͘ͳΒϚϯάϦϯά͞Εͳ͍Α͏ʹ͢Δɻ -
CoreMediaIO DAL Plug-In import Foundation import CoreMediaIO @_cdecl("simpleDALPluginMain") func simpleDALPluginMain(allocator:
CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef { NSLog("simpleDALPluginMain") return pluginRef } SwiftͰಉ͡ɻ -
CoreMediaIO DAL Plug-In PlugIn *plugIn = [PlugIn SharedPlugIn]; plugIn.objectId =
objectID; Device *device = [[Device alloc] init]; CMIOObjectID deviceId; error = CMIOObjectCreate(PlugInRef(), kCMIOObjectSystemObject, kCMIODeviceClassID, &deviceId); ... Stream *stream = [[Stream alloc] init]; CMIOObjectID streamId; error = CMIOObjectCreate(PlugInRef(), deviceId, kCMIOStreamClassID, &streamId); ... stream.objectId = streamId; [[ObjectStore SharedObjectStore] setObject:stream forObjectId:streamId]; device.streamId = streamId; // Tell the system about the Device error = CMIOObjectsPublishedAndDied(PlugInRef(), kCMIOObjectSystemObject, 1, &deviceId, 0, 0); ... // Tell the system about the Stream error = CMIOObjectsPublishedAndDied(PlugInRef(), deviceId, 1, &streamId, 0, 0); ... ΠχγϟϥΠζ͕ݺͼग़͞ΕͨΒ σόΠεετϦʔϜΛొ͢Δɻ ͜ΕͰΧϝϥσόΠεͱͯ͠બ Ͱ͖ΔΑ͏ʹͳΔɻ -
CoreMediaIO DAL Plug-In CMSampleBufferRef buffer; err = CMIOSampleBufferCreateForImageBuffer( kCFAllocatorDefault, pixelBuffer,
format, &timing, self.sequenceNumber, kCMIOSampleBufferNoDiscontinuities, &buffer ); if (err != noErr) { DLog(@"CMIOSampleBufferCreateForImageBuffer err %d", err); } CMSimpleQueueEnqueue(self.queue, buffer); // Inform the clients that the queue has been altered if (self.alteredProc != NULL) { (self.alteredProc)(self.objectId, buffer, self.alteredRefCon); } ͋ͱαϯϓϧόοϑΝΛ Ͳ͏ʹ͔ͯ͠࡞ͬͯ ΩϡʔʹՃ͍͚ͯͩ͘͠ɻ -
Development Tips UnknownͷAVCaptureDeviceͱͯ͠ݟ͑Δ import Cocoa import AVFoundation class ViewController: NSViewController,
AVCaptureVideoDataOutputSampleBufferDelegate { private let session = AVCaptureSession() override func viewDidLoad() { super.viewDidLoad() view.wantsLayer = true AVCaptureDevice.DiscoverySession(deviceTypes: [.externalUnknown], mediaType: .video, position: .unspecified).devices.forEach { do { let input = try AVCaptureDeviceInput(device: $0) // ଞʹϓϥάΠϯ͕͋Δ߹దʹௐ͢Δ if session.canAddInput(input) { session.addInput(input) } } catch { print(error) } } let previewLayer = AVCaptureVideoPreviewLayer() previewLayer.autoresizingMask = [.layerWidthSizable, .layerHeightSizable] previewLayer.session = session if let layer = view.layer { previewLayer.frame = layer.bounds layer.addSublayer(previewLayer) } session.startRunning() } } σόοάಈ࡞֬ೝΛ؆୯ʹ͢Δʹɾɾɾ -
Development Tips UnknownͷAVCaptureDeviceͱͯ͠ݟ͑Δ import Cocoa import AVFoundation class ViewController: NSViewController,
AVCaptureVideoDataOutputSampleBufferDelegate { private let session = AVCaptureSession() override func viewDidLoad() { super.viewDidLoad() view.wantsLayer = true AVCaptureDevice.DiscoverySession(deviceTypes: [.externalUnknown], mediaType: .video, position: .unspecified).devices.forEach { do { let input = try AVCaptureDeviceInput(device: $0) // ଞʹϓϥάΠϯ͕͋Δ߹దʹௐ͢Δ if session.canAddInput(input) { session.addInput(input) } } catch { print(error) } } let previewLayer = AVCaptureVideoPreviewLayer() previewLayer.autoresizingMask = [.layerWidthSizable, .layerHeightSizable] previewLayer.session = session if let layer = view.layer { previewLayer.frame = layer.bounds layer.addSublayer(previewLayer) } session.startRunning() } } -
Development Tips UnknownͷAVCaptureDeviceͱͯ͠ݟ͑Δ import Cocoa import AVFoundation class ViewController: NSViewController,
AVCaptureVideoDataOutputSampleBufferDelegate { private let session = AVCaptureSession() override func viewDidLoad() { super.viewDidLoad() view.wantsLayer = true AVCaptureDevice.DiscoverySession(deviceTypes: [.externalUnknown], mediaType: .video, position: .unspecified).devices.forEach { do { let input = try AVCaptureDeviceInput(device: $0) // ଞʹϓϥάΠϯ͕͋Δ߹దʹௐ͢Δ if session.canAddInput(input) { session.addInput(input) } } catch { print(error) } } let previewLayer = AVCaptureVideoPreviewLayer() previewLayer.autoresizingMask = [.layerWidthSizable, .layerHeightSizable] previewLayer.session = session if let layer = view.layer { previewLayer.frame = layer.bounds layer.addSublayer(previewLayer) } session.startRunning() } } ϓϥάΠϯΤΫεςϯγϣϯͱ͍ͬͨιϑτΣΞ ͱʹ͔͘σόοά͕େม͕ͩɺ͜Ε͚ͩͰಉ͡ϓϩηεͰಈ͘ͷͰ ϩάͳͲ͕શ෦ݟ͑ΔɻPreviewLayerͰಈ࡞֬ೝͰ͖Δɻ ͨͩผͷΞϓϦͰಈ͔͢߹ͱڍಈ͕ҟͳΔ͜ͱ͕͋ΔͷͰదʹ֬ೝ͢Δ -
Development Tips αϯϓϧόοϑΝΛCIImageʹͯ͠ҐஔΛม͑ͯ߹ func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
from connection: AVCaptureConnection) { if output == self.cameraCapture.output { guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } let cameraImage = CIImage(cvImageBuffer: imageBuffer) guard let screenImageBuffer = lastScreenImageBuffer else { return } let screenImage = CIImage(cvImageBuffer: screenImageBuffer) let translatedCameraImage = cameraImage.transformed(by: CGAffineTransform(translationX: screenImage.extent.width - cameraImage.extent.width, y: 0)) let compositedImage = translatedCameraImage.composited(over: screenImage) ... αϯϓϧόοϑΝͱʹ͔͘CIImageʹͯ͠͠·͏ͱɺૢ࡞͕؆୯ɻ ߹֦େॖখɾҠಈCIImageͷϝιουΛݺͿ͚ͩɻ ͜ΕΠϯυͷΩϟϓνϟͱΧϝϥͷө૾Λ߹͍ͯ͠Δɻ -
Development Tips αϯϓϧόοϑΝΛCIImageʹͯ͠ςΩετΛΦʔόʔϨΠ let windowImage = CGWindowListCreateImage(.null, .optionIncludingWindow, CGWindowID(windowID), [.bestResolution,
.boundsIgnoreFraming]) { let ciImage = CIImage(cgImage: windowImage) if let text = self.settings["text"] as? String, !text.isEmpty { let bitmapImageRep = NSBitmapImageRep(ciImage: ciImage) let g = NSGraphicsContext(bitmapImageRep: bitmapImageRep) NSGraphicsContext.saveGraphicsState() NSGraphicsContext.current = g (text as NSString).draw(at: NSPoint(x: 200, y: ciImage.extent.height - 400), withAttributes: [.fon NSFont.boldSystemFont(ofSize: 400), .foregroundColor: NSColor.black]) NSGraphicsContext.restoreGraphicsState() αϯϓϧόοϑΝʹςΩετΛࡌͤΔͷ CIImageʹͯ͠draw͢ΕOKɻ -
Development Tips εΫϦʔϯશମͷΩϟϓνϟ class ScreenCapture: NSObject { private let session
= AVCaptureSession() let output = AVCaptureVideoDataOutput() override init() { session.sessionPreset = .high if let input = AVCaptureScreenInput(displayID: CGMainDisplayID()) { if session.canAddInput(input) { session.addInput(input) if session.canAddOutput(output) { session.addOutput(output) } } } } func startRunning() { session.startRunning() } func stopRunning() { session.stopRunning() } } εΫϦʔϯશମͷΩϟϓνϟʹ AVCaptureScreenInput͕༻ҙ͞Ε͍ͯΔɻ -
Development Tips ΠϯυͷҰཡ if let windowList = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)
{ arrayController.content = (windowList as NSArray).filter{ (entry) -> Bool in if let entry = entry as? NSDictionary, let sharingState = entry[kCGWindowSharingState] as? Int, sharingState != CGWindowSharingType.none.rawValue { return true } return false } Πϯυ͝ͱͷεΫϦʔϯγϣοτΛࡱΔʹ ·ͣΠϯυͷҰཡ͔ΒIDΛऔಘͯ͠ɺ -
Development Tips ΠϯυͷΩϟϓνϟ if let windowID = self.settings["windowID"] as? Int,
let windowImage = CGWindowListCreateImage(.null, .optionIncludingWindow, CGWindowID(windowID), [.bestResolution, .boundsIgnoreFraming]) { let ciImage = CIImage(cgImage: windowImage) ઐ༻ͷؔʹΠϯυIDΛ͢ɻ -
Development Tips σʔλͷड͚͠ʢϓϩηεؒ௨৴ʣ • File I/O • Distributed Notification •
XPC Service • HTTPS (Maybe) • Unix IPC (Maybe) ઃఆΛม͑ͨΓ֎͔ΒσʔλΛ͍ͨ͠ͱ͍͏ͱ͖ɺ ϓϩηεؒ௨৴͕͑Δʢͣʣɻ ͨͩFileܦ༝XPC Serviceಈ͖·ͤΜͰͨ͠ɻ ͱͱͷAppleͷαϯϓϧͰMach PortΛͬͨ ϓϩηεؒ௨৴Λ͍ͯ͠ΔͷͰɺগͳ͘ͱ ͦΕ͑Δʢͣʣ