クロマキー合成を使い透過動画をAR空間に表示する

1. ΫϩϚΩʔ߹੒Λ࢖͍
   ಁաಈըΛ"3ۭؒʹදࣔ͢Δ TBUPTIJ
   
   ෰෦
   ஐ
   "CFNB57
   (JU)VC
   TBUPTIJ
   
   5XJUUFS
   !TINEFWFMPQ
   
   J04%$
    NJO
   4FTTJPO !1
2. ෰෦
   ஐ ೥݄ʹ"CFNB57ೖࣾɻ
   J04νʔϜͰΞϓϦ։ൃɻଟ਺ͷػೳΛ ΰϦΰϦ࣮૷ɻ
   ೥݄ΑΓ/FX%FWJDFνʔϜʹ ͯεϚʔτεϐʔΧʔ΍57σόΠεͳ Ͳ৽ن։୓Λத৺ͱͨ͠ௐࠪɾ։ൃΛ ୲౰ɻ
   αΠόʔΤʔδΣϯτࣾ಺ΤϯδχΞά ϧʔϓʮ93ΪϧυʯϦʔμʔɻ !2
3. ૝ఆಡऀ 
   "3,JU
   αϯϓϧಈ͔ͨ͜͠ͱ͋Δ͘Β͍
   
   .FUBM
   গ͠͸෼͔Δ
   
   .FUBM
   4IBEFS
   ࢖ͬͨ͜ͱ͕ͳ͍
   
   6OJUZͰ͸ͳ͘ωΠςΟϒͰॻ͖͍ͨ !3
4. ૝ఆಡऀ .FUBMͷֶशʹ͸ͪ͜Β͕͓͢͢Ί !4 IUUQTCPPUIQNKBJUFNT TIV
   ͞Μ
   .FUBMೖ໳
5. ࠓ೔ͷτʔΫͰ఻͍͑ͨ͜ͱ !5
6. ࠓ೔ͷτʔΫͰ఻͍͑ͨ͜ͱ 
   "3,JUΛ࢖͍ͦͷதͰө૾Λ࠶ੜ͢Δํ๏
   
   ͜ΕͰ৭ʑͰ͖ͦ͏ʂͱ͍͏ظ଴ײ !6 ൒೥લͷࣗ෼ʹݟͤͨΒϚδͰتͿ಺༰ʂ
   
   ಈ͘αϯϓϧίʔυ΋͋ͬͯ͋Γ͕͍ͨͥʂ
7. ΫϩϚΩʔ߹੒ !7
8. ΫϩϚΩʔ߹੒ ग़య
   ϑϦʔඦՊࣄయʰ΢ΟΩϖσΟΞʢ8JLJQFEJBʣʱ
   IUUQTKBXJLJQFEJBPSHXJLJ&"'&"%&&&"%&#$ ΫϩϚΩʔʢ$ISPNB
   LFZʣ΋͘͠͸ΫϩϚ Ωʔ߹੒ʢΫϩϚΩʔ͝͏͍ͤʣ͸ΩʔΠϯά ͷҰछͰɺಛఆͷ৭ͷ੒෼͔Βө૾ͷҰ෦Λಁ ໌ʹ͠ɺͦ͜ʹผͷө૾Λ߹੒͢Δٕज़ɻ !8
9. ग़య
   IUUQTEFWFMPQFSBQQMFDPNEPDVNFOUBUJPODPSFJNBHFBQQMZJOH@B@DISPNB@LFZ@F⒎FDU !9
10. ࠓճ࡞Δͷ͸ͪ͜Β !10
11. !11 ग़య
    .JLB3JLB
    0⒏DJBM
    άϦʔϯόοΫΞΠυϧɹ.JLB 3JLB
    
    IUUQTXXXZPVUVCFDPNXBUDI WX0OTBMKG"P
12. !12
13. ໨࣍ !13
14. ໨࣍ "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ
    ಁաಈըΛ࠶ੜ͢Δ
    
    
    
    
    
    ॲཧͷྲྀΕ
    
    
    
    
    
    ΫϩϚΩʔॲཧ.FUBMγΣʔμʔղઆ
    ͓·͚
    6*7JFXΛಁաදࣔͯ͠ΈΔ
    %FNP !14
15. ໨࣍ 
    "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ
    ಁաಈըΛ࠶ੜ͢Δ
    
    
    
    
    
    ॲཧͷྲྀΕ
    
    
    
    
    
    ΫϩϚΩʔॲཧ.FUBMγΣʔμʔղઆ
    ͓·͚
    6*7JFXΛಁաදࣔͯ͠ΈΔ
    %FNP !15
16. "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ !16
17. !17 4DFOF,JU 4QSJUF,JU .FUBM $POUFOU
    5FDIOPMPHZ
18. !18 4DFOF,JU 4QSJUF,JU .FUBM $POUFOU
    5FDIOPMPHZ ˞
    ࠓճ͸"34$/7JFXΛ࢖༻ɻ
    
    
    
    
    4$/3FOEFSFSͰඳըͨ͠Γ.FUBMͷΈͰඳը͢Δ͜ͱ΋Մೳɻ
19. "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ !19
20. "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ 4QSJUF,JUͷ4,7JEFP/PEF͸ಈըΛ࠶ੜͰ͖Δ !20
21. "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ 4QSJUF,JUͷ4,7JEFP/PEF͸ಈըΛ࠶ੜͰ͖Δ !21 4DFOF,JUͷ4$//PEF͸4,7JEFP/PEFΛදࣔཁૉʹͰ͖Δ
22. "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ 4QSJUF,JUͷ4,7JEFP/PEF͸ಈըΛ࠶ੜͰ͖Δ !22 4$//PEF͸"3ۭؒ಺ʹ഑ஔͰ͖Δ 4DFOF,JUͷ4$//PEF͸4,7JEFP/PEFΛදࣔཁૉʹͰ͖Δ
23. !23 4$/4DFOF 4$//PEF 4$/1MBOF 4,4DFOF 4$/.BUFSJBM 4,7JEFP/PEF "71MBZFS
24. func createVideoNode(size: CGFloat, videoUrl: URL) -> SCNNode { // αΠζ͕খ͍͞ͱϏσΦͷղ૾౓͕མͪΔ let skSceneSize = CGSize(width: 1024, height: 1024) // AVPlayerੜ੒ let avPlayer = AVPlayer(url: videoUrl) // SKVideoNodeੜ੒ let skVideoNode = SKVideoNode(avPlayer: avPlayer) skVideoNode.position = CGPoint(x: skSceneSize.width / 2.0, y: skSceneSize.height / 2.0) skVideoNode.size = skSceneSize skVideoNode.yScale = -1.0 // ࠲ඪܥΛ্Լٯʹ͢Δ skVideoNode.play() // SKSceneੜ੒ let skScene = SKScene(size: skSceneSize) skScene.addChild(skVideoNode) // SCNMaterialੜ੒ let material = SCNMaterial() material.diffuse.contents = skScene material.isDoubleSided = true // SCNNodeੜ੒ let node = SCNNode() // SCNPlane(=SCNGeometryΛܧঝͨ͠Ϋϥε)ੜ੒ node.geometry = SCNPlane(width: size, height: size) node.geometry?.materials = [material] node.scale = SCNVector3(1, 0.5625, 1) return node } !24 "71MBZFS
    
    4,7JEFP/PEF
    
    4,4DFOF
    
    4$/.BUFSJBM
    
    4$/(FPNFUSZ
    
    4$//PEF
25. func createVideoNode(size: CGFloat, videoUrl: URL) -> SCNNode { // αΠζ͕খ͍͞ͱϏσΦͷղ૾౓͕མͪΔ let skSceneSize = CGSize(width: 1024, height: 1024) // AVPlayerੜ੒ let avPlayer = AVPlayer(url: videoUrl) // SKVideoNodeੜ੒ let skVideoNode = SKVideoNode(avPlayer: avPlayer) skVideoNode.position = CGPoint(x: skSceneSize.width / 2.0, y: skSceneSize.height / 2.0) skVideoNode.size = skSceneSize skVideoNode.yScale = -1.0 // ࠲ඪܥΛ্Լٯʹ͢Δ skVideoNode.play() // SKSceneੜ੒ let skScene = SKScene(size: skSceneSize) skScene.addChild(skVideoNode) // SCNMaterialੜ੒ let material = SCNMaterial() material.diffuse.contents = skScene material.isDoubleSided = true // SCNNodeੜ੒ let node = SCNNode() // SCNPlane(=SCNGeometryΛܧঝͨ͠Ϋϥε)ੜ੒ node.geometry = SCNPlane(width: size, height: size) node.geometry?.materials = [material] node.scale = SCNVector3(1, 0.5625, 1) return node } !25 "71MBZFS
    
    4,7JEFP/PEF
    
    4,4DFOF
    
    4$/.BUFSJBM
    
    4$/(FPNFUSZ
    
    4$//PEF
26. func createVideoNode(size: CGFloat, videoUrl: URL) -> SCNNode { // αΠζ͕খ͍͞ͱϏσΦͷղ૾౓͕མͪΔ let skSceneSize = CGSize(width: 1024, height: 1024) // AVPlayerੜ੒ let avPlayer = AVPlayer(url: videoUrl) // SKVideoNodeੜ੒ let skVideoNode = SKVideoNode(avPlayer: avPlayer) skVideoNode.position = CGPoint(x: skSceneSize.width / 2.0, y: skSceneSize.height / 2.0) skVideoNode.size = skSceneSize skVideoNode.yScale = -1.0 // ࠲ඪܥΛ্Լٯʹ͢Δ skVideoNode.play() // SKSceneੜ੒ let skScene = SKScene(size: skSceneSize) skScene.addChild(skVideoNode) // SCNMaterialੜ੒ let material = SCNMaterial() material.diffuse.contents = skScene material.isDoubleSided = true // SCNNodeੜ੒ let node = SCNNode() // SCNPlane(=SCNGeometryΛܧঝͨ͠Ϋϥε)ੜ੒ node.geometry = SCNPlane(width: size, height: size) node.geometry?.materials = [material] node.scale = SCNVector3(1, 0.5625, 1) return node } !26 "71MBZFS
    
    4,7JEFP/PEF
    
    4,4DFOF
    
    4$/.BUFSJBM
    
    4$/(FPNFUSZ
    
    4$//PEF
27. func createVideoNode(size: CGFloat, videoUrl: URL) -> SCNNode { // αΠζ͕খ͍͞ͱϏσΦͷղ૾౓͕མͪΔ let skSceneSize = CGSize(width: 1024, height: 1024) // AVPlayerੜ੒ let avPlayer = AVPlayer(url: videoUrl) // SKVideoNodeੜ੒ let skVideoNode = SKVideoNode(avPlayer: avPlayer) skVideoNode.position = CGPoint(x: skSceneSize.width / 2.0, y: skSceneSize.height / 2.0) skVideoNode.size = skSceneSize skVideoNode.yScale = -1.0 // ࠲ඪܥΛ্Լٯʹ͢Δ skVideoNode.play() // SKSceneੜ੒ let skScene = SKScene(size: skSceneSize) skScene.addChild(skVideoNode) // SCNMaterialੜ੒ let material = SCNMaterial() material.diffuse.contents = skScene material.isDoubleSided = true // SCNNodeੜ੒ let node = SCNNode() // SCNPlane(=SCNGeometryΛܧঝͨ͠Ϋϥε)ੜ੒ node.geometry = SCNPlane(width: size, height: size) node.geometry?.materials = [material] node.scale = SCNVector3(1, 0.5625, 1) return node } !27 "71MBZFS
    
    4,7JEFP/PEF
    
    4,4DFOF
    
    4$/.BUFSJBM
    
    4$/(FPNFUSZ
    
    4$//PEF
28. func createVideoNode(size: CGFloat, videoUrl: URL) -> SCNNode { // αΠζ͕খ͍͞ͱϏσΦͷղ૾౓͕མͪΔ let skSceneSize = CGSize(width: 1024, height: 1024) // AVPlayerੜ੒ let avPlayer = AVPlayer(url: videoUrl) // SKVideoNodeੜ੒ let skVideoNode = SKVideoNode(avPlayer: avPlayer) skVideoNode.position = CGPoint(x: skSceneSize.width / 2.0, y: skSceneSize.height / 2.0) skVideoNode.size = skSceneSize skVideoNode.yScale = -1.0 // ࠲ඪܥΛ্Լٯʹ͢Δ skVideoNode.play() // SKSceneੜ੒ let skScene = SKScene(size: skSceneSize) skScene.addChild(skVideoNode) // SCNMaterialੜ੒ let material = SCNMaterial() material.diffuse.contents = skScene material.isDoubleSided = true // SCNNodeੜ੒ let node = SCNNode() // SCNPlane(=SCNGeometryΛܧঝͨ͠Ϋϥε)ੜ੒ node.geometry = SCNPlane(width: size, height: size) node.geometry?.materials = [material] node.scale = SCNVector3(1, 0.5625, 1) return node } !28 "71MBZFS
    
    4,7JEFP/PEF
    
    4,4DFOF
    
    4$/.BUFSJBM
    
    4$/(FPNFUSZ
    
    4$//PEF
29. func createVideoNode(size: CGFloat, videoUrl: URL) -> SCNNode { // αΠζ͕খ͍͞ͱϏσΦͷղ૾౓͕མͪΔ let skSceneSize = CGSize(width: 1024, height: 1024) // AVPlayerੜ੒ let avPlayer = AVPlayer(url: videoUrl) // SKVideoNodeੜ੒ let skVideoNode = SKVideoNode(avPlayer: avPlayer) skVideoNode.position = CGPoint(x: skSceneSize.width / 2.0, y: skSceneSize.height / 2.0) skVideoNode.size = skSceneSize skVideoNode.yScale = -1.0 // ࠲ඪܥΛ্Լٯʹ͢Δ skVideoNode.play() // SKSceneੜ੒ let skScene = SKScene(size: skSceneSize) skScene.addChild(skVideoNode) // SCNMaterialੜ੒ let material = SCNMaterial() material.diffuse.contents = skScene material.isDoubleSided = true // SCNNodeੜ੒ let node = SCNNode() // SCNPlane(=SCNGeometryΛܧঝͨ͠Ϋϥε)ੜ੒ node.geometry = SCNPlane(width: size, height: size) node.geometry?.materials = [material] node.scale = SCNVector3(1, 0.5625, 1) return node } !29 "71MBZFS
    
    4,7JEFP/PEF
    
    4,4DFOF
    
    4$/.BUFSJBM
    
    4$/(FPNFUSZ
    
    4$//PEF
30. override func viewDidLoad() { super.viewDidLoad() sceneView.delegate = self sceneView.scene = SCNScene() let videoUrl = Bundle.main.url(forResource: "video001", withExtension: "mp4")! let videoNode = createVideoNode(size: 1, videoUrl: videoUrl) videoNode.position = SCNVector3(0, 0, -3.0) sceneView.scene.rootNode.addChildNode(videoNode) } !30
31. !31
32. !32
33. ໨࣍ 
    "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ
    ಁաಈըΛ࠶ੜ͢Δ
    
    
    
    
    
    ॲཧͷྲྀΕ
    
    
    
    
    
    ΫϩϚΩʔॲཧ.FUBMγΣʔμʔղઆ
    ͓·͚
    6*7JFXΛಁաදࣔͯ͠ΈΔ
    %FNP !33
34. ໨࣍ "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ
    
    ಁաಈըΛ࠶ੜ͢Δ
    
    
    
    
    
    ॲཧͷྲྀΕ
    
    
    
    
    
    ΫϩϚΩʔॲཧ.FUBMγΣʔμʔղઆ
    ͓·͚
    6*7JFXΛಁաදࣔͯ͠ΈΔ
    %FNP !34
35. ॲཧͷྲྀΕ !35
36. !36 4$/4DFOF 4$//PEF 4$/1MBOF 4,4DFOF 4$/.BUFSJBM 4,7JEFP/PEF "71MBZFS
37. !37 4$/4DFOF 4$//PEF 4$/1MBOF 7JEFP.FUBM7JFX
38. !38 4$/4DFOF 4$//PEF 4$/1MBOF 7JEFP.FUBM7JFX .5,7JFXΛܧঝͨ͠ΧελϜΫϥε
39. !39 7JEFP.FUBM7JFX
    .5,7JFX "71MBZFS $*$POUFYU .5,7JFX GVOD
    ESBX ݱࡏͷಈըσʔλΛը૾Խ Ұ࣌.5,7JFXʹϨϯμϦϯά ΫϩϚΩʔγΣʔμʔॲཧ
40. override func draw(_ dirtyRect: CGRect) { guard let device = device, let drawable = currentDrawable, let tempDrawable = bufferMtkView.currentDrawable, let image = makeCurrentVideoImage() else { return } ciContext.render(image, to: tempDrawable.texture, commandBuffer: nil, bounds: bounds, colorSpace: colorSpace) colorPixelFormat = tempDrawable.texture.pixelFormat let commandBuffer = commandQueue.makeCommandBuffer()! let commandEncoder = commandBuffer.makeComputeCommandEncoder()! commandEncoder.setComputePipelineState(pipelineState) commandEncoder.setTexture(tempDrawable.texture, index: 0) commandEncoder.setTexture(drawable.texture, index: 1) let factors: [Float] = [ 0, // red 1, // green 0, // blue 0.43, // threshold 0.11 // smoothing ] for i in 0..<factors.count { var factor = factors[i] let size = max(MemoryLayout<Float>.size, 16) let buffer = device.makeBuffer( bytes: &factor, length: size, options: [.storageModeShared] ) commandEncoder.setBuffer(buffer, offset: 0, index: i) } commandEncoder.dispatchThreadgroups(threadgroupsPerGrid, threadsPerThreadgroup: threadsPerThreadgroup) commandEncoder.endEncoding() commandBuffer.present(drawable) commandBuffer.commit() } !40
41. !41 7JEFP.FUBM7JFX
    .5,7JFX "71MBZFS $*$POUFYU .5,7JFX GVOD
    ESBX ݱࡏͷಈըσʔλΛը૾Խ Ұ࣌.5,7JFXʹϨϯμϦϯά ΫϩϚΩʔγΣʔμʔॲཧ
42. override func draw(_ dirtyRect: CGRect) { guard let device = device, let drawable = currentDrawable, let tempDrawable = bufferMtkView.currentDrawable, let image = makeCurrentVideoImage() else { return } ciContext.render(image, to: tempDrawable.texture, commandBuffer: nil, bounds: bounds, colorSpace: colorSpace) colorPixelFormat = tempDrawable.texture.pixelFormat let commandBuffer = commandQueue.makeCommandBuffer()! let commandEncoder = commandBuffer.makeComputeCommandEncoder()! commandEncoder.setComputePipelineState(pipelineState) commandEncoder.setTexture(tempDrawable.texture, index: 0) commandEncoder.setTexture(drawable.texture, index: 1) let factors: [Float] = [ 0, // red 1, // green 0, // blue 0.43, // threshold 0.11 // smoothing ] for i in 0..<factors.count { var factor = factors[i] let size = max(MemoryLayout<Float>.size, 16) let buffer = device.makeBuffer( bytes: &factor, length: size, options: [.storageModeShared] ) commandEncoder.setBuffer(buffer, offset: 0, index: i) } commandEncoder.dispatchThreadgroups(threadgroupsPerGrid, threadsPerThreadgroup: threadsPerThreadgroup) commandEncoder.endEncoding() commandBuffer.present(drawable) commandBuffer.commit() } !42
43. private let videoOutput = AVPlayerItemVideoOutput.init(pixelBufferAttributes: [ kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, ] as [String: Any]) func setupPlayer(url: URL) { player = AVPlayer(url: url) guard let player = player, let videoItem = player.currentItem else { return } videoItem.add(videoOutput) } private func makeCurrentVideoImage() -> CIImage? { guard let player = player, let videoItem = player.currentItem else { return nil } let time = videoItem.currentTime() guard videoOutput.hasNewPixelBuffer(forItemTime: time), let pixelBuffer = videoOutput.copyPixelBuffer(forItemTime: time, itemTimeForDisplay: nil) else { return nil } return CIImage(cvPixelBuffer: pixelBuffer) } !43
44. private let videoOutput = AVPlayerItemVideoOutput.init(pixelBufferAttributes: [ kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, ] as [String: Any]) func setupPlayer(url: URL) { player = AVPlayer(url: url) guard let player = player, let videoItem = player.currentItem else { return } videoItem.add(videoOutput) } private func makeCurrentVideoImage() -> CIImage? { guard let player = player, let videoItem = player.currentItem else { return nil } let time = videoItem.currentTime() guard videoOutput.hasNewPixelBuffer(forItemTime: time), let pixelBuffer = videoOutput.copyPixelBuffer(forItemTime: time, itemTimeForDisplay: nil) else { return nil } return CIImage(cvPixelBuffer: pixelBuffer) } !44
45. !45 7JEFP.FUBM7JFX
    .5,7JFX "71MBZFS $*$POUFYU .5,7JFX GVOD
    ESBX ݱࡏͷಈըσʔλΛը૾Խ Ұ࣌.5,7JFXʹϨϯμϦϯά ΫϩϚΩʔγΣʔμʔॲཧ
46. override func draw(_ dirtyRect: CGRect) { guard let device = device, let drawable = currentDrawable, let tempDrawable = bufferMtkView.currentDrawable, let image = makeCurrentVideoImage() else { return } ciContext.render(image, to: tempDrawable.texture, commandBuffer: nil, bounds: bounds, colorSpace: colorSpace) colorPixelFormat = tempDrawable.texture.pixelFormat let commandBuffer = commandQueue.makeCommandBuffer()! let commandEncoder = commandBuffer.makeComputeCommandEncoder()! commandEncoder.setComputePipelineState(pipelineState) commandEncoder.setTexture(tempDrawable.texture, index: 0) commandEncoder.setTexture(drawable.texture, index: 1) let factors: [Float] = [ 0, // red 1, // green 0, // blue 0.43, // threshold 0.11 // smoothing ] for i in 0..<factors.count { var factor = factors[i] let size = max(MemoryLayout<Float>.size, 16) let buffer = device.makeBuffer( bytes: &factor, length: size, options: [.storageModeShared] ) commandEncoder.setBuffer(buffer, offset: 0, index: i) } commandEncoder.dispatchThreadgroups(threadgroupsPerGrid, threadsPerThreadgroup: threadsPerThreadgroup) commandEncoder.endEncoding() commandBuffer.present(drawable) commandBuffer.commit() } !46
47. !47 7JEFP.FUBM7JFX
    .5,7JFX "71MBZFS $*$POUFYU .5,7JFX GVOD
    ESBX ݱࡏͷಈըσʔλΛը૾Խ Ұ࣌.5,7JFXʹϨϯμϦϯά ΫϩϚΩʔγΣʔμʔॲཧ
48. override func draw(_ dirtyRect: CGRect) { guard let device = device, let drawable = currentDrawable, let tempDrawable = bufferMtkView.currentDrawable, let image = makeCurrentVideoImage() else { return } ciContext.render(image, to: tempDrawable.texture, commandBuffer: nil, bounds: bounds, colorSpace: colorSpace) colorPixelFormat = tempDrawable.texture.pixelFormat let commandBuffer = commandQueue.makeCommandBuffer()! let commandEncoder = commandBuffer.makeComputeCommandEncoder()! commandEncoder.setComputePipelineState(pipelineState) commandEncoder.setTexture(tempDrawable.texture, index: 0) commandEncoder.setTexture(drawable.texture, index: 1) let factors: [Float] = [ 0, // red 1, // green 0, // blue 0.43, // threshold 0.11 // smoothing ] for i in 0..<factors.count { var factor = factors[i] let size = max(MemoryLayout<Float>.size, 16) let buffer = device.makeBuffer( bytes: &factor, length: size, options: [.storageModeShared] ) commandEncoder.setBuffer(buffer, offset: 0, index: i) } commandEncoder.dispatchThreadgroups(threadgroupsPerGrid, threadsPerThreadgroup: threadsPerThreadgroup) commandEncoder.endEncoding() commandBuffer.present(drawable) commandBuffer.commit() } !48
49. ໨࣍ "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ
    
    ಁաಈըΛ࠶ੜ͢Δ
    
    
    
    
    
    ॲཧͷྲྀΕ
    
    
    
    
    
    ΫϩϚΩʔॲཧ.FUBMγΣʔμʔղઆ
    ͓·͚
    6*7JFXΛಁաදࣔͯ͠ΈΔ
    %FNP !49
50. ໨࣍ "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ
    
    ಁաಈըΛ࠶ੜ͢Δ
    
    
    
    
    
    ॲཧͷྲྀΕ
    
    
    
    
    
    ΫϩϚΩʔॲཧ.FUBMγΣʔμʔղઆ
    ͓·͚
    6*7JFXΛಁաදࣔͯ͠ΈΔ
    %FNP !50
51. .FUBM
    γΣʔμʔ !51
52. .FUBM
    γΣʔμʔ !52 छྨ ֓ཁ 7FSUFY ֤௖఺ͷ࠲ඪ৘ใΛฦ٫͢Δ'VODUJPO 'SBHNFOU ֤௖఺ͷ৭৘ใΛฦ٫͢Δ'VODUJPO $PNQVUF ,FSOFM େن໛ͳσʔλΛฒྻॲཧ͢Δࡍͷ൚༻తͳΧελϜ 'VODUJPO
53. γΣʔμʔͰͷΫϩϚΩʔ߹੒ॲཧ ,FSOFMͷγΣʔμʔͰ
    
    NBTL$PMPS ྘৭ Λ:$S$C΁ม׵
    
    ࠲ඪͷ3(#Λ:$S$C΁ม׵
    
    :$S$Cɺᮢ஋ɺ྘৭ͱͷڑ཭ʹΑΓಁա౓Λࢉग़
    
    ࠲ඪͷ3(#Λࢉग़ͨ͠ಁա౓Ͱग़ྗ͢Δ৭Λܭࢉ !53
54. #include <metal_stdlib> using namespace metal; /// - SeeAlso: http://www.fundza.com/rman_shaders/smoothstep/index.html kernel void ChromaKeyFilter( // ೖྗը૾৘ใ texture2d<float, access::read> inTexture [[ texture(0) ]], // ग़ྗը૾৘ใ texture2d<float, access::write> outTexture [[ texture(1) ]], // ϚεΫର৅ͷ੺੒෼ const device float *colorRed [[ buffer(0) ]], // ϚεΫର৅ͷ྘੒෼ const device float *colorGreen [[ buffer(1) ]], // ϚεΫର৅ͷ੨੒෼ const device float *colorBlue [[ buffer(2) ]], // ಁա஋͕0-1ͷྖҬͷԼݶ const device float *threshold [[ buffer(3) ]], // ಁա஋͕0-1ͷྖҬͷԼݶ͔Β্ݶ·Ͱͷઈର஋ const device float *smoothing [[ buffer(4) ]], // ࠲ඪ৘ใ uint2 gid [[ thread_position_in_grid ]]) { const float4 inColor = inTexture.read(gid); // maskͷର৅ͷ৭ɻ྘৭Λಁա͢ΔͨΊmaskColor͸float3(0, 1, 0)ʹͳΔ const float3 maskColor = float3(*colorRed, *colorGreen, *colorBlue); // RGB͔ΒY΁ͷࣹ const float3 YVector = float3(0.2989, 0.5866, 0.1145); // maskColor(྘৭)ΛYCrCb΁ม׵ const float maskY = dot(maskColor, YVector); const float maskCr = 0.7131 * (maskColor.r - maskY); const float maskCb = 0.5647 * (maskColor.b - maskY); // ࠲ඪͷRGBΛYCrCb΁ม׵ const float Y = dot(inColor.rgb, YVector); const float Cr = 0.7131 * (inColor.r - Y); const float Cb = 0.5647 * (inColor.b - Y); // YCrCbɺᮢ஋ɺ྘৭ͱͷڑ཭ʹΑΓಁա౓Λࢉग़ const float alpha = smoothstep( *threshold, *threshold + *smoothing, distance(float2(Cr, Cb), float2(maskCr, maskCb)) ); // ࠲ඪͷRGBΛࢉग़ͨ͠ಁա౓Ͱग़ྗ͢Δ৭Λܭࢉ const float4 outColor = alpha * float4(inColor.r, inColor.g, inColor.b, 1.0); outTexture.write(outColor, gid); } !54
55. #include <metal_stdlib> using namespace metal; /// - SeeAlso: http://www.fundza.com/rman_shaders/smoothstep/index.html kernel void ChromaKeyFilter( // ೖྗը૾৘ใ texture2d<float, access::read> inTexture [[ texture(0) ]], // ग़ྗը૾৘ใ texture2d<float, access::write> outTexture [[ texture(1) ]], // ϚεΫର৅ͷ੺੒෼ const device float *colorRed [[ buffer(0) ]], // ϚεΫର৅ͷ྘੒෼ const device float *colorGreen [[ buffer(1) ]], // ϚεΫର৅ͷ੨੒෼ const device float *colorBlue [[ buffer(2) ]], // ಁա஋͕0-1ͷྖҬͷԼݶ const device float *threshold [[ buffer(3) ]], // ಁա஋͕0-1ͷྖҬͷԼݶ͔Β্ݶ·Ͱͷઈର஋ const device float *smoothing [[ buffer(4) ]], // ࠲ඪ৘ใ uint2 gid [[ thread_position_in_grid ]]) { !55
56. { const float4 inColor = inTexture.read(gid); // maskͷର৅ͷ৭ɻ྘৭Λಁա͢ΔͨΊmaskColor͸float3(0, 1, 0)ʹͳΔ const float3 maskColor = float3(*colorRed, *colorGreen, *colorBlue); // RGB͔ΒY΁ͷࣹ const float3 YVector = float3(0.2989, 0.5866, 0.1145); // maskColor(྘৭)ΛYCrCb΁ม׵ const float maskY = dot(maskColor, YVector); const float maskCr = 0.7131 * (maskColor.r - maskY); const float maskCb = 0.5647 * (maskColor.b - maskY); // ࠲ඪͷRGBΛYCrCb΁ม׵ const float Y = dot(inColor.rgb, YVector); const float Cr = 0.7131 * (inColor.r - Y); const float Cb = 0.5647 * (inColor.b - Y); // YCrCbɺᮢ஋ɺ྘৭ͱͷڑ཭ʹΑΓಁա౓Λࢉग़ const float alpha = smoothstep( *threshold, *threshold + *smoothing, distance(float2(Cr, Cb), float2(maskCr, maskCb)) ); // ࠲ඪͷRGBΛࢉग़ͨ͠ಁա౓Ͱग़ྗ͢Δ৭Λܭࢉ const float4 outColor = alpha * float4(inColor.r, inColor.g, inColor.b, 1.0); outTexture.write(outColor, gid); } !56
57. !57
58. !58 7JEFP.FUBM7JFX
    .5,7JFX "71MBZFS $*$POUFYU .5,7JFX GVOD
    ESBX ݱࡏͷಈըσʔλΛը૾Խ Ұ࣌.5,7JFXʹϨϯμϦϯά ΫϩϚΩʔγΣʔμʔॲཧ
59. !59
60. !60
61. ໨࣍ "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ
    
    ಁաಈըΛ࠶ੜ͢Δ
    
    
    
    
    
    ॲཧͷྲྀΕ
    
    
    
    
    
    ΫϩϚΩʔॲཧ.FUBMγΣʔμʔղઆ
    ͓·͚
    6*7JFXΛಁաදࣔͯ͠ΈΔ
    %FNP !61
62. ໨࣍ "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ
    ಁաಈըΛ࠶ੜ͢Δ
    
    
    
    
    
    ॲཧͷྲྀΕ
    
    
    
    
    
    ΫϩϚΩʔॲཧ.FUBMγΣʔμʔղઆ
    
    ͓·͚
    6*7JFXΛಁաදࣔͯ͠ΈΔ
    %FNP !62
63. 7JFXΛಁաදࣔ͢Δ !63
64. 7JFXΛಁաදࣔ͢Δ 6*7JFXΛը૾Խ͢Δख๏Ͱ੒ޭ
    ͨͩ͠ඳըύϑΥʔϚϯε͸མͪΔ !64
65. !65 7JEFP.FUBM7JFX
    .5,7JFX "71MBZFS $*$POUFYU .5,7JFX GVOD
    ESBX ݱࡏͷಈըσʔλΛը૾Խ Ұ࣌.5,7JFXʹϨϯμϦϯά ΫϩϚΩʔγΣʔμʔॲཧ
66. !66 7JEFP.FUBM7JFX
    .5,7JFX "71MBZFS $*$POUFYU .5,7JFX GVOD
    ESBX 6*7JFXΛը૾Խ Ұ࣌.5,7JFXʹϨϯμϦϯά ΫϩϚΩʔγΣʔμʔॲཧ
67. override func draw(_ dirtyRect: CGRect) { guard let device = device, let drawable = currentDrawable, let tempDrawable = bufferMtkView.currentDrawable, let image = makeCurrentVideoImage() else { return } ciContext.render(image, to: tempDrawable.texture, commandBuffer: nil, bounds: bounds, colorSpace: colorSpace) colorPixelFormat = tempDrawable.texture.pixelFormat let commandBuffer = commandQueue.makeCommandBuffer()! let commandEncoder = commandBuffer.makeComputeCommandEncoder()! commandEncoder.setComputePipelineState(pipelineState) commandEncoder.setTexture(tempDrawable.texture, index: 0) commandEncoder.setTexture(drawable.texture, index: 1) let factors: [Float] = [ 0, // red 1, // green 0, // blue 0.43, // threshold 0.11 // smoothing ] for i in 0..<factors.count { var factor = factors[i] let size = max(MemoryLayout<Float>.size, 16) let buffer = device.makeBuffer( bytes: &factor, length: size, options: [.storageModeShared] ) commandEncoder.setBuffer(buffer, offset: 0, index: i) } commandEncoder.dispatchThreadgroups(threadgroupsPerGrid, threadsPerThreadgroup: threadsPerThreadgroup) commandEncoder.endEncoding() commandBuffer.present(drawable) commandBuffer.commit() } !67
68. override func draw(_ dirtyRect: CGRect) { guard let device = device, let drawable = currentDrawable, let tempDrawable = bufferMtkView.currentDrawable, let image = makeCurrentViewImage() else { return } ciContext.render(image, to: tempDrawable.texture, commandBuffer: nil, bounds: bounds, colorSpace: colorSpace) colorPixelFormat = tempDrawable.texture.pixelFormat let commandBuffer = commandQueue.makeCommandBuffer()! let commandEncoder = commandBuffer.makeComputeCommandEncoder()! commandEncoder.setComputePipelineState(pipelineState) commandEncoder.setTexture(tempDrawable.texture, index: 0) commandEncoder.setTexture(drawable.texture, index: 1) let factors: [Float] = [ 0, // red 1, // green 0, // blue 0.43, // threshold 0.11 // smoothing ] for i in 0..<factors.count { var factor = factors[i] let size = max(MemoryLayout<Float>.size, 16) let buffer = device.makeBuffer( bytes: &factor, length: size, options: [.storageModeShared] ) commandEncoder.setBuffer(buffer, offset: 0, index: i) } commandEncoder.dispatchThreadgroups(threadgroupsPerGrid, threadsPerThreadgroup: threadsPerThreadgroup) commandEncoder.endEncoding() commandBuffer.present(drawable) commandBuffer.commit() } !68
69. func makeCurrentViewImage(view: UIView) -> CIImage? { UIGraphicsBeginImageContextWithOptions(view.frame.size, true, 0) view.drawHierarchy(in: view.bounds, afterScreenUpdates: false) let image = UIGraphicsGetImageFromCurrentImageContext()! UIGraphicsEndImageContext(); return CIImage(image: image) } !69
70. 8,8FC7JFXΛಁաͤ͞Δͱ !70
71. !71
72. !72
73. 8FC35$઀ଓͨ͠7JFXΛಁաͤ͞Δͱ !73
74. !74
75. ໨࣍ "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ
    ಁաಈըΛ࠶ੜ͢Δ
    
    
    
    
    
    ॲཧͷྲྀΕ
    
    
    
    
    
    ΫϩϚΩʔॲཧ.FUBMγΣʔμʔղઆ
    
    ͓·͚
    6*7JFXΛಁաදࣔͯ͠ΈΔ
    %FNP !75
76. ໨࣍ "3,JUΛ࢖͍ͳ͕ΒಈըΛ࠶ੜ͢Δ
    ಁաಈըΛ࠶ੜ͢Δ
    
    
    
    
    
    ॲཧͷྲྀΕ
    
    
    
    
    
    ΫϩϚΩʔॲཧ.FUBMγΣʔμʔղઆ
    ͓·͚
    6*7JFXΛಁաදࣔͯ͠ΈΔ
    
    %FNP !76
77. %FNP 
    Χϝϥө૾΁ͷΫϩϚΩʔγΣʔμʔॲཧ
    
    ಁաಈը࠶ੜ !77
78. ิ଍
    J04Ҏ߱Ͱ͸ 
    )&7$
    7JEFP
    XJUI
    "MQIB
    
    લޙͷΧϝϥಉ࣌࢖༻
    
    "37JFX !78
79. αϯϓϧίʔυʹ͍ͭͯ 
    $*'JMUFSͰͷ੩ࢭըΫϩϚΩʔॲཧ
    
    .FUBMͰͷಈըΫϩϚΩʔॲཧ
    
    "3,JUͱಈըදࣔ
    
    "3,JUͱಁաಈըදࣔ
    
    "3,JUͱ8FC35$ͱಁաදࣔ !79
80. ؔ࿈৘ใɾΫϨδοτ άϦʔϯόοΫΞΠυϧɹ.JLB 3JLB
    IUUQTXXXZPVUVCFDPNXBUDI WX0OTBMKG"P
    )PMPHSBQIJD
    1SPKFDUJPO
    IUUQTXXXTIBEFSUPZDPNWJFX.UK9[N
    <J04
    ><"3,JU>
    ۙॴͷறं৔ͰɺେܕσΟεϓϨΠͰϏσΦΛݟΔʂ
    IUUQTEFWDMBTTNFUIPEKQTNBSUQIPOFJQIPOFJPTBSLJUWJEFP
    "3,JUͷͨΊͷ%਺ֶ
    IUUQTRJJUBDPNLCPZJUFNTGFGEEBDGC
    "3,JU
    
    4QSJUF,JU
    
    TFU
    QJYFM#V⒎FS"UUSJCVUFT
    UP
    4,7JEFP/PEF
    PS
    NBLF
    USBOTQBSFOU
    QJYFMT
    JO
    WJEFP
    DISPNBLFZ
    F⒎FDU
    BOPUIFS
    XBZ
    IUUQTTUBDLPWFSqPXDPNRVFTUJPOTBSLJUTQSJUFLJUTFUQJYFMCV⒎FSBUUSJCVUFTUPTLWJEFPOPEFPSNBLFUSBOTQBSFOU
    )&7$
    7JEFP
    XJUI
    "MQIB
    IUUQTEFWFMPQFSBQQMFDPNWJEFPTQMBZXXED !80
81. ࡞Γ͍ͨͷ͸͜ΜͳΠϝʔδ !81
82. !82 ग़య
    ,FJJDIJ
    .BUTVEB
    ):1&33&"-*5:
    
    IUUQTXXXZPVUVCFDPNXBUDI W:+HJW:[4T
83. !83 ग़య
    ϤγμλΧϢΩ
    
    !@@4UFX@@
    IUUQTUXJUUFSDPN@@4UFX@@TUBUVT T
84. 
    &OKPZ
    J04%$
    
    IUUQTRJJUBDPNTBUPTIJJUFNTEFBFBDCFCFG !84 5XJUUFS !TINEFWFMPQ ϑΥϩʔɺ౤ථ͓ئ͍͠·͢ʂ
    ιʔείʔυ౳΁ͷϦϯΫ͸ͪ͜Β͔Β IUUQTHJUIVCDPNTBUPTIJ"3,JU$ISPNB,FZ4BNQMFT
85. ิ଍
    ଞʹͲ͏͍͏ํ๏͕͋Δͷ͔ !85
86. !86  ΫϩϚΩʔॲཧ$*'JMUFSΛDBQUVSF0VUQVU @EJE0VUQVUGSPN Ͱద༻  4,&⒎FDU/PEFʹΫϩϚΩʔॲཧ$*'JMUFSઃఆ  4$/.BUFSJBMͷTIBEFS.PEJpFSTʹΫϩϚΩʔγΣʔμʔॲཧઃఆ  4$/.BUFSJBMʹΞϧϑΝ஋ઃఆγΣʔμʔॲཧɺUSBOTQBSFOUDPOUFOUTʹϚεΫಈըઃఆ  όοϑΝྖҬʹΫϩϚΩʔγΣʔμʔద༻݁ՌΛอ࣋͠ඳըʹ࢖༻
87. Ҋ֎৭ʑ͋ͬͨ !87