100日間AR表現を実装して見つけた面白い実装を全力解説

Transcript

1. ೔ؒ"3දݱΛ࣮૷ͯ͠
   ݟ͚ͭͨ໘ന͍࣮૷Λશྗղઆʂ
   4BUPTIJ)BUUPSJ
   J04%$
   !TINEFWFMPQ
   

   View Slide

2. IUUQTUXJUUFSDPNIBTIUBHIVOESFE@EBZT@BS@DIBMMFOHF TSDIBTIUBH@DMJDLGMJWF
   

   View Slide

3. IUUQTHJUIVCDPNTBUPTIJ"3@%BZT
   

   View Slide

4. View Slide

5. ໨࣍
   إϖΠϯτ
   ݱ࣮εϓϥτΡʔϯ
   "3ͳͧͱ͖
   #MPUUFS.FEJB෩ޮՌ
   &ZF5SBDL)FBU.BQ
   

   View Slide

6. View Slide

7. IUUQTEFWFMPQFSBQQMFDPNEPDVNFOUBUJPOBSLJUDPOUFOU@BODIPSTUSBDLJOH@BOE@WJTVBMJ[JOH@GBDFT
   

   View Slide

8. View Slide

9. View Slide

10. class DrawnImageView: UIImageView {
    
    
    private var swiped: Bool = false
    
    
    private var lastPoint: CGPoint = .zero
    
    
    ...
    
    
    override func touchesMoved(_ touches: Set, with event: UIEvent?) {
    
    
    super.touchesMoved(touches, with: event)
    
    
    guard let touch = touches.first else { return }
    
    
    swiped = true
    
    
    let currentPoint = touch.location(in: self)
    
    
    if lastPoint == .zero {
    
    
    lastPoint = currentPoint
    
    
    }
    
    
    drawLine(from: lastPoint, to: currentPoint)
    
    
    lastPoint = currentPoint
    
    
    }
    
    
    override func touchesEnded(_ touches: Set, with event: UIEvent?) {
    
    
    super.touchesEnded(touches, with: event)
    
    
    lastPoint = .zero
    
    
    }
    
    
    private func drawLine(from fromPoint: CGPoint, to toPoint: CGPoint) { ... }
    
    
    }
    

    View Slide

11. class DrawnImageView: UIImageView {
    
    
    private var swiped: Bool = false
    
    
    private var lastPoint: CGPoint = .zero
    
    
    ...
    
    
    override func touchesMoved(_ touches: Set, with event: UIEvent?) {
    
    
    super.touchesMoved(touches, with: event)
    
    
    guard let touch = touches.first else { return }
    
    
    swiped = true
    
    
    let currentPoint = touch.location(in: self)
    
    
    if lastPoint == .zero {
    
    
    lastPoint = currentPoint
    
    
    }
    
    
    drawLine(from: lastPoint, to: currentPoint)
    
    
    lastPoint = currentPoint
    
    
    }
    
    
    override func touchesEnded(_ touches: Set, with event: UIEvent?) {
    
    
    super.touchesEnded(touches, with: event)
    
    
    lastPoint = .zero
    
    
    }
    
    
    private func drawLine(from fromPoint: CGPoint, to toPoint: CGPoint) { ... }
    
    
    }
    

    View Slide

12. private func drawLine(from fromPoint: CGPoint, to toPoint: CGPoint) {
    
    
    UIGraphicsBeginImageContext(frame.size)
    
    
    guard let context = UIGraphicsGetCurrentContext() else { return }
    
    
    image?.draw(in: bounds)
    
    
    context.move(to: fromPoint)
    
    
    context.addLine(to: toPoint)
    
    
    context.setLineCap(.round)
    
    
    context.setBlendMode(.normal)
    
    
    context.setLineWidth(8)
    
    
    context.setStrokeColor(UIColor.black.cgColor)
    
    
    context.strokePath()
    
    
    image = UIGraphicsGetImageFromCurrentImageContext()
    
    
    UIGraphicsEndImageContext()
    
    
    }
    

    View Slide

13. View Slide

14. func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor) {
    
    
    guard let faceGeometry = node.geometry as? ARSCNFaceGeometry,
    
    
    let faceAnchor = anchor as? ARFaceAnchor
    
    
    else { return }
    
    
    faceGeometry.update(from: faceAnchor.geometry)
    
    
    if let imageView = imageView {
    
    
    DispatchQueue.main.async {
    
    
    let material = faceGeometry.firstMaterial!
    
    
    material.diffuse.contents = imageView.image
    
    
    }
    
    
    }
    
    
    }
    
    
    "34$/7JFX%FMFHBUF
    

    View Slide

15. View Slide

16. IUUQTEFWFMPQFSBQQMFDPNEPDVNFOUBUJPOSFBMJUZLJUDSFBUJOH@BO@BQQ@GPS@GBDFQBJOUJOH@JO@BS
    

    View Slide

17. View Slide

18. 8*1
    

    View Slide

19. ໨࣍
    إϖΠϯτ
    ݱ࣮εϓϥτΡʔϯ
    "3ͳͧͱ͖
    #MPUUFS.FEJB෩ޮՌ
    &ZF5SBDL)FBU.BQ
    

    View Slide

20. View Slide

21. -J%"3ͰͷεΩϟϯͱ.FTIԽ
    

    View Slide

22. override func viewWillAppear(_ animated: Bool) {
    
    
    super.viewWillAppear(animated)
    
    
    let configuration = ARWorldTrackingConfiguration()
    
    
    configuration.sceneReconstruction = .mesh
    
    
    sceneView.session.run(configuration)
    
    
    }
    

    View Slide

23. func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
    
    
    for anchor in anchors {
    
    
    if let meshAnchor = anchor as? ARMeshAnchor {
    
    
    guard let camera = session.currentFrame?.camera else { return }
    
    
    let meshGeometry = buildGeometry(meshAnchor: meshAnchor, camera: camera)
    
    
    let node = SCNNode(geometry: meshGeometry)
    
    
    node.simdTransform = meshAnchor.transform
    
    
    let shape = SCNPhysicsShape(geometry: node.geometry!,
    
    
    options: [.type: SCNPhysicsShape.ShapeType.concavePolyhedron])
    
    
    node.physicsBody = SCNPhysicsBody(type: .static, shape: shape)
    
    
    knownAnchors[anchor.identifier] = node
    
    
    sceneView.scene.rootNode.addChildNode(node)
    
    
    }
    
    
    }
    
    
    }
    "34FTTJPO%FMFHBUF
    

    View Slide

24. func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
    
    
    for anchor in anchors {
    
    
    if let meshAnchor = anchor as? ARMeshAnchor {
    
    
    guard let camera = session.currentFrame?.camera else { return }
    
    
    let meshGeometry = buildGeometry(meshAnchor: meshAnchor, camera: camera)
    
    
    let node = SCNNode(geometry: meshGeometry)
    
    
    node.simdTransform = meshAnchor.transform
    
    
    let shape = SCNPhysicsShape(geometry: node.geometry!,
    
    
    options: [.type: SCNPhysicsShape.ShapeType.concavePolyhedron])
    
    
    node.physicsBody = SCNPhysicsBody(type: .static, shape: shape)
    
    
    knownAnchors[anchor.identifier] = node
    
    
    sceneView.scene.rootNode.addChildNode(node)
    
    
    }
    
    
    }
    
    
    }
    "34FTTJPO%FMFHBUF
    

    View Slide

25. func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
    
    
    for anchor in anchors {
    
    
    if let node = knownAnchors[anchor.identifier] {
    
    
    if let meshAnchor = anchor as? ARMeshAnchor {
    
    
    guard let camera = session.currentFrame?.camera else { return }
    
    
    let meshGeometry = buildGeometry(meshAnchor: meshAnchor, camera: camera)
    
    
    node.geometry = meshGeometry
    
    
    node.simdTransform = meshAnchor.transform
    
    
    let shape = SCNPhysicsShape(geometry: node.geometry!,
    
    
    options: [.type: SCNPhysicsShape.ShapeType.concavePolyhedron])
    
    
    node.physicsBody = SCNPhysicsBody(type: .static, shape: shape)
    
    
    knownAnchors[anchor.identifier] = node
    
    
    }
    
    
    }
    
    
    }
    
    
    }
    "34FTTJPO%FMFHBUF
    

    View Slide

26. func buildGeometry(meshAnchor: ARMeshAnchor, camera: ARCamera) -> SCNGeometry {
    
    
    let vertices = meshAnchor.geometry.vertices
    
    
    let faces = meshAnchor.geometry.faces
    
    
    let size = camera.imageResolution
    
    
    let vertexSource = SCNGeometrySource(buffer: vertices.buffer,
    
    
    vertexFormat: vertices.format,
    
    
    semantic: .vertex,
    
    
    vertexCount: vertices.count,
    
    
    dataOffset: vertices.offset,
    
    
    dataStride: vertices.stride)
    
    
    let modelMatrix = meshAnchor.transform
    
    
    var textCords = [CGPoint]()
    
    
    for index in 0..
    
    let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * index)
    
    
    let vertex = vertexPointer.assumingMemoryBound(to: (Float, Float, Float).self).pointee
    
    
    let vertex4 = SIMD4(vertex.0, vertex.1, vertex.2, 1)
    
    
    let world_vertex4 = simd_mul(modelMatrix, vertex4)
    
    
    let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z)
    
    
    let pt = camera.projectPoint(world_vector3,
    
    
    orientation: .portrait,
    
    
    viewportSize: CGSize(width: CGFloat(size.height),
    
    
    height: CGFloat(size.width)))
    
    
    let v = 1.0 - Float(pt.x) / Float(size.height)
    
    
    let u = Float(pt.y) / Float(size.width)
    
    
    let c = CGPoint(x: CGFloat(v), y: CGFloat(u))
    
    
    textCords.append(c)
    
    
    }
    
    
    let textureSource = SCNGeometrySource(textureCoordinates: textCords)
    
    
    let normalsSource = SCNGeometrySource(meshAnchor.geometry.normals, semantic: .normal)
    
    
    let faceData = Data(bytes: faces.buffer.contents(), count: faces.buffer.length)
    
    
    let geometryElement = SCNGeometryElement(data: faceData, primitiveType: .triangles, primitiveCount: faces.count, bytesPerIndex: faces.bytesPerIndex)
    
    
    let nodeGeometry = SCNGeometry(sources: [vertexSource, textureSource, normalsSource], elements: [geometryElement])
    
    
    let material = SCNMaterial()
    
    
    material.isDoubleSided = true
    
    
    let skScene = SKScene(size: CGSize(width: SPRITE_SIZE, height: SPRITE_SIZE))
    
    
    skScene.backgroundColor = .clear
    
    
    material.diffuse.contents = skScene
    
    
    nodeGeometry.materials = [material]
    
    
    return nodeGeometry
    
    
    }
    

    View Slide

27. let vertices = meshAnchor.geometry.vertices
    
    
    let vertexSource = SCNGeometrySource(buffer: vertices.buffer,
    
    
    vertexFormat: vertices.format,
    
    
    semantic: .vertex,
    
    
    vertexCount: vertices.count,
    
    
    dataOffset: vertices.offset,
    
    
    dataStride: vertices.stride)
    

    View Slide

28. var textCords = [CGPoint]()
    
    
    for index in 0..
    
    let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * index)
    
    
    let vertex = vertexPointer.assumingMemoryBound(to: (Float, Float, Float).self).pointee
    
    
    let vertex4 = SIMD4(vertex.0, vertex.1, vertex.2, 1)
    
    
    let world_vertex4 = simd_mul(modelMatrix, vertex4)
    
    
    let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z)
    
    
    let pt = camera.projectPoint(world_vector3,
    
    
    orientation: .portrait,
    
    
    viewportSize: CGSize(width: CGFloat(size.height),
    
    
    height: CGFloat(size.width)))
    
    
    let v = 1.0 - Float(pt.x) / Float(size.height)
    
    
    let u = Float(pt.y) / Float(size.width)
    
    
    let c = CGPoint(x: CGFloat(v), y: CGFloat(u))
    
    
    textCords.append(c)
    
    
    }
    
    
    let textureSource = SCNGeometrySource(textureCoordinates: textCords)
    

    View Slide

29. let normalsSource = SCNGeometrySource(meshAnchor.geometry.normals, semantic: .normal)
    

    View Slide

30. let faceData = Data(bytes: faces.buffer.contents(), count: faces.buffer.length)
    
    
    let geometryElement = SCNGeometryElement(data: faceData, primitiveType: .triangles, primitiveCount:
    faces.count, bytesPerIndex: faces.bytesPerIndex)
    

    View Slide

31. let nodeGeometry = SCNGeometry(sources: [vertexSource, textureSource, normalsSource], elements:
    [geometryElement])
    
    
    let material = SCNMaterial()
    
    
    material.isDoubleSided = true
    
    
    let skScene = SKScene(size: CGSize(width: SPRITE_SIZE, height: SPRITE_SIZE))
    
    
    skScene.backgroundColor = .clear
    
    
    material.diffuse.contents = skScene
    
    
    nodeGeometry.materials = [material]
    
    
    return nodeGeometry
    

    View Slide

32. View Slide

33. ΧϥʔϘʔϧڍಈ
    

    View Slide

34. private func launchColorBall() {
    
    
    let ball = SCNNode()
    
    
    let sphere = SCNSphere(radius: 0.08)
    
    
    ball.geometry = sphere
    
    
    let hue = CGFloat(arc4random()) / CGFloat(UInt32.max)
    
    
    ball.geometry!.firstMaterial?.diffuse.contents = SKColor(hue: hue, saturation: 1, brightness: 1, alpha: 1)
    
    
    ball.geometry!.firstMaterial?.fresnelExponent = 1.0
    
    
    ball.physicsBody = .dynamic()
    
    
    ball.physicsBody!.restitution = 0.9
    
    
    ball.physicsBody!.categoryBitMask = 0x4
    
    
    ball.physicsBody!.contactTestBitMask = ~0
    
    
    ball.physicsBody!.collisionBitMask = ~(0x4)
    
    
    guard let camera = sceneView.pointOfView else { return }
    
    
    let cameraPos = SCNVector3Make(0, 0, -0.5)
    
    
    let position = camera.convertPosition(cameraPos, to: nil)
    
    
    ball.position = position
    
    
    let (direction, _) = getUserVector()
    
    
    ball.physicsBody!.velocity = SCNVector3Make(
    
    
    direction.x * 8.0,
    
    
    direction.y * 8.0,
    
    
    direction.z * 8.0)
    
    
    sceneView.scene.rootNode.addChildNode(ball)
    
    
    }
    

    View Slide

35. private func getUserVector() -> (SCNVector3, SCNVector3) {
    
    
    if let frame = sceneView.session.currentFrame {
    
    
    let mat = SCNMatrix4(frame.camera.transform) // 4x4 transform matrix describing camera in world space
    
    
    let dir = SCNVector3(-1 * mat.m31, -1 * mat.m32, -1 * mat.m33) // orientation of camera in world space
    
    
    let pos = SCNVector3(mat.m41, mat.m42, mat.m43) // location of camera in world space
    
    
    return (dir, pos)
    
    
    }
    
    
    return (SCNVector3(0, 0, -1), SCNVector3(0, 0, -0.2))
    
    
    }
    

    View Slide

36. override func viewDidLoad() {
    
    
    super.viewDidLoad()
    
    
    ...
    
    
    sceneView.scene.physicsWorld.contactDelegate = self
    
    
    }
    

    View Slide

37. func physicsWorld(_ world: SCNPhysicsWorld, didBegin contact: SCNPhysicsContact) {
    
    
    var ball: SCNNode? = nil
    
    
    var target: SCNNode? = nil
    
    
    if contact.nodeA.physicsBody!.type == .dynamic {
    
    
    ball = contact.nodeA
    
    
    target = contact.nodeB
    
    
    } else if contact.nodeB.physicsBody!.type == .dynamic {
    
    
    ball = contact.nodeB
    
    
    target = contact.nodeA
    
    
    }
    
    
    if let ball = ball {
    
    
    let pointA = SCNVector3Make(contact.contactPoint.x, contact.contactPoint.y, contact.contactPoint.z + 20)
    
    
    let pointB = SCNVector3Make(contact.contactPoint.x, contact.contactPoint.y, contact.contactPoint.z - 20)
    
    
    let results = sceneView.scene.rootNode.hitTestWithSegment(from: pointA, to: pointB, options: [:])
    
    
    if !results.isEmpty, let hit = results.first {
    
    
    let color = ball.geometry!.firstMaterial!.diffuse.contents as! SKColor
    
    
    addPaintAtLocation(hit.textureCoordinates(withMappingChannel: 0), color: color, targetNode: target!)
    
    
    }
    
    
    ball.removeFromParentNode()
    
    
    }
    
    
    }
    4$/1IZTJDT$POUBDU%FMFHBUF
    

    View Slide

38. private func addPaintAtLocation(_ _p: CGPoint, color: SKColor, targetNode: SCNNode) {
    
    
    guard let skScene = targetNode.geometry!.firstMaterial!.diffuse.contents as? SKScene else { return }
    
    
    var p = _p
    
    
    p.x *= SPRITE_SIZE
    
    
    p.y *= SPRITE_SIZE
    
    
    let node: SKNode = SKSpriteNode()
    
    
    node.position = p
    
    
    let subNode = SKSpriteNode(imageNamed: "splash.png")
    
    
    subNode.color = color
    
    
    subNode.colorBlendFactor = 1
    
    
    node.addChild(subNode)
    
    
    skScene.addChild(node)
    
    
    }
    
    
    

    View Slide

39. View Slide

40. View Slide

41. 8*1
    

    View Slide

42. ໨࣍
    إϖΠϯτ
    ݱ࣮εϓϥτΡʔϯ
    "3ͳͧͱ͖
    #MPUUFS.FEJB෩ޮՌ
    &ZF5SBDL)FBU.BQ
    

    View Slide

43. View Slide

44. View Slide

45. private func addNode(image: UIImage, position: SCNVector3, rotation: SCNVector4? = nil) {
    
    
    let node = SCNNode()
    
    
    let geometry = SCNPlane(width: 0.4, height: 0.4)
    
    
    geometry.firstMaterial?.diffuse.contents = image
    
    
    geometry.firstMaterial?.isDoubleSided = true
    
    
    node.geometry = geometry
    
    
    if let rotation = rotation {
    
    
    node.rotation = rotation
    
    
    }
    
    
    node.position = position
    
    
    sceneView.scene.rootNode.addChildNode(node)
    
    
    }
    
    
    private func addNodes() {
    
    
    addNode(image: UIImage(named: "nazo001")!, position: SCNVector3(0, 0, -1.1))
    
    
    addNode(image: UIImage(named: "nazo002")!,
    
    
    position: SCNVector3(-0.5, 0.15, -1.5),
    
    
    rotation: SCNVector4(0, 1, 0, 0.5 * Double.pi))
    
    
    addNode(image: UIImage(named: "nazo003")!,
    
    
    position: SCNVector3(0, 0.1, -1.5),
    
    
    rotation: SCNVector4(0, 1, 0, 0.5 * Double.pi))
    
    
    }
    

    View Slide

46. 8*1
    

    View Slide

47. ໨࣍
    إϖΠϯτ
    ݱ࣮εϓϥτΡʔϯ
    "3ͳͧͱ͖
    #MPUUFS.FEJB෩ޮՌ
    &ZF5SBDL)FBU.BQ
    

    View Slide

48. View Slide

49. View Slide

50. IUUQTRJJUBDPNQJDGLJUFNTCFEDEFD
    

    View Slide

51. IUUQTEFWFMPQFSBQQMFDPNEPDVNFOUBUJPOBSLJUDBNFSB@MJHIUJOH@BOE@F
    ff
    FDUTF
    ff
    FDUJOH@QFPQMF@PDDMVTJPO@JO@DVTUPN@SFOEFSFST
    

    View Slide

52. ਓମ෦෼ͷԑऔΓ
    ͭͷ#MVSద༻
    

    View Slide

53. func update() {
    
    
    let commandBuffer = commandQueue.makeCommandBuffer()!
    
    
    ...
    
    
    guard let currentFrame = session.currentFrame else { return }
    
    
    let pixelBuffer = currentFrame.capturedImage
    
    
    if CVPixelBufferGetPlaneCount(pixelBuffer) < 2 { return }
    
    
    capturedImageTextureY = createTexture(fromPixelBuffer: pixelBuffer, pixelFormat: .r8Unorm, planeIndex: 0)
    
    
    capturedImageTextureCbCr = createTexture(fromPixelBuffer: pixelBuffer, pixelFormat: .rg8Unorm, planeIndex:1)
    
    
    ...
    
    
    alphaTexture = matteGenerator.generateMatte(from: currentFrame, commandBuffer: commandBuffer)
    
    
    ...
    
    
    }
    

    View Slide

54. func update() {
    
    
    ...
    
    
    if let width = alphaTexture?.width, let height = alphaTexture?.height {
    
    
    let colorDesc = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .bgra8Unorm,
    
    
    width: width, height: height, mipmapped: false)
    
    
    colorDesc.usage = [.shaderRead, .shaderWrite]
    
    
    whiteBlurTexture = device.makeTexture(descriptor: colorDesc)
    
    
    outerBlurTexture = device.makeTexture(descriptor: colorDesc)
    
    
    let threadCountW = (width + threadgroupSize.width - 1) / threadgroupSize.width
    
    
    let threadCountH = (height + threadgroupSize.height - 1) / threadgroupSize.height
    
    
    let threadgroupCount = MTLSizeMake(threadCountW, threadCountH, 1)
    
    
    let computeEncoder = commandBuffer.makeComputeCommandEncoder()!
    
    
    computeEncoder.setComputePipelineState(computeState)
    
    
    computeEncoder.setTexture(alphaTexture, index: 0)
    
    
    computeEncoder.setTexture(whiteBlurTexture, index: 1)
    
    
    computeEncoder.setTexture(outerBlurTexture, index: 2)
    
    
    computeEncoder.dispatchThreadgroups(threadgroupCount, threadsPerThreadgroup: threadgroupSize)
    
    
    computeEncoder.endEncoding()
    
    
    }
    
    
    time += 1
    
    
    let whiteIntensity = Int(6) | 0x01
    
    
    let kernel1 = MPSImageTent(device: device, kernelWidth: whiteIntensity, kernelHeight: whiteIntensity)
    
    
    kernel1.encode(commandBuffer: commandBuffer,
    
    
    inPlaceTexture: &whiteBlurTexture!, fallbackCopyAllocator: nil)
    
    
    let outerIntensity = Int((sin(Float(time)/10) + 2) * 40) | 0x01
    
    
    let kernel2 = MPSImageTent(device: device, kernelWidth: outerIntensity, kernelHeight: outerIntensity)
    
    
    kernel2.encode(commandBuffer: commandBuffer,
    
    
    inPlaceTexture: &outerBlurTexture!, fallbackCopyAllocator: nil)
    
    
    ...
    
    
    }
    

    View Slide

55. kernel void matteConvert(texture2d inTexture [[ texture(0) ]],
    
    
    texture2d outWhiteTexture [[ texture(1) ]],
    
    
    texture2d outOuterTexture [[ texture(2) ]],
    
    
    uint2 gid [[thread_position_in_grid]]) {
    
    
    uint2 textureIndex(gid);
    
    
    if (inTexture.read(textureIndex).r > 0.1) {
    
    
    outWhiteTexture.write(half4(0.0), gid);
    
    
    outOuterTexture.write(half4(0.0), gid);
    
    
    return;
    
    
    }
    
    
    constexpr int scale = 15;
    
    
    constexpr int radius = scale / 2;
    
    
    half color = 0.0;
    
    
    for (int i = 0; i < scale; i++) {
    
    
    for (int j = 0; j < scale; j++) {
    
    
    uint2 textureIndex(gid.x + (i - radius), gid.y + (j - radius));
    
    
    half alpha = inTexture.read(textureIndex).r;
    
    
    if (alpha > 0.1) {
    
    
    color = 1.0;
    
    
    break;
    
    
    }
    
    
    }
    
    
    if (color > 0.0) {
    
    
    break;
    
    
    }
    
    
    }
    
    
    outWhiteTexture.write(half4(color, color, color, 1.0), gid);
    
    
    outOuterTexture.write(half4(color, 0.0, color, 1.0), gid);
    
    
    }
    4IBEFSNFUBM
    

    View Slide

56. ΧϝϥΩϟϓνϟը૾ͱ߹੒
    

    View Slide

57. func update() {
    
    
    ...
    
    
    guard let renderPassDescriptor = mtkView.currentRenderPassDescriptor,
    
    
    let currentDrawable = mtkView.currentDrawable else { return }
    
    
    let compositeRenderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor)!
    
    
    compositeImagesWithEncoder(renderEncoder: compositeRenderEncoder)
    
    
    compositeRenderEncoder.endEncoding()
    
    
    commandBuffer.present(currentDrawable)
    
    
    commandBuffer.commit()
    
    
    }
    

    View Slide

58. fragment half4 compositeImageFragmentShader(CompositeColorInOut in [[ stage_in ]],
    
    
    texture2d capturedImageTextureY [[ texture(0) ]],
    
    
    texture2d capturedImageTextureCbCr [[ texture(1) ]],
    
    
    texture2d whiteColorTexture [[ texture(2) ]],
    
    
    texture2d outerColorTexture [[ texture(3) ]],
    
    
    texture2d alphaTexture [[ texture(4) ]])
    
    
    {
    
    
    constexpr sampler s(address::clamp_to_edge, filter::linear);
    
    
    float2 cameraTexCoord = in.texCoordCamera;
    
    
    float4 rgb = ycbcrToRGBTransform(capturedImageTextureY.sample(s, cameraTexCoord),
    
    
    capturedImageTextureCbCr.sample(s, cameraTexCoord));
    
    
    half4 cameraColor = half4(rgb);
    
    
    half4 whiteColor = half4(whiteColorTexture.sample(s, cameraTexCoord));
    
    
    half4 outerColor = half4(outerColorTexture.sample(s, cameraTexCoord)) * 2.0;
    
    
    return cameraColor + whiteColor + outerColor;
    
    
    }
    4IBEFSNFUBM
    

    View Slide

59. View Slide

60. ໨࣍
    إϖΠϯτ
    ݱ࣮εϓϥτΡʔϯ
    "3ͳͧͱ͖
    #MPUUFS.FEJB෩ޮՌ
    &ZF5SBDL)FBU.BQ
    

    View Slide

61. View Slide

62. var m_data: [UInt8] = [UInt8](repeating: 0, count: 375*3 * 812*3)
    
    
    var heatMapNode: SCNNode = {
    
    
    let node = SCNNode(geometry: SCNPlane(width: 2, height: 2))
    
    
    let program = SCNProgram()
    
    
    program.vertexFunctionName = "heatMapVert"
    
    
    program.fragmentFunctionName = "heatMapFrag"
    
    
    node.geometry?.firstMaterial?.program = program
    
    
    node.geometry?.firstMaterial?.blendMode = SCNBlendMode.add
    
    
    return node
    
    
    }()
    

    View Slide

63. func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) {
    
    
    virtualPhoneNode.transform = (sceneView.pointOfView?.transform)!
    
    
    let options: [String: Any] = [SCNHitTestOption.backFaceCulling.rawValue: false,
    
    
    SCNHitTestOption.searchMode.rawValue: 1,
    
    
    SCNHitTestOption.ignoreChildNodes.rawValue : false,
    
    
    SCNHitTestOption.ignoreHiddenNodes.rawValue : false]
    
    
    let hitTestLeftEye = virtualPhoneNode.hitTestWithSegment(
    
    
    from: virtualPhoneNode.convertPosition(eyeRaycastData!.leftEye.worldPosition, from: nil),
    
    
    to: virtualPhoneNode.convertPosition(eyeRaycastData!.leftEyeEnd.worldPosition, from: nil),
    
    
    options: options)
    
    
    let hitTestRightEye = virtualPhoneNode.hitTestWithSegment(
    
    
    from: virtualPhoneNode.convertPosition(eyeRaycastData!.rightEye.worldPosition, from: nil),
    
    
    to: virtualPhoneNode.convertPosition(eyeRaycastData!.rightEyeEnd.worldPosition, from: nil),
    
    
    options: options)
    
    
    if (hitTestLeftEye.count > 0 && hitTestRightEye.count > 0) {
    
    
    let coords = screenPositionFromHittest(hitTestLeftEye[0], secondResult: hitTestRightEye[0])
    
    
    incrementHeatMapAtPosition(x:Int(coords.x * 3), y:Int(coords.y * 3))
    
    
    let data = NSData(bytes: &m_data, length: phoneWidth * phoneHeight)
    
    
    heatMapNode.geometry?.firstMaterial?.setValue(data, forKey: "heatmapTexture")
    
    
    DispatchQueue.main.async(execute: {() -> Void in
    
    
    self.target.center = CGPoint(x: CGFloat(coords.x), y:CGFloat(coords.y))
    
    
    })
    
    
    }
    
    
    }
    "34$/7JFX%FMFHBUF
    

    View Slide

64. private func incrementHeatMapAtPosition(x: Int, y: Int) {
    
    
    let radius: Int = 46 // in pixels
    
    
    let maxIncrement: Float = 25
    
    
    for curX in x - radius ... x + radius {
    
    
    for curY in y - radius ... y + radius {
    
    
    let idx = posToIndex(x: curX, y: curY)
    
    
    if (idx == -1) { continue }
    
    
    let offset = simd_float2(Float(curX - x), Float(curY - y))
    
    
    let len = simd_length(offset)
    
    
    if (len >= Float(radius)) { continue }
    
    
    let incrementValue = Int((1 - (len / Float(radius))) * maxIncrement)
    
    
    if (255 - m_data[idx] > incrementValue) {
    
    
    m_data[idx] = UInt8(Int(m_data[idx]) + incrementValue)
    
    
    } else {
    
    
    m_data[idx] = 255
    
    
    }
    
    
    }
    
    
    }
    
    
    }
    
    
    private func posToIndex(x: Int, y: Int) -> Int {
    
    
    if (x < 0 || x >= phoneWidth ||
    
    
    y < 0 || y >= phoneHeight) {
    
    
    return -1
    
    
    }
    
    
    return x + y * phoneWidth
    
    
    }
    

    View Slide

65. fragment half4 heatMapFrag(SimpleVertex in [[stage_in]],
    
    
    device r8unorm *heatmapTexture [[buffer(0)]])
    
    
    {
    
    
    int x = round(in.uv.x * 375 * 3);
    
    
    int y = round(in.uv.y * 812 * 3);
    
    
    int index = x + y * 375 * 3;
    
    
    float value = heatmapTexture[index];
    
    
    return ColorForHeat(value);
    
    
    }
    4IBEFSNFUBM
    

    View Slide

66. half4 ColorForHeat(float heat) {
    
    
    float pval = 4.0 * (1.0 - heat) + 0.99;
    
    
    float lb = pval - floor (pval);
    
    
    int pvalCategory = abs(int(floor(pval)));
    
    
    switch (pvalCategory) {
    
    
    case 0:
    
    
    return half4(1.0, 1.0 - lb, 1.0 - lb, heat);
    
    
    case 1:
    
    
    return half4(1.0, lb, 0.0, heat);
    
    
    case 2:
    
    
    return half4(1.0 - lb, 1.0, 0.0, heat);
    
    
    case 3:
    
    
    return half4(0.0, 1.0, lb, heat);
    
    
    case 4:
    
    
    return half4(0.0, 1.0 - lb, 1.0, heat);
    
    
    case 5:
    
    
    return half4(0.0, 0.0, 1.0 - lb, heat);
    
    
    }
    
    
    return half4(1.0, 1.0, 1.0, heat);
    
    
    }
    
    
    4IBEFSNFUBM
    

    View Slide

67. 8*1
    

    View Slide

68. ·ͱΊ
    إϖΠϯτ
    ݱ࣮εϓϥτΡʔϯ
    "3ͳͧͱ͖
    #MPUUFS.FEJB෩ޮՌ
    &ZF5SBDL)FBU.BQ
    BOENPSF
    IUUQTHJUIVCDPNTBUPTIJ"3@%BZT
    IUUQTUXJUUFSDPNIBTIUBHIVOESFE@EBZT@BS@DIBMMFOHF TSDIBTIUBH@DMJDLGMJWF
    

    View Slide