iPhoneのセンサー情報をmacOSアプリでリアルタイム活用するための技術

iOSDC Japan 2023 #iosdc

iPhoneのセンサー情報をmacOSアプリでリアルタイム活用するための技術
https://fortee.jp/iosdc-japan-2023/proposal/00d3250d-c32f-4466-b582-ecd9c27f5266

アバターアプリのGitHub:
https://github.com/vcamapp/app

Twitter: http://twitter.com/tanakasan2525/

Transcript

1. ©︎
   iPhoneͷηϯαʔ৘ใΛ
   

   macOSΞ
   プ
   Ϧ
   で
   ϦΞϧλΠϜ׆༻͢ΔͨΊͷٕज़
   ͨͳͨͭ / Tatsuya Tanaka
   iOSDC Japan 2023 #iosdc
   

   View Slide

2. ©︎
   ͨͳͨͭ / Tatsuya Tanaka
   💎μΠϠϞϯυ💎εϙϯαʔͷϠϑʔגࣜձࣾ
   
   
   ɾiOSΞϓϦࠇଳ
   @tattn
   @tanakasan2525
   

   View Slide

3. ©︎
   ͨͳͨͭ / Tatsuya Tanaka
   @tattn
   @tanakasan2525
   💎μΠϠϞϯυ💎εϙϯαʔͷϠϑʔגࣜձࣾ
   
   
   ɾiOSΞϓϦࠇଳ
   

   View Slide

4. ©︎
   ͨͳͨͭ / Tatsuya Tanaka
   • 💎μΠϠϞϯυ💎εϙϯαʔͷϠϑʔגࣜձࣾ
   
   
   • iOSΞϓϦࠇଳ
   @tattn
   @tanakasan2525
   

   View Slide

5. ©︎
   ͨͳͨͭ / Tatsuya Tanaka
   ⭐ืूதˣˣ
   

   View Slide

6. ©︎
   macOSͰ͸ARKit͕࢖͑ͳ͍
   VNFaceLandmarks2D ARFaceAnchor.blendshapes
   Vision ARKit
   

   View Slide

7. ©︎
   2022 Yahoo Japan Corporation All rights reserved.
   BlendShapes
   

   View Slide

8. ©︎
   ୅දతͳiPhoneͱMacؒͷ௨৴ํ๏
   Core Bluetooth Multipeer Connectivity Network
   ɾWiFiͳ͠Ͱܨ͛Δ
   

   ɾ௿ిྗ
   
   
   ɾόοΫάϥ΢ϯυOK
   ɾѻ͍ʹ͍͘I/F
   
   
   ɾOSver౳ͷ࢓༷ࠩҟ
   ɾP2P
   
   
   ɾঢ়ଶΛಉظ͠΍͍͢
   ɾApple only
   ɾҰൠతͳ௨৴ํ๏
   
   
   ɾϞμϯͳI/F
   

   View Slide

9. ©︎
   ϓϩτίϧ
   Network
   TCP UDP QUIC ΧελϜ
   

   View Slide

10. ©︎
    Bonjour: θϩίϯϑΟΪϡϨʔγϣϯϓϩτίϧ
    Illustration by Stable Diffusion
    NWBrowser(
    
    
    for: .bonjour(type: "_awesomeapp._udp", domain: nil),
    
    
    using: parameters
    
    
    )
    let listener = try NWListener(using: parameters, on: port)
    
    
    listener.service = .init(type: "_awesomeapp._udp")
    αʔϏε໊ + ϓϩτίϧ + υϝΠϯ
    

    View Slide

11. ©︎
    Dataܕ΁ͷม׵
    Data
    

    View Slide

12. ©︎
    Dataܕ΁ͷม׵
    public struct VCamMotion: Equatable {
    
    
    public var version: UInt32
    
    
    public var head: Head
    
    
    public var hands: Hands
    
    
    public var blendShape: BlendShape
    
    
    
    
    public struct Head: Equatable {
    
    
    public var translation: SIMD3
    
    
    public var rotation: simd_quatf
    
    
    }
    
    
    
    
    public struct Hands: Equatable {
    
    
    public var right: Hand
    
    
    public var left: Hand
    
    
    }
    
    
    
    
    public struct Hand: Equatable {
    
    
    public var wrist: SIMD2
    
    
    public var thumbCMC: SIMD2
    
    
    public var littleMCP: SIMD2
    
    
    public var thumbTip: SIMD2
    
    
    public var indexTip: SIMD2
    
    
    public var middleTip: SIMD2
    
    
    public var ringTip: SIMD2
    
    
    public var littleTip: SIMD2
    
    
    }
    
    
    }
    public struct BlendShape: Equatable {
    
    
    public var lookAtPoint: SIMD2
    
    
    public var browDownLeft: Float
    
    
    public var browDownRight: Float
    
    
    public var browInnerUp: Float
    
    
    public var browOuterUpLeft: Float
    
    
    public var browOuterUpRight: Float
    
    
    public var cheekPuff: Float
    
    
    public var cheekSquintLeft: Float
    
    
    public var cheekSquintRight: Float
    
    
    public var eyeBlinkLeft: Float
    
    
    public var eyeBlinkRight: Float
    
    
    public var eyeLookDownLeft: Float
    
    
    public var eyeLookDownRight: Float
    
    
    public var eyeLookInLeft: Float
    
    
    public var eyeLookInRight: Float
    
    
    public var eyeLookOutLeft: Float
    
    
    public var eyeLookOutRight: Float
    
    
    public var eyeLookUpLeft: Float
    
    
    public var eyeLookUpRight: Float
    
    
    public var eyeSquintLeft: Float
    
    
    public var eyeSquintRight: Float
    
    
    public var eyeWideLeft: Float
    
    
    public var eyeWideRight: Float
    
    
    public var jawForward: Float
    
    
    public var jawLeft: Float
    
    
    public var jawOpen: Float
    
    
    public var jawRight: Float
    
    
    public var mouthClose: Float
    
    
    public var mouthDimpleLeft: Float
    
    
    public var mouthDimpleRight: Float
    
    
    public var mouthFrownLeft: Float
    
    
    public var mouthFrownRight: Float
    
    
    public var mouthFunnel: Float
    
    
    public var mouthLeft: Float
    
    
    public var mouthLowerDownLeft: Float
    
    
    public var mouthLowerDownRight: Float
    
    
    public var mouthPressLeft: Float
    
    
    public var mouthPressRight: Float
    
    
    public var mouthPucker: Float
    
    
    public var mouthRight: Float
    
    
    public var mouthRollLower: Float
    
    
    public var mouthRollUpper: Float
    
    
    public var mouthShrugLower: Float
    
    
    public var mouthShrugUpper: Float
    
    
    public var mouthSmileLeft: Float
    
    
    public var mouthSmileRight: Float
    
    
    public var mouthStretchLeft: Float
    
    
    public var mouthStretchRight: Float
    
    
    public var mouthUpperUpLeft: Float
    
    
    public var mouthUpperUpRight: Float
    
    
    public var noseSneerLeft: Float
    
    
    public var noseSneerRight: Float
    
    
    public var tongueOut: Float
    
    
    }
    Data
    17%
    ※iPhone 12ͷ৔߹
    JSONEncoderͷ৔߹
    (0.003 s/frame)
    ඵؒͷॲཧ࣌ؒͷ
    ͕࢖ΘΕͯ͠·͏
    

    View Slide

13. ©︎
    Dataܕ΁ͷߴ଎ͳม׵ํ๏
    struct Head: Equatable {
    
    
    let x: Float
    
    
    let y: Float
    
    
    let z: Float
    
    
    }
    var head = Head(x: 0, y: 1, z: 2)
    
    
    var encoded = Data(bytesNoCopy: &head, count: MemoryLayout.size, deallocator: .none)
    
    
    let decoded = encoded.withUnsafeBytes { $0.load(as: Head.self) }
    
    
    XCTAssertEqual(head, decoded)
    // Dataܕ΁ͷม׵
    // ݩͷܕ΁ͷม׵
    όΠτྻͱͯͦ͠ͷ··ม׵
    

    View Slide

14. ©︎
    ϓϩύςΟͷ࣋ͪํʹ஫ҙ͕ඞཁ
    struct Head: Equatable {
    
    
    let x: Float
    
    
    let y: Float
    
    
    let z: Float
    
    
    }
    var encoded = Data(bytesNoCopy: &head, count: MemoryLayout.size, deallocator: .none)
    
    
    let decoded = encoded.withUnsafeBytes { $0.load(as: Head.self) }
    
    
    XCTAssertEqual(head, decoded)
    

    View Slide

15. ©︎
    ϓϩύςΟͷ࣋ͪํʹ஫ҙ͕ඞཁ
    struct Head: Equatable {
    
    
    let x: Float
    
    
    let y: Float
    
    
    let z: Float
    
    
    let isTracking: Bool
    
    
    }
    var head = Head(x: 0, y: 1, z: 2, isTracking: true)
    ※64bit CPU؀ڥ
    var encoded = Data(bytesNoCopy: &head, count: MemoryLayout.size, deallocator: .none)
    
    
    let decoded = encoded.withUnsafeBytes { $0.load(as: Head.self) }
    
    
    XCTAssertEqual(head, decoded)
    

    View Slide

16. ©︎
    ϓϩύςΟͷ࣋ͪํʹ஫ҙ͕ඞཁ
    struct Head: Equatable {
    
    
    let x: Float
    
    
    let y: Float
    
    
    let z: Float
    
    
    let isTracking: Bool
    
    
    }
    var head = Head(x: 0, y: 1, z: 2, isTracking: true)
    ※64bit CPU؀ڥ
    var encoded = Data(bytesNoCopy: &head, count: MemoryLayout.size, deallocator: .none)
    
    
    let decoded = encoded.withUnsafeBytes { $0.load(as: Head.self) }
    
    
    XCTAssertEqual(head, decoded)
    struct Head: Equatable {
    
    
    let x: Float
    
    
    let y: Float
    
    
    let z: Float
    
    
    let isTracking: Bool
    
    
    }
    

    View Slide

17. ©︎
    ϓϩύςΟͷ࣋ͪํʹ஫ҙ͕ඞཁ
    var head = Head(x: 0, y: 1, z: 2, isTracking: true)
    struct Head: Equatable {
    
    
    let isTracking: Bool
    
    
    let x: Float
    
    
    let y: Float
    
    
    let z: Float
    
    
    }
    var head = Head(isTracking: true, x: 0, y: 1, z: 2)
    ※64bit CPU؀ڥ
    var encoded = Data(bytesNoCopy: &head, count: MemoryLayout.size, deallocator: .none)
    
    
    let decoded = encoded.withUnsafeBytes { $0.load(as: Head.self) }
    
    
    XCTAssertEqual(head, decoded)
    struct Head: Equatable {
    
    
    let x: Float
    
    
    let y: Float
    
    
    let z: Float
    
    
    let isTracking: Bool
    
    
    }
    

    View Slide

18. ©︎
    structͷMemoryLayout
    let layout = MemoryLayout.self
    
    
    print(layout.size) // 13
    
    
    print(layout.alignment) // 4
    
    
    print(layout.stride) // 16
    let layout = MemoryLayout.self
    
    
    print(layout.size) // 16
    
    
    print(layout.alignment) // 4
    
    
    print(layout.stride) // 16
    1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6
    x y z isTracking
    1 word (64 bit CPUͳΒ8 byte) Ͱѻ͍΍͍͢Α͏ʹpadding͕ೖΔ
    1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6
    x y z
    isTracking
    pad
    struct Head: Equatable {
    
    
    let isTracking: Bool
    
    
    let x: Float
    
    
    let y: Float
    
    
    let z: Float
    
    
    }
    struct Head: Equatable {
    
    
    let x: Float
    
    
    let y: Float
    
    
    let z: Float
    
    
    let isTracking: Bool
    
    
    }
    

    View Slide

19. ©︎
    ҆શੑΛऔͬͯҰͭҰͭม׵͢Δ
    struct Head: Equatable {
    
    
    var x: Float
    
    
    var y: Float
    
    
    var z: Float
    
    
    var isTracking: Bool
    
    
    mutating func encodeToData() -> Data {
    
    
    return Data(bytesNoCopy: &x, count: MemoryLayout.size, deallocator: .none) +
    
    
    Data(bytesNoCopy: &y, count: MemoryLayout.size, deallocator: .none) +
    
    
    Data(bytesNoCopy: &z, count: MemoryLayout.size, deallocator: .none) +
    
    
    Data(bytesNoCopy: &isTracking, count: MemoryLayout.size, deallocator: .none)
    
    
    }
    
    
    static func decode(_ data: Data) -> Self {
    
    
    var offset = 0
    
    
    func nextValue(_ pointer: UnsafeRawPointer?, as: T.Type = T.self) -> T {
    
    
    defer {
    
    
    offset += MemoryLayout.size
    
    
    }
    
    
    return pointer!.advanced(by: offset).load(as: T.self)
    
    
    }
    
    
    return data.withUnsafeBytes {
    
    
    Head(
    
    
    x: nextValue($0.baseAddress),
    
    
    y: nextValue($0.baseAddress),
    
    
    z: nextValue($0.baseAddress),
    
    
    isTracking: nextValue($0.baseAddress)
    
    
    )
    
    
    }
    
    
    }
    
    
    }
    

    View Slide

20. ©︎
    Swift MacrosͰ҆શͳม׵Λࣗಈੜ੒͢Δ
    @DataSerialize
    
    
    struct Head: Equatable {
    
    
    let x: Float
    
    
    let y: Float
    
    
    let z: Float
    
    
    let isTracking: Bool
    
    
    }
    var head = Head(x: 0, y: 1, z: 2, isTracking: true)
    
    
    var encoded = head.encodeToData()
    
    
    var decoded = Head.decode(encoded)
    

    View Slide

21. ©︎
    Swift MacrosͰ҆શͳม׵Λࣗಈੜ੒͢Δ
    @attached(member, names: arbitrary)
    
    
    public macro DataSerialize() = #externalMacro(module: "SerializeMacros", type: "DataSerializeMacro")
    public struct DataSerializeMacro: MemberMacro {
    
    
    public static func expansion(
    
    
    of node: AttributeSyntax,
    
    
    providingMembersOf declaration: some DeclGroupSyntax,
    
    
    in context: some MacroExpansionContext
    
    
    ) throws -> [DeclSyntax] {
    
    
    let properties = declaration.memberBlock.members
    
    
    .compactMap { $0.decl.as(VariableDeclSyntax.self) }
    
    
    .map {
    
    
    (
    
    
    name: $0.bindings.first!.pattern.as(IdentifierPatternSyntax.self)!.identifier.text,
    
    
    type: $0.bindings.first!.typeAnnotation!.type.as(IdentifierTypeSyntax.self)!.trimmedDescription
    
    
    )
    
    
    }
    
    
    return [
    
    
    """
    
    
    static func decode(_ data: Data) -> Self {
    
    
    var offset = 0
    
    
    func nextValue(_ pointer: UnsafeRawPointer, as: T.Type = T.self) -> T {
    
    
    defer {
    
    
    offset += MemoryLayout.size
    
    
    }
    
    
    return pointer.advanced(by: offset).load(as: T.self)
    
    
    }
    
    
    return data.withUnsafeBytes {
    
    
    Self.init(
    
    
    \(raw: properties.map { name, type in
    
    
    """
    
    
    \(name): nextValue($0.baseAddress!)
    
    
    """
    
    
    }.joined(separator: ",\n"))
    
    
    )
    
    
    }
    
    
    }
    
    
    """,
    
    
    """
    
    
    mutating func encodeToData() -> Data {
    
    
    \(raw: properties.enumerated().map {
    
    
    let (index, (name, type)) = $0
    
    
    return """
    
    
    let d\(index) = Data(bytesNoCopy: &\(name), count: MemoryLayout<\(type)>.size, deallocator: .none)
    
    
    """
    
    
    }.joined(separator: "\n"))
    
    
    return \(raw: (0..
    
    }
    
    
    """
    
    
    ]
    
    
    }
    
    
    }
    ※ίʔυলུͷͨΊೖΕࢠඇରԠ
    😇
    ͕࣌ؒͳ͍ʂ
    

    View Slide

22. ©︎
    ύϑΥʔϚϯεൺֱ: Encode + Decode
    ϓϩύςΟ͝ͱʹϙΠϯλͰม׵T
    $PEBCMFͰม׵T
    ҰׅͰϙΠϯλͰม׵T
    -
    ഒ
    ରCodableൺ
    4XJGU.BDSPT
    
    60ϑϨʔϜ෼ͷॲཧʹ͔͔Δ࣌ؒ
    ૣ͍
    ૣ͍
    304
    35ഒ
    ˞վળͷ༨஍͸·ͩ·ͩ͋Γ
    

    View Slide

23. ©︎
    ύϑΥʔϚϯεൺֱ: Encode + Decode
    ϓϩύςΟ͝ͱʹϙΠϯλͰม׵T
    $PEBCMFͰม׵T
    ҰׅͰϙΠϯλͰม׵T
    -
    ഒ
    ରCodableൺ
    4XJGU.BDSPT
    
    60ϑϨʔϜ෼ͷॲཧʹ͔͔Δ࣌ؒ
    ૣ͍
    ૣ͍
    304
    35ഒ
    ˞վળͷ༨஍͸·ͩ·ͩ͋Γ
    

    View Slide