$30 off During Our Annual Pro Sale. View Details »

iPhoneのセンサー情報をmacOSアプリでリアルタイム活用するための技術

 iPhoneのセンサー情報をmacOSアプリでリアルタイム活用するための技術

iOSDC Japan 2023 #iosdc

iPhoneのセンサー情報をmacOSアプリでリアルタイム活用するための技術
https://fortee.jp/iosdc-japan-2023/proposal/00d3250d-c32f-4466-b582-ecd9c27f5266

アバターアプリのGitHub:
https://github.com/vcamapp/app

Twitter: http://twitter.com/tanakasan2525/

Tatsuya Tanaka

September 03, 2023
Tweet

More Decks by Tatsuya Tanaka

Other Decks in Programming

Transcript

  1. ©︎
    iPhoneͷηϯαʔ৘ใΛ

    macOSΞ
    プ
    Ϧ
    で
    ϦΞϧλΠϜ׆༻͢ΔͨΊͷٕज़
    ͨͳͨͭ / Tatsuya Tanaka
    iOSDC Japan 2023 #iosdc

    View Slide

  2. ©︎
    ͨͳͨͭ / Tatsuya Tanaka
    💎μΠϠϞϯυ💎εϙϯαʔͷϠϑʔגࣜձࣾ


    ɾiOSΞϓϦࠇଳ
    @tattn
    @tanakasan2525

    View Slide

  3. ©︎
    ͨͳͨͭ / Tatsuya Tanaka
    @tattn
    @tanakasan2525
    💎μΠϠϞϯυ💎εϙϯαʔͷϠϑʔגࣜձࣾ


    ɾiOSΞϓϦࠇଳ

    View Slide

  4. ©︎
    ͨͳͨͭ / Tatsuya Tanaka
    • 💎μΠϠϞϯυ💎εϙϯαʔͷϠϑʔגࣜձࣾ


    • iOSΞϓϦࠇଳ
    @tattn
    @tanakasan2525

    View Slide

  5. ©︎
    ͨͳͨͭ / Tatsuya Tanaka
    ⭐ืूதˣˣ

    View Slide

  6. ©︎
    macOSͰ͸ARKit͕࢖͑ͳ͍
    VNFaceLandmarks2D ARFaceAnchor.blendshapes
    Vision ARKit

    View Slide

  7. ©︎
    2022 Yahoo Japan Corporation All rights reserved.
    BlendShapes

    View Slide

  8. ©︎
    ୅දతͳiPhoneͱMacؒͷ௨৴ํ๏
    Core Bluetooth Multipeer Connectivity Network
    ɾWiFiͳ͠Ͱܨ͛Δ

    ɾ௿ిྗ


    ɾόοΫάϥ΢ϯυOK
    ɾѻ͍ʹ͍͘I/F


    ɾOSver౳ͷ࢓༷ࠩҟ
    ɾP2P


    ɾঢ়ଶΛಉظ͠΍͍͢
    ɾApple only
    ɾҰൠతͳ௨৴ํ๏


    ɾϞμϯͳI/F

    View Slide

  9. ©︎
    ϓϩτίϧ
    Network
    TCP UDP QUIC ΧελϜ

    View Slide

  10. ©︎
    Bonjour: θϩίϯϑΟΪϡϨʔγϣϯϓϩτίϧ
    Illustration by Stable Diffusion
    NWBrowser(


    for: .bonjour(type: "_awesomeapp._udp", domain: nil),


    using: parameters


    )
    let listener = try NWListener(using: parameters, on: port)


    listener.service = .init(type: "_awesomeapp._udp")
    αʔϏε໊ + ϓϩτίϧ + υϝΠϯ

    View Slide

  11. ©︎
    Dataܕ΁ͷม׵
    Data

    View Slide

  12. ©︎
    Dataܕ΁ͷม׵
    public struct VCamMotion: Equatable {


    public var version: UInt32


    public var head: Head


    public var hands: Hands


    public var blendShape: BlendShape




    public struct Head: Equatable {


    public var translation: SIMD3


    public var rotation: simd_quatf


    }




    public struct Hands: Equatable {


    public var right: Hand


    public var left: Hand


    }




    public struct Hand: Equatable {


    public var wrist: SIMD2


    public var thumbCMC: SIMD2


    public var littleMCP: SIMD2


    public var thumbTip: SIMD2


    public var indexTip: SIMD2


    public var middleTip: SIMD2


    public var ringTip: SIMD2


    public var littleTip: SIMD2


    }


    }
    public struct BlendShape: Equatable {


    public var lookAtPoint: SIMD2


    public var browDownLeft: Float


    public var browDownRight: Float


    public var browInnerUp: Float


    public var browOuterUpLeft: Float


    public var browOuterUpRight: Float


    public var cheekPuff: Float


    public var cheekSquintLeft: Float


    public var cheekSquintRight: Float


    public var eyeBlinkLeft: Float


    public var eyeBlinkRight: Float


    public var eyeLookDownLeft: Float


    public var eyeLookDownRight: Float


    public var eyeLookInLeft: Float


    public var eyeLookInRight: Float


    public var eyeLookOutLeft: Float


    public var eyeLookOutRight: Float


    public var eyeLookUpLeft: Float


    public var eyeLookUpRight: Float


    public var eyeSquintLeft: Float


    public var eyeSquintRight: Float


    public var eyeWideLeft: Float


    public var eyeWideRight: Float


    public var jawForward: Float


    public var jawLeft: Float


    public var jawOpen: Float


    public var jawRight: Float


    public var mouthClose: Float


    public var mouthDimpleLeft: Float


    public var mouthDimpleRight: Float


    public var mouthFrownLeft: Float


    public var mouthFrownRight: Float


    public var mouthFunnel: Float


    public var mouthLeft: Float


    public var mouthLowerDownLeft: Float


    public var mouthLowerDownRight: Float


    public var mouthPressLeft: Float


    public var mouthPressRight: Float


    public var mouthPucker: Float


    public var mouthRight: Float


    public var mouthRollLower: Float


    public var mouthRollUpper: Float


    public var mouthShrugLower: Float


    public var mouthShrugUpper: Float


    public var mouthSmileLeft: Float


    public var mouthSmileRight: Float


    public var mouthStretchLeft: Float


    public var mouthStretchRight: Float


    public var mouthUpperUpLeft: Float


    public var mouthUpperUpRight: Float


    public var noseSneerLeft: Float


    public var noseSneerRight: Float


    public var tongueOut: Float


    }
    Data
    17%
    ※iPhone 12ͷ৔߹
    JSONEncoderͷ৔߹
    (0.003 s/frame)
    ඵؒͷॲཧ࣌ؒͷ
    ͕࢖ΘΕͯ͠·͏

    View Slide

  13. ©︎
    Dataܕ΁ͷߴ଎ͳม׵ํ๏
    struct Head: Equatable {


    let x: Float


    let y: Float


    let z: Float


    }
    var head = Head(x: 0, y: 1, z: 2)


    var encoded = Data(bytesNoCopy: &head, count: MemoryLayout.size, deallocator: .none)


    let decoded = encoded.withUnsafeBytes { $0.load(as: Head.self) }


    XCTAssertEqual(head, decoded)
    // Dataܕ΁ͷม׵
    // ݩͷܕ΁ͷม׵
    όΠτྻͱͯͦ͠ͷ··ม׵

    View Slide

  14. ©︎
    ϓϩύςΟͷ࣋ͪํʹ஫ҙ͕ඞཁ
    struct Head: Equatable {


    let x: Float


    let y: Float


    let z: Float


    }
    var encoded = Data(bytesNoCopy: &head, count: MemoryLayout.size, deallocator: .none)


    let decoded = encoded.withUnsafeBytes { $0.load(as: Head.self) }


    XCTAssertEqual(head, decoded)

    View Slide

  15. ©︎
    ϓϩύςΟͷ࣋ͪํʹ஫ҙ͕ඞཁ
    struct Head: Equatable {


    let x: Float


    let y: Float


    let z: Float


    let isTracking: Bool


    }
    var head = Head(x: 0, y: 1, z: 2, isTracking: true)
    ※64bit CPU؀ڥ
    var encoded = Data(bytesNoCopy: &head, count: MemoryLayout.size, deallocator: .none)


    let decoded = encoded.withUnsafeBytes { $0.load(as: Head.self) }


    XCTAssertEqual(head, decoded)

    View Slide

  16. ©︎
    ϓϩύςΟͷ࣋ͪํʹ஫ҙ͕ඞཁ
    struct Head: Equatable {


    let x: Float


    let y: Float


    let z: Float


    let isTracking: Bool


    }
    var head = Head(x: 0, y: 1, z: 2, isTracking: true)
    ※64bit CPU؀ڥ
    var encoded = Data(bytesNoCopy: &head, count: MemoryLayout.size, deallocator: .none)


    let decoded = encoded.withUnsafeBytes { $0.load(as: Head.self) }


    XCTAssertEqual(head, decoded)
    struct Head: Equatable {


    let x: Float


    let y: Float


    let z: Float


    let isTracking: Bool


    }

    View Slide

  17. ©︎
    ϓϩύςΟͷ࣋ͪํʹ஫ҙ͕ඞཁ
    var head = Head(x: 0, y: 1, z: 2, isTracking: true)
    struct Head: Equatable {


    let isTracking: Bool


    let x: Float


    let y: Float


    let z: Float


    }
    var head = Head(isTracking: true, x: 0, y: 1, z: 2)
    ※64bit CPU؀ڥ
    var encoded = Data(bytesNoCopy: &head, count: MemoryLayout.size, deallocator: .none)


    let decoded = encoded.withUnsafeBytes { $0.load(as: Head.self) }


    XCTAssertEqual(head, decoded)
    struct Head: Equatable {


    let x: Float


    let y: Float


    let z: Float


    let isTracking: Bool


    }

    View Slide

  18. ©︎
    structͷMemoryLayout
    let layout = MemoryLayout.self


    print(layout.size) // 13


    print(layout.alignment) // 4


    print(layout.stride) // 16
    let layout = MemoryLayout.self


    print(layout.size) // 16


    print(layout.alignment) // 4


    print(layout.stride) // 16
    1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6
    x y z isTracking
    1 word (64 bit CPUͳΒ8 byte) Ͱѻ͍΍͍͢Α͏ʹpadding͕ೖΔ
    1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6
    x y z
    isTracking
    pad
    struct Head: Equatable {


    let isTracking: Bool


    let x: Float


    let y: Float


    let z: Float


    }
    struct Head: Equatable {


    let x: Float


    let y: Float


    let z: Float


    let isTracking: Bool


    }

    View Slide

  19. ©︎
    ҆શੑΛऔͬͯҰͭҰͭม׵͢Δ
    struct Head: Equatable {


    var x: Float


    var y: Float


    var z: Float


    var isTracking: Bool


    mutating func encodeToData() -> Data {


    return Data(bytesNoCopy: &x, count: MemoryLayout.size, deallocator: .none) +


    Data(bytesNoCopy: &y, count: MemoryLayout.size, deallocator: .none) +


    Data(bytesNoCopy: &z, count: MemoryLayout.size, deallocator: .none) +


    Data(bytesNoCopy: &isTracking, count: MemoryLayout.size, deallocator: .none)


    }


    static func decode(_ data: Data) -> Self {


    var offset = 0


    func nextValue(_ pointer: UnsafeRawPointer?, as: T.Type = T.self) -> T {


    defer {


    offset += MemoryLayout.size


    }


    return pointer!.advanced(by: offset).load(as: T.self)


    }


    return data.withUnsafeBytes {


    Head(


    x: nextValue($0.baseAddress),


    y: nextValue($0.baseAddress),


    z: nextValue($0.baseAddress),


    isTracking: nextValue($0.baseAddress)


    )


    }


    }


    }

    View Slide

  20. ©︎
    Swift MacrosͰ҆શͳม׵Λࣗಈੜ੒͢Δ
    @DataSerialize


    struct Head: Equatable {


    let x: Float


    let y: Float


    let z: Float


    let isTracking: Bool


    }
    var head = Head(x: 0, y: 1, z: 2, isTracking: true)


    var encoded = head.encodeToData()


    var decoded = Head.decode(encoded)

    View Slide

  21. ©︎
    Swift MacrosͰ҆શͳม׵Λࣗಈੜ੒͢Δ
    @attached(member, names: arbitrary)


    public macro DataSerialize() = #externalMacro(module: "SerializeMacros", type: "DataSerializeMacro")
    public struct DataSerializeMacro: MemberMacro {


    public static func expansion(


    of node: AttributeSyntax,


    providingMembersOf declaration: some DeclGroupSyntax,


    in context: some MacroExpansionContext


    ) throws -> [DeclSyntax] {


    let properties = declaration.memberBlock.members


    .compactMap { $0.decl.as(VariableDeclSyntax.self) }


    .map {


    (


    name: $0.bindings.first!.pattern.as(IdentifierPatternSyntax.self)!.identifier.text,


    type: $0.bindings.first!.typeAnnotation!.type.as(IdentifierTypeSyntax.self)!.trimmedDescription


    )


    }


    return [


    """


    static func decode(_ data: Data) -> Self {


    var offset = 0


    func nextValue(_ pointer: UnsafeRawPointer, as: T.Type = T.self) -> T {


    defer {


    offset += MemoryLayout.size


    }


    return pointer.advanced(by: offset).load(as: T.self)


    }


    return data.withUnsafeBytes {


    Self.init(


    \(raw: properties.map { name, type in


    """


    \(name): nextValue($0.baseAddress!)


    """


    }.joined(separator: ",\n"))


    )


    }


    }


    """,


    """


    mutating func encodeToData() -> Data {


    \(raw: properties.enumerated().map {


    let (index, (name, type)) = $0


    return """


    let d\(index) = Data(bytesNoCopy: &\(name), count: MemoryLayout<\(type)>.size, deallocator: .none)


    """


    }.joined(separator: "\n"))


    return \(raw: (0..

    }


    """


    ]


    }


    }
    ※ίʔυলུͷͨΊೖΕࢠඇରԠ
    😇
    ͕࣌ؒͳ͍ʂ

    View Slide

  22. ©︎
    ύϑΥʔϚϯεൺֱ: Encode + Decode
    ϓϩύςΟ͝ͱʹϙΠϯλͰม׵T
    $PEBCMFͰม׵T
    ҰׅͰϙΠϯλͰม׵T
    -

    ରCodableൺ
    4XJGU.BDSPT

    60ϑϨʔϜ෼ͷॲཧʹ͔͔Δ࣌ؒ
    ૣ͍
    ૣ͍
    304
    35ഒ
    ˞վળͷ༨஍͸·ͩ·ͩ͋Γ

    View Slide

  23. ©︎
    ύϑΥʔϚϯεൺֱ: Encode + Decode
    ϓϩύςΟ͝ͱʹϙΠϯλͰม׵T
    $PEBCMFͰม׵T
    ҰׅͰϙΠϯλͰม׵T
    -

    ରCodableൺ
    4XJGU.BDSPT

    60ϑϨʔϜ෼ͷॲཧʹ͔͔Δ࣌ؒ
    ૣ͍
    ૣ͍
    304
    35ഒ
    ˞վળͷ༨஍͸·ͩ·ͩ͋Γ

    View Slide