Upgrade to Pro — share decks privately, control downloads, hide ads and more …

ArCore Basics and Tools

ArCore Basics and Tools

## ARCore - Basics and Tools

In this talk, we will learn how to build an application with the ARCore Android SDK. We'll start with a quick look at physically based rendering, and how to find or create these 3D models.

Then we'll learn the basics of adding virtual objects and interfaces to our environment, using the Sceneform library.

Etienne Caron

April 24, 2019
Tweet

More Decks by Etienne Caron

Other Decks in Programming

Transcript

  1. class ArPopUpShopActivity :_AppCompatActivity()_{ override fun onCreate(savedInstanceState: Bundle?)_{ super.onCreate(savedInstanceState) //_If device

    is not supported, early return. if (!checkIsSupportedDeviceOrFinish()) return }a }b Activity ArPopUpShopActivity activity_main.xml ArFragment
  2. class ArPopUpShopActivity :_AppCompatActivity()_{ override fun onCreate(savedInstanceState: Bundle?)_{ super.onCreate(savedInstanceState) //_If device

    is not supported, early return. if (!checkIsSupportedDeviceOrFinish()) return }a }b fun_Activity.checkIsSupportedDeviceOrFinish():_Boolean_{ val_openGlVersionString_=_activityManager .deviceConfigurationInfo .glEsVersion if_(parseDouble(openGlVersionString) < MIN_OPENGL_VERSION)_{ Timber.e("Sceneform requires OpenGL ES 3.0 later") finish() return_false }1 return_true }2 Activity ArPopUpShopActivity activity_main.xml ArFragment
  3. class ArPopUpShopActivity :_AppCompatActivity()_{ override fun onCreate(savedInstanceState: Bundle?)_{ super.onCreate(savedInstanceState) //_If device

    is not supported, early return. if (!checkIsSupportedDeviceOrFinish()) return }a }b Activity ArPopUpShopActivity activity_main.xml ArFragment
  4. class ArPopUpShopActivity :_AppCompatActivity()_{ override fun onCreate(savedInstanceState: Bundle?)_{ super.onCreate(savedInstanceState) //_If device

    is not supported, early return. if (!checkIsSupportedDeviceOrFinish()) return //_`FrameLayout` holding an `ArFragment` setContentView(R.layout.activity_main) }a }b Activity ArPopUpShopActivity activity_main.xml ArFragment
  5. class ArPopUpShopActivity :_AppCompatActivity()_{ override fun onCreate(savedInstanceState: Bundle?)_{ super.onCreate(savedInstanceState) //_If device

    is not supported, early return. if (!checkIsSupportedDeviceOrFinish()) return //_`FrameLayout` holding a `ArCustomFragment` setContentView(R.layout.activity_main) }a }b <FrameLayout android:layout_width="match_parent" android:layout_height="match_parent"> <fragment android:name="com.google.ar.sceneform.ux.ArFragment" android:id="@+id/ux_fragment" android:layout_width="match_parent" android:layout_height=“match_parent"_/> </FrameLayout> Activity ArPopUpShopActivity activity_main.xml ArFragment
  6. class ArPopUpShopActivity :_AppCompatActivity()_{ override fun onCreate(savedInstanceState: Bundle?)_{ super.onCreate(savedInstanceState) //_If device

    is not supported, early return. if (!checkIsSupportedDeviceOrFinish()) return //_`FrameLayout` holding a `ArCustomFragment` setContentView(R.layout.activity_main) }a }b Activity ArPopUpShopActivity activity_main.xml ArFragment
  7. class ArPopUpShopActivity :_AppCompatActivity()_{ private lateinit var arFragment :_ArFragment override fun

    onCreate(savedInstanceState: Bundle?)_{ super.onCreate(savedInstanceState) //_If device is not supported, early return. if (!checkIsSupportedDeviceOrFinish()) return //_`FrameLayout` holding a `ArCustomFragment` setContentView(R.layout.activity_main) arFragment =_supportFragmentManager .findFragmentById(R.id.ux_fragment) as_ArFragment }a }b Activity ArPopUpShopActivity activity_main.xml ArFragment ArFragment
  8. # This file uses centimeters as units for non-parametric coordinates.

    # Blender v2.78 (sub 0) OBJ File: '' # www.blender.org mtllib andy.mtl g default v 0.036531 5.203676 -0.001768 v 0.035000 5.204560 -0.002500 v 0.033469 5.205443 -0.001768 v 0.032835 5.205810 -0.000000 v 0.033469 5.205443 0.001768 v 0.035000 5.204560 0.002500 v 0.036531 5.203676 0.001769 v 0.037165 5.203310 -0.000000 v 0.036951 5.204877 -0.001531 v 0.035625 5.205642 -0.002165 v 0.034299 5.206408 -0.001531 v 0.033750 5.206725 -0.000000 v 0.034299 5.206408 0.001531 v 0.035625 5.205642 0.002165 v 0.036951 5.204877 0.001531 v 0.037500 5.204560 -0.000000 v 0.036848 5.205993 -0.000884 v 0.036083 5.206435 -0.001250 v 0.035317 5.206877 -0.000884 v 0.035000 5.207060 -0.000000 v 0.035317 5.206877 0.000884 v 0.036083 5.206435 0.001250 v 0.036848 5.205993 0.000884 v 0.037165 5.205810 -0.000000
  9. newmtl unlit_material illum 2 Kd 0.00 0.00 0.00 Ka 0.00

    0.00 0.00 Tf 1.00 1.00 1.00 map_Kd andy.png Ni 1.00
  10. apply plugin: 'com.google.ar.sceneform.plugin' sceneform.asset('sampledata/andy/andy.obj', 'default', 'sampledata/andy/andy.sfa', 'src/main/res/raw/andy') sceneform.asset('sampledata/andyCustomMat/andy2.gltf', 'default', 'sampledata/andyCustomMat/andy2.sfa',

    'src/main/res/raw/andy2') sceneform.asset('sampledata/damagedHelmet/DamagedHelmet.gltf', 'sampledata/damagedHelmet/', 'sampledata/damagedHelmet/DamagedHelmet.sfa', 'src/main/res/raw/damaged_helmet')
  11. { materials: [ { name: 'unlit_material', parameters: [ { baseColor:

    'andy', }, { baseColorTint: [ 1, 1, 1, 1, ], }, { metallic: 1, }, { roughness: 1, }, { opacity: null, }, ], source: 'build/sceneform_sdk/default_materials/obj_material.sfm', }, ], model: { attributes: [ 'Position', 'TexCoord', 'Orientation', ], collision: {}, file: 'sampledata/andy/andy.obj', name: 'andy', recenter: 'root', }, samplers: [ { file: 'sampledata/andy/andy.png', name: 'andy', pipeline_name: 'andy.png', }, ], version: ‘0.54:2', } andy.sfa
  12. private ModelRenderable andyRenderable; ModelRenderable.builder() .setSource(this, R.raw.andy) .build() .thenAccept(renderable -> andyRenderable

    = renderable) .exceptionally( throwable -> { Toast toast = Toast.makeText(this, “Load error.”, Toast.LENGTH_LONG); toast.setGravity(Gravity.CENTER, 0, 0); toast.show(); return null; }); // Later on… // In tap listener, quits if not loaded yet. if (andyRenderable == null) { return; } // Otherwise handle the tap event. sceneform Node Renderable ViewRenderable ModelRenderable Builder.build():Future Builder.build():Future We'll cover this in a sec.
  13. private ModelRenderable andyRenderable; ModelRenderable.builder() .setSource(this, R.raw.andy) .build() .thenAccept(renderable -> andyRenderable

    = renderable) .exceptionally( throwable -> { Toast toast = Toast.makeText(this, “Load error.”, Toast.LENGTH_LONG); toast.setGravity(Gravity.CENTER, 0, 0); toast.show(); return null; }); // Later on… // In tap listener, quits if not loaded yet. if (andyRenderable == null) { return; } // Otherwise handle the tap event. sceneform Node Renderable ViewRenderable ModelRenderable Builder.build():Future Builder.build():Future We'll cover this in a sec.
  14. /** * Small utility function to convert `Future` * instances

    to RxJava friendly `Single` instances. */ private fun <T> Future<T>.toSingle() = Single.fromFuture(this) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) sceneform Node Renderable ViewRenderable ModelRenderable Builder.build():Future Builder.build():Future We'll cover this in a sec.
  15. /** * Small utility function to convert `Future` * instances

    to RxJava friendly `Single` instances. */ private fun <T> Future<T>.toSingle() = Single.fromFuture(this) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) /** * Utility function that returns a * Single<ModelRenderable> * * @param model Source model raw resource ID. * @return Single<ModelRenderable> */ fun Activity.singleModelRenderable(@RawRes model: Int) = ModelRenderable .builder() .setSource(this, model) .build() .toSingle() sceneform Node Renderable ViewRenderable ModelRenderable Builder.build():Future Builder.build():Future We'll cover this in a sec.
  16. class ArPopUpShopActivity :_AppCompatActivity()_{ override fun onCreate(savedInstanceState: Bundle?)_{ super.onCreate(savedInstanceState) //_If device

    is not supported, early return. if (!checkIsSupportedDeviceOrFinish()) return //_`FrameLayout` holding a `ArCustomFragment` setContentView(R.layout.activity_main) }a }b Activity ArPopUpShopActivity activity_main.xml ArFragment
  17. class ArPopUpShopActivity :_AppCompatActivity()_{ override fun onCreate(savedInstanceState: Bundle?)_{ super.onCreate(savedInstanceState) //_If device

    is not supported, early return. if (!checkIsSupportedDeviceOrFinish()) return //_`FrameLayout` holding a `ArCustomFragment` setContentView(R.layout.activity_main) // Load the Andy 3D model singleModelRenderable(R.raw.andy) }a }b Activity ArPopUpShopActivity activity_main.xml ArFragment
  18. class ArPopUpShopActivity :_AppCompatActivity()_{ override fun onCreate(savedInstanceState: Bundle?)_{ super.onCreate(savedInstanceState) //_If device

    is not supported, early return. if (!checkIsSupportedDeviceOrFinish()) return //_`FrameLayout` holding a `ArCustomFragment` setContentView(R.layout.activity_main) // Load the Andy 3D model singleModelRenderable(R.raw.andy) .subscribe_{_TODO(“Init tap handler”)_} }a }b Activity ArPopUpShopActivity activity_main.xml ArFragment
  19. fun initOnTapPlane(andyRenderable: ModelRenderable)_{ val tapListener =_OnTapArPlaneListener {_hitResult, plane, _ ->

    val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem) andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable andyNode.select() }a arFragment.setOnTapArPlaneListener(tapListener) }b Activity ArPopUpShopActivity activity_main.xml ArFragment
  20. fun initOnTapPlane(andyRenderable: ModelRenderable)_{ val tapListener =_OnTapArPlaneListener {_hitResult, plane, _ ->

    val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem) andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable andyNode.select() }a arFragment.setOnTapArPlaneListener(tapListener) }b sceneform BaseArFragment onPeekTouch(htr,me) setOnTapArPlaneListener(l) ArFragment Activity ArPopUpShopActivity activity_main.xml ArFragment
  21. sceneform BaseArFragment onPeekTouch(htr,me) setOnTapArPlaneListener(l) ArFragment fun initOnTapPlane(andyRenderable: ModelRenderable)_{ val tapListener

    =_OnTapArPlaneListener {_hitResult, plane, _ -> val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem) andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable andyNode.select() }a arFragment.setOnTapArPlaneListener(tapListener) }b Activity ArPopUpShopActivity activity_main.xml ArFragment
  22. sceneform BaseArFragment onPeekTouch(htr,me) setOnTapArPlaneListener(l) ArFragment fun initOnTapPlane(andyRenderable: ModelRenderable)_{ val tapListener

    =_OnTapArPlaneListener {_hitResult, plane, _ -> val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem) andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable andyNode.select() }a arFragment.setOnTapArPlaneListener(tapListener) }b ModelRenderable Builder.build():Future Activity ArPopUpShopActivity activity_main.xml ArFragment
  23. sceneform BaseArFragment onPeekTouch(htr,me) setOnTapArPlaneListener(l) ArFragment fun initOnTapPlane(andyRenderable: ModelRenderable)_{ val tapListener

    =_OnTapArPlaneListener {_hitResult, plane, _ -> val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem) andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable andyNode.select() }a arFragment.setOnTapArPlaneListener(tapListener) }b ModelRenderable Builder.build():Future Activity ArPopUpShopActivity activity_main.xml ArFragment r arcore Anchor cloudAnchorId pose detach() TrackingState PAUSED STOPPED TRACKING
  24. sceneform BaseArFragment onPeekTouch(htr,me) setOnTapArPlaneListener(l) ArFragment fun initOnTapPlane(andyRenderable: ModelRenderable)_{ val tapListener

    =_OnTapArPlaneListener {_hitResult, plane, _ -> val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem) andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable andyNode.select() }a arFragment.setOnTapArPlaneListener(tapListener) }b ModelRenderable Builder.build():Future Activity ArPopUpShopActivity activity_main.xml ArFragment r arcore Anchor cloudAnchorId pose detach() TrackingState PAUSED STOPPED TRACKING AnchorNode Node
  25. sceneform BaseArFragment onPeekTouch(htr,me) setOnTapArPlaneListener(l) ArFragment fun initOnTapPlane(andyRenderable: ModelRenderable)_{ val tapListener

    =_OnTapArPlaneListener {_hitResult, plane, _ -> val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem) andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable andyNode.select() }a arFragment.setOnTapArPlaneListener(tapListener) }b ModelRenderable Builder.build():Future Activity ArPopUpShopActivity activity_main.xml ArFragment r arcore Anchor cloudAnchorId pose detach() TrackingState PAUSED STOPPED TRACKING AnchorNode Node ArSceneView
  26. sceneform BaseArFragment onPeekTouch(htr,me) setOnTapArPlaneListener(l) ArFragment fun initOnTapPlane(andyRenderable: ModelRenderable)_{ val tapListener

    =_OnTapArPlaneListener {_hitResult, plane, _ -> val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem) andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable andyNode.select() }a arFragment.setOnTapArPlaneListener(tapListener) }b ModelRenderable Builder.build():Future Activity ArPopUpShopActivity activity_main.xml ArFragment r arcore Anchor cloudAnchorId pose detach() TrackingState PAUSED STOPPED TRACKING AnchorNode Node ArSceneView
  27. sceneform BaseArFragment onPeekTouch(htr,me) setOnTapArPlaneListener(l) ArFragment fun initOnTapPlane(andyRenderable: ModelRenderable)_{ val tapListener

    =_OnTapArPlaneListener {_hitResult, plane, _ -> val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem) andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable andyNode.select() }a arFragment.setOnTapArPlaneListener(tapListener) }b ModelRenderable Builder.build():Future Activity ArPopUpShopActivity activity_main.xml ArFragment Node ArSceneView AnchorNode TranformableNode
  28. sceneform BaseArFragment onPeekTouch(htr,me) setOnTapArPlaneListener(l) ArFragment fun initOnTapPlane(andyRenderable: ModelRenderable)_{ val tapListener

    =_OnTapArPlaneListener {_hitResult, plane, _ -> val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem) andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable andyNode.select() }a arFragment.setOnTapArPlaneListener(tapListener) }b ModelRenderable Builder.build():Future Activity ArPopUpShopActivity activity_main.xml ArFragment Node ArSceneView AnchorNode TranformableNode
  29. <?xml version="1.0" encoding="utf-8"?> <TextView xmlns:android="http://schemas.android.com/apk/res/android" android:id="@+id/priceLabel" android:layout_width="wrap_content" android:layout_height="wrap_content" android:background="@drawable/rounded_dark_grey_bg" android:gravity="center"

    android:orientation="vertical" android:padding="6dp" android:text="5$" android:textAlignment="center" /> sceneform Renderable ViewRenderable ModelRenderable Builder.build():Future Builder.build():Future
  30. singleViewRenderable(R.layout.price_tag) .subscribe { priceTagRenderable -> priceTagRenderable .view .findViewById<TextView>(R.id.priceLabel) .text =

    “$price$" } sceneform Renderable ViewRenderable ModelRenderable Builder.build():Future Builder.build():Future
  31. singleViewRenderable(R.layout.price_tag) .subscribe { priceTagRenderable -> priceTagRenderable .view .findViewById<TextView>(R.id.priceLabel) .text =

    “$price$" } sceneform Renderable ViewRenderable ModelRenderable Builder.build():Future Builder.build():Future
  32. singleViewRenderable(R.layout.price_tag) .subscribe { priceTagRenderable -> priceTagRenderable .view .findViewById<TextView>(R.id.priceLabel) .text =

    “$price$" } sceneform Renderable ViewRenderable ModelRenderable Builder.build():Future Builder.build():Future
  33. val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem)

    andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable val priceTagNode = Node() priceTag.isShadowCaster = false priceTag.sizer = FixedWidthViewSizer(0.1f) // 10cm max width priceTagNode.renderable = priceTag priceTagNode.localRotation = Quaternion.axisAngle(Vector3(1f, 0f, 0f), -90f) priceTagNode.localPosition = Vector3(0f, 0.02f, 0.2f) andyNode.addChild(priceTagNode)
  34. val anchor =_hitResult.createAnchor() val anchorNode =_AnchorNode(anchor) anchorNode.setParent(arFragment.arSceneView.scene) val andyNode =_TransformableNode(arFragment.transformationSystem)

    andyNode.setParent(anchorNode) andyNode.renderable =_andyRenderable val priceTagNode = Node() priceTag.isShadowCaster = false priceTag.sizer = FixedWidthViewSizer(0.1f) // 10cm max width priceTagNode.renderable = priceTag priceTagNode.localRotation = Quaternion.axisAngle(Vector3(1f, 0f, 0f), -90f) priceTagNode.localPosition = Vector3(0f, 0.02f, 0.2f) andyNode.addChild(priceTagNode)
  35. singleViewRenderable(R.layout.selected_card_view) .subscribe { selectedView -> selectedView.sizer = FixedWidthViewSizer(0.2f) selectedView.isShadowCaster =

    false val selectedNode = Node().apply { renderable = selectedView } view.findViewById<Button>(R.id.favoriteButton) .setOnClickListener { // Your code here... } }
  36. class ArCustomFragment : ArFragment() { /** * We initialize the

    returned configuration here. * * Loads up augmented image DB, assigns it to session config. * * @return the configuration that will be used by ArFragment. */ override fun getSessionConfiguration(session: Session): Config { return super.getSessionConfiguration(session).also { config -> config.augmentedImageDatabase = context ?.assets ?.open("good_images.imgdb") .let { inputStream -> AugmentedImageDatabase .deserialize(session, inputStream) } } } }
  37. val listener = Scene.OnUpdateListener { arSceneView.arFrame?.run { if( camera.trackingState ==

    TrackingState.TRACKING) { val trackables = getUpdatedTrackables( AugmentedImage::class.java ) if(trackables.isNotEmpty()) { // Handler code here... } } } } arSceneView.scene.addOnUpdateListener(listener)
  38. val listener = Scene.OnUpdateListener { arSceneView.arFrame?.run { if( camera.trackingState ==

    TrackingState.TRACKING) { val trackables = getUpdatedTrackables( AugmentedImage::class.java ) if(trackables.isNotEmpty()) { // Handler code here... } } } } arSceneView.scene.addOnUpdateListener(listener)
  39. // Init View Renderable viewRenderable.apply { isShadowCaster = false isShadowReceiver

    = false sizer = FixedWidthViewSizer(1f) } // Init an Augmented Image Anchor. augmentedImage.createAnchor(augmentedImage.centerPose) // Create AnchorNode with augmented image's anchor. val anchorNode = AnchorNode(augmentedImage.anchors.first()) // Assign Renderable to a node 40cm above and 1m behind. val storefrontNode = Node().apply { renderable = viewRenderable localPosition = Vector3(0f,0.4f,-1f) } // Assign store UX node to the anchor node. anchorNode.addChild(storefrontNode)
  40. { materials: [ { name: 'unlit_material', parameters: [ { andyColor:

    [ 1, 1, 0, ], }, ], source: 'sampledata/andy02/custom.mat', }, ], model: { attributes: [ 'Position', 'TexCoord', 'Orientation', ], collision: {}, file: 'sampledata/andy02/andy.obj', name: 'andy', recenter: 'root', }, }
  41. material { name: "Custom material", parameters: [ { type: "float3",

    name: "andyColor" } ], requires: [ "position" ], shadingModel: "lit", } fragment { void material(inout MaterialInputs material) { prepareMaterial(material); material.baseColor.rgb = materialParams.andyColor.rgb; material.metallic = 1.0; material.roughness = 0.5; } }