Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Add Reality to your App with ARCore and Sceneform

Add Reality to your App with ARCore and Sceneform

AR is a buzzword we hear everywhere. But is it really possible to build enhanced AR experiences quickly? There were quite a few attempts to bring AR to Android, but they were not easy to implement or required special hardware. ARCore solves both of the problems. It is a fast and performant SDK providing great API to build AR focused apps using just phone camera. During the talk we will discuss ARCore main concepts, geometry detection, 3D object rendering with Sceneform. I will also share potential problems and lessons learnt when building e-commerce AR experience at Jet.com.

Yuliya Kaleda

June 25, 2018
Tweet

More Decks by Yuliya Kaleda

Other Decks in Technology

Transcript

  1. Tango several cameras sensors high cost $ few devices (Lenovo

    Phab 2 Pro, Asus Zenfone) D E P R E C A T E D
  2. <manifest xmlns:android="http://schemas.android.com/apk/res/android" android:targetSandboxVersion="2" package=“com.jet.baselib"> <uses-feature android:name="android.hardware.camera.ar" android:required="true"/> <application android:name=“JetApp"> .

    . . <activity android:name=“.activities.SplashScreenActivity" /> <meta-data android:name="com.google.ar.core" android:value=“required" /> </application> </manifest> AR Required
  3. <manifest xmlns:android="http://schemas.android.com/apk/res/android" android:targetSandboxVersion="2" package="com.jet.baselib"> <application android:name=“JetApp"> . . . <activity

    android:name=“.activities.SplashScreenActivity" /> <meta-data android:name="com.google.ar.core" android:value="optional" /> </application> </manifest> AR Optional
  4. private void showArBanner() { ArCoreApk.Availability availability = ArCoreApk.getInstance() .checkAvailability(arCard.getContext()); if

    (availability.isTransient()) { new Handler().postDelayed(() -> showArBanner(), 200); } HelperViews.setVisibleOrGone(arCard, availability.isSupported()); } AR Optional
  5. private void showArBanner() { ArCoreApk.Availability availability = ArCoreApk.getInstance() .checkAvailability(arCard.getContext()); if

    (availability.isTransient()) { new Handler().postDelayed(() -> showArBanner(), 200); } HelperViews.setVisibleOrGone(arCard, availability.isSupported()); } AR Optional
  6. private boolean requestedARCoreInstall = true; public void onResume() { try

    { switch (ArCoreApk.getInstance().requestInstall(activity, requestedARCoreInstall)) { case INSTALLED: //Success break; case INSTALL_REQUESTED: //Request installation requestedARCoreInstall = false; return; } } catch (UnavailableUserDeclinedInstallationException e) { message = "Please install or update ARCore"; } catch (Exception e) { message = "This device does not support AR"; } } ARCore installation
  7. private boolean requestedARCoreInstall = true; public void onResume() { try

    { switch (ArCoreApk.getInstance().requestInstall(activity, requestedARCoreInstall)) { case INSTALLED: //Success break; case INSTALL_REQUESTED: //Request installation requestedARCoreInstall = false; return; } } catch (UnavailableUserDeclinedInstallationException e) { message = "Please install or update ARCore"; } catch (Exception e) { message = "This device does not support AR"; } } ARCore installation
  8. public void onResume() { if (session == null) { try

    { switch (ArCoreApk.getInstance().requestInstall(activity, requestedARCoreInstall)) { case INSTALLED: session = new Session(activity); break; case INSTALL_REQUESTED: requestedARCoreInstall = false; return; } } catch (UnavailableUserDeclinedInstallationException e) { ... } Config config = new Config(session); session.configure(config); } session.resume(); } Session
  9. Session Config light estimation mode (AMBIENT_INTENSITY, DISABLED) plane finding mode

    (DISABLED, HORIZONTAL, VERTICAL, HORIZONTAL_AND_VERTICAL) update mode (BLOCKING, LATEST_CAMERA_IMAGE) cloud anchor mode (DISABLED, ENABLED)
  10. public void onResume() { if (session == null) { try

    { switch (ArCoreApk.getInstance().requestInstall(activity, requestedARCoreInstall)) { case INSTALLED: session = new Session(activity); break; case INSTALL_REQUESTED: requestedARCoreInstall = false; return; } } catch (UnavailableUserDeclinedInstallationException e) { ... } // Create default config Config config = new Config(session); session.configure(config); } session.resume(); } Session Config
  11. @Override public void onResume() { if (session == null) {

    return; } Frame frame = session.update(); } Frame
  12. @Override public void onResume() { if (session == null) {

    return; } Frame frame = session.update(); Camera camera = frame.getCamera(); } Frame
  13. @Override public void onResume() { if (session == null) {

    return; } Frame frame = session.update(); Camera camera = frame.getCamera(); float lightIntensity = frame.getLightEstimate().getPixelIntensity(); } Frame
  14. @Override public void onResume() { if (session == null) {

    return; } Frame frame = session.update(); Camera camera = frame.getCamera(); float lightIntensity = frame.getLightEstimate().getPixelIntensity(); List<HitResult> hitResults = frame.hitTest(position.x, position.y); //List<HitResult> hitResults = frame.hitTest(motionEvent); } Frame
  15. @Override public void onResume() { if (session == null) {

    return; } Frame frame = session.update(); Camera camera = frame.getCamera(); float lightIntensity = frame.getLightEstimate().getPixelIntensity(); //List<HitResult> hitResults = frame.hitTest(position.x, position.y); List<HitResult> hitResults = frame.hitTest(motionEvent); } Frame
  16. @Override public void onResume() { . . . List<HitResult> hitResults

    = frame.hitTest(position.x, position.y); if (hitResults.size() > 0) { HitResult hit = getClosestHit(hitResults); } } private HitResult getClosestHit(List<HitResult> hitResults) { for (HitResult hitResult : hitResults) { if (hitResult.getTrackable() instanceof Plane) { return hitResult; } } return hitResults.get(0); } HitResult
  17. @Override public void onResume() { . . . List<HitResult> hitResults

    = frame.hitTest(position.x, position.y); if (hitResults.size() > 0) { HitResult hit = getClosestHit(hitResults); Anchor anchor = hit.createAnchor(); } } Anchor
  18. public class ArActivity extends JetActivity { private ArSceneView arSceneView; @Override

    protected void onCreate(Bundle savedInstanceState) { arSceneView = findViewById(R.id.sceneview); } @Override protected void onResume() { if (session != null) { Config config = new Config(session); config.setUpdateMode(Config.UpdateMode.LATEST_CAMERA_IMAGE); session.configure(config); arSceneView.setupSession(session); try { arSceneView.resume(); } catch (CameraNotAvailableException ex) { } } } } UI setup
  19. 3D rendering geometry texture o GoogleHome_GEO v 9.16292e-006 0.00498037 -0.0290344

    v -0.00769833 0.00176132 -0.0292111 v -0.0148647 0.00176132 -0.0261927 v -0.0210186 0.00176132 -0.0213912 v -0.0257407 0.00176132 -0.0151337 v -0.0287091 0.00176132 -0.00784677 v -0.0297216 0.00176132 -2.6893e-005 v -0.0287091 0.00176132 0.00779299 v -0.0257407 0.00176132 0.01508 v -0.0210186 0.00176132 0.0213374 v -0.0148647 0.00176132 0.0261389 v -0.00769833 0.00176132 0.0291573 v -7.85162e-006 0.00176132 0.0301868 v 0.00768262 0.00176132 0.0291573 v 0.014849 0.00176132 0.0261389 v 0.0210029 0.00176132 0.0213374 v 0.025725 0.00176132 0.01508 v 0.0286934 0.00176132 0.00779299 … .obj .png
  20. .sfa file { materials: [ { name: "diffuseTex", parameters: [

    { baseColor: "googlehome", }, { baseColorTint: [ 0.47999999999999998, 0.47999999999999998, 0.47999999999999998, 1, ], }, { metallic: 1, }, { roughness: 0.088044999999999998, }, { opacity: null, }, ], source: "build/sceneform_sdk/default_materials/obj_material.sfm", }, ], model: { attributes: [ "Position", "TexCoord", "Orientation", ], file: "../app/sampledata/googlehome.obj", name: "googlehome", suggested_collision: { center: { x: 0, y: 0, z: 0, }, size: { x: 1, y: 1, z: 1, }, type: "Box", }, }, samplers: [ { file: "../app/sampledata/googlehome.png", name: "googlehome", pipeline_name: "googlehome.png", }, ], version: "0.51:1", }
  21. Sceneform Plugin apply plugin: 'com.android.application' android { } dependencies {

    } apply plugin: 'com.google.ar.sceneform.plugin' sceneform.asset('../app/sampledata/googlehome.obj', 'default', '../app/sampledata/googlehome.sfa', 'src/main/assets/googlehome') :createAsset-src_47main_47assets_47googlehome => .sfa :compileAsset-src_47main_47assets_47googlehome => .sfb
  22. private ModelRenderable googleHomeRenderable; public void drawProduct(MotionEvent event) { ModelRenderable.builder() .setSource(this,

    Uri.parse(“googlehome.sfb")) //R.raw.googlehome .build() .thenAccept(renderable -> googleHomeRenderable = renderable) .exceptionally( throwable -> { Log.e(TAG, "Unable to load Renderable.", throwable); return null; }); } Finally draw
  23. private ModelRenderable googleHomeRenderable; public void drawProduct(MotionEvent event) { . .

    . Frame frame = arSceneView.getArFrame(); if (frame.getCamera().getTrackingState() != TrackingState.TRACKING) return; List<HitResult> hitResults = frame.hitTest(event); if (hitResults.size() > 0) { HitResult hit = getClosestHit(hitResults); Anchor anchor = hit.createAnchor(); } } Finally draw
  24. private ModelRenderable googleHomeRenderable; public void drawProduct(MotionEvent event) { . .

    . Frame frame = arSceneView.getArFrame(); if (frame.getCamera().getTrackingState() != TrackingState.TRACKING) return; List<HitResult> hitResults = frame.hitTest(event); if (hitResults.size() > 0) { HitResult hit = getClosestHit(hitResults); Anchor anchor = hit.createAnchor(); AnchorNode anchorNode = new AnchorNode(anchor); anchorNode.setParent(arSceneView.getScene()); } } Finally draw
  25. private ModelRenderable googleHomeRenderable; public void drawProduct(MotionEvent event) { . .

    . Frame frame = arSceneView.getArFrame(); if (frame.getCamera().getTrackingState() != TrackingState.TRACKING) return; List<HitResult> hitResults = frame.hitTest(event); if (hitResults.size() > 0) { HitResult hit = getClosestHit(hitResults); Anchor anchor = hit.createAnchor(); AnchorNode anchorNode = new AnchorNode(anchor); anchorNode.setParent(arSceneView.getScene()); Node googleHomeNode = new Node(); googleHomeNode.setRenderable(googleHomeRenderable); anchorNode.addChild(googleHomeNode); } } Finally draw
  26. Performance don’t rely on world coordinates, use anchors > 12

    anchors don’t use vibration detect a plane if available, else closest feature point 30 fps or higher lock orientation
  27. private void moveActiveObject(Session session, Frame frame, MotionEvent event) { List<HitResult>

    hitResults = frame.hitTest(event); Anchor newAnchor = null; if (hitResults.size() > 0) { HitResult hit = hitResults.get(0); newAnchor = hit.createAnchor(); } if (newAnchor != null) { Pose cameraPose = frame.getCamera().getPose(); Pose anchorPose = newAnchor.getPose(); float distance = getDistanceBetweenPoses(cameraPose, anchorPose); if (distance > OBJ_MAX_DISTANCE) return; activeArObject.getAnchor().detach(); activeArObject.setAnchor(newAnchor); } } Distance
  28. if (anchor.getTrackingState() != TrackingState.TRACKING) { //Do not draw objects }

    if (camera.getTrackingState() == TrackingState.PAUSED) { return; } Tracking State TRACKING PAUSED STOPPED