Upgrade to PRO for Only $50/Year—Limited-Time Offer! 🔥

Bringing Machine Learning in Android with Media...

Bringing Machine Learning in Android with MediaPipe - DroidJam 2023

Ahmad Arif Faizin

October 07, 2023
Tweet

More Decks by Ahmad Arif Faizin

Other Decks in Technology

Transcript

  1. DroidJam 2023 65% of consumers have trust in the businesses

    which use AI technology - Forbes Advisor
  2. • Lower latency & close knit interactions • Offline availability

    • Privacy preserving • Cost savings On-Device
  3. Model is at the core of an on-device ML solution

    Model Inference Output Input ML app • Customized to the ad-hoc use cases • Light-weight and efficient • Target to hardware • Sparsified for best performance
  4. ML Pipeline streamlines the process from raw inputs to output

    results Model Inference Output Live Camera ML app Flow Control Post-processing Synchronization Data Preprocessing • Domain-specific processing (e.g. vision / NLP / audio) • E2E acceleration across CPU / GPU / EdgeTPU / DSP • Cross-platform deployment to Android / iOS, web, baremetal
  5. Display Live Camera ML app Buffer Management Format Conversion Image

    Filtering Data Subsampling Data Lifecycle Timestamp Extraction Timestamp Alignment Thread Management GPU/CPU Data Transfer Multi-threaded GPU Compute iOS Metal OpenGL ES Trace Collection Performance Profiling C++ Programming Resource Caching Asset Loading GPU Timing Measurement Cross-platform Abstraction Data Marshalling CPU Affinity Java Native Interface Model Inference Flow Control Post-processing Synchronization Data Preprocessing Both involves a lot of complexity that hinders fast development
  6. MediaPipe abstracts this complexity into MediaPipe Tasks Model Inference Display

    MediaPipe Tasks Live Camera ML app Flow Control Post-processing Synchronization Data Preprocessing
  7. … while meeting your custom modeling needs with MediaPipe Model

    Maker Model Inference Display MediaPipe Model Maker Live Camera ML app Flow Control Post-processing Synchronization Data Preprocessing Custom model
  8. DroidJam 2023 • Create App to get image from Gallery

    or Camera • Alternative solution ◦ Gallery ▪ PhotoPicker ActivityResultContracts.PickVisualMedia() ▪ Intent ACTION_GET_CONTENT ▪ Intent ACTION_PICK ◦ Camera ▪ Intent ACTION_IMAGE_CAPTURE ▪ ActivityResultContracts.TakePicture() ▪ CameraX Starter Project
  9. DroidJam 2023 // 1. Setup Image Classifier (ImageClassifierHelper.kt) val baseOptionsBuilder

    = BaseOptions.builder() .setDelegate(Delegate.GPU) // CPU, GPU .setModelAssetPath(MODEL_PATH) val optionsBuilder = ImageClassifier.ImageClassifierOptions.builder() .setScoreThreshold(0.1f) // minimum 10% .setMaxResults(3) .setRunningMode(RunningMode.IMAGE) .setBaseOptions(baseOptionsBuilder.build()) val options = optionsBuilder.build() val imageClassifier = ImageClassifier.createFromOptions(context, options)
  10. DroidJam 2023 // 2. Create instance of ImageClassifierHelper // Get

    Data from Camera (in Activity) // Convert Uri to Bitmap imageUri?.let { uri -> if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { val source = ImageDecoder.createSource(contentResolver, imageUri) ImageDecoder.decodeBitmap(source) } else { MediaStore.Images.Media.getBitmap(contentResolver, uri) }.copy(Bitmap.Config.ARGB_8888, true)?.let { bitmap -> imageClassifierHelper.classifyImage(bitmap) } }
  11. DroidJam 2023 // 3. Convert the input Bitmap object to

    an MPImage object to run inference // ImageClassifierHelper.kt fun classifyImage(bitmap: Bitmap) { val mpImage: MPImage = BitmapImageBuilder(bitmap).build() val imageProcessingOptions = ImageProcessingOptions.builder().build() val startTime = SystemClock.uptimeMillis() imageClassifier?.classify(mpImage, imageProcessingOptions).also { result -> val inferenceTime = SystemClock.uptimeMillis() - startTime imageClassifierListener?.onResults(result, inferenceTime) } if (imageClassifier == null) { imageClassifierListener?.onError( "Image classifier failed to classify." ) } }
  12. // Output [Classifications {categories= [ <Category "computer keyboard" (displayName= score=0.41453125

    index=621)>, <Category "laptop" (displayName= score=0.35921875 index=509)> ], headIndex=0, headName=Optional[probability] } ] Note: It’s only works in real devices, not in Emulator
  13. DroidJam 2023 val cameraProviderFuture = ProcessCameraProvider.getInstance(this) cameraProviderFuture.addListener({ val cameraProvider =

    cameraProviderFuture.get() val preview = Preview.Builder() .setTargetAspectRatio(AspectRatio.RATIO_4_3) .build() .also { it.setSurfaceProvider(binding.viewFinder.surfaceProvider) } cameraProvider.unbindAll() cameraProvider.bindToLifecycle( this, CameraSelector.DEFAULT_BACK_CAMERA, preview ) }, ContextCompat.getMainExecutor(this)) • Request Permission CAMERA • CameraX implementation Starter Project
  14. DroidJam 2023 // 1. Setup Options Configuration (ImageClassifierHelper.kt) val optionsBuilder

    = ImageClassifier.ImageClassifierOptions.builder() .setScoreThreshold(0.1f) .setMaxResults(3) .setRunningMode(RunningMode.LIVE_STREAM) // IMAGE, VIDEO, LIVE_STREAM .setBaseOptions(baseOptionsBuilder.build()) if (runningMode == RunningMode.LIVE_STREAM) { optionsBuilder.setResultListener(this::returnLivestreamResult) optionsBuilder.setErrorListener(this::returnLivestreamError) }
  15. DroidJam 2023 // 2. Setup ImageAnalysis for CameraX (in Activity)

    val imageAnalyzer = ImageAnalysis.Builder() .setTargetAspectRatio(AspectRatio.RATIO_4_3) .setTargetRotation(binding.viewFinder.display.rotation) .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888) .build() .also { it.setAnalyzer(Executors.newSingleThreadExecutor()) { image -> imageClassifierHelper.classifyLiveStreamFrame(image) } } ... cameraProvider.bindToLifecycle( this, CameraSelector.DEFAULT_BACK_CAMERA, preview, imageAnalyzer )
  16. DroidJam 2023 // 3. Convert the input Bitmap object to

    an MPImage object to run inference fun classifyLiveStreamFrame(image: ImageProxy) { ... val mpImage = BitmapImageBuilder(bitmapBuffer).build() // Used for rotating the frame image so it matches our models val imageProcessingOptions = ImageProcessingOptions.builder() .setRotationDegrees(image.imageInfo.rotationDegrees) .build() val frameTime = SystemClock.uptimeMillis() // Run inference imageClassifier?.classifyAsync(mpImage, imageProcessingOptions, frameTime) }
  17. DroidJam 2023 // 1. Setup Object Detector (ObjectDetectorHelper.kt) val baseOptionsBuilder

    = BaseOptions.builder() .setDelegate(Delegate.GPU) // CPU, GPU .setModelAssetPath(MODEL_PATH) val optionsBuilder = ObjectDetector.ObjectDetectorOptions.builder() .setScoreThreshold(0.1f) .setMaxResults(3) .setRunningMode(RunningMode.LIVE_STREAM) .setBaseOptions(baseOptionsBuilder.build()) val options = optionsBuilder.build() val imageClassifier = ImageClassifier.createFromOptions(context, options)
  18. DroidJam 2023 // 2. Setup ImageAnalysis for CameraX (in Activity)

    val imageAnalyzer = ImageAnalysis.Builder() .setTargetAspectRatio(AspectRatio.RATIO_16_9) // adjust with model .setTargetRotation(binding.viewFinder.display.rotation) .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888) .build() .also { it.setAnalyzer(Executors.newSingleThreadExecutor()) { image -> objectDetectorHelper.detectLivestreamFrame(image) } }
  19. DroidJam 2023 // 3. Convert the input Bitmap object to

    an MPImage object to run inference fun detectLiveStreamFrame(image: ImageProxy) { ... val mpImage = BitmapImageBuilder(bitmapBuffer).build() // Used for rotating the frame image so it matches our models val imageProcessingOptions = ImageProcessingOptions.builder() .setRotationDegrees(image.imageInfo.rotationDegrees) .build() val frameTime = SystemClock.uptimeMillis() // Run inference objectDetector?.detectAsync(mpImage, imageProcessingOptions, frameTime) }
  20. DroidJam 2023 // 4a. Draw Box Using Custom View in

    XML class OverlayView(context: Context?, attrs: AttributeSet?) : View(context, attrs) { override fun draw(canvas: Canvas) { super.draw(canvas) // Draw bounding box around detected objects val drawableRect = RectF(left, top, right, bottom) canvas.drawRect(drawableRect, boxPaint) // Draw text for detected object canvas.drawText( drawableText, left, top + bounds.height(), textPaint ) } } }
  21. DroidJam 2023 // 4b. Create Box Using Composable @Composable fun

    ResultsOverlay(...) { val detections = results.detections() if (detections != null) { for (detection in detections) { ... Box( modifier = Modifier .border(3.dp, Turquoise) .width(boxWidth.dp) .height(boxHeight.dp) ) Box(modifier = Modifier.padding(3.dp)) { Text( text = resultText, modifier = Modifier .background(Color.Black) .padding(5.dp, 0.dp), color = Color.White, ) } } }
  22. DroidJam 2023 References • MediaPipe Documentation • Introducing MediaPipe for

    On-Device Machine Learning • Introduction to ML on Android with MediaPipe • Easy on-device Machine Learning with MediaPipe • ML Kit: Turnkey APIs to use on-device ML in mobile apps | Session • What's new in Machine Learning for Google Developers