Machine Learning On Mobile With MLKit

Machine Learning On
Mobile With MLKit
Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018

View Slide

Can't we already do that?

View Slide

https://www.tensorﬂow.org/lite/

View Slide


View Slide

TensorFlow Lite API Reference

View Slide

class TensorFlowLiteDemo(modelFile: File, labels: Array) {
private val labelProbArray: Array = arrayOf(FloatArray(labels.size))
private val tflite: Interpreter = Interpreter(modelFile)
fun classifyImage(bitmap: Bitmap) {
val imageData = convertBitmapToByteBuffer(bitmap)
tflite.run(imageData, labelProbArray)
}
}

View Slide

Developer
Experience

View Slide


View Slide

On-device & Cloud

View Slide

Ofﬂine/On-device Machine Learning
→ Reduced accuracy
→ Larger downloads
→ Increased resource usage

View Slide

MLKit on iOS - Setup
pod 'Firebase/Core'
pod 'Firebase/MLVision'
# If using an on-device API:
pod 'Firebase/MLVisionTextModel'

View Slide

Text recognition (iOS)
let vision = Vision.vision()
let textRecognizer = vision.onDeviceTextRecognizer()
let image = VisionImage(image: uiImage)
textRecognizer.process(visionImage) { result, error in
guard error == nil, let result = result else {
// ...
return
}
// Recognized text
}

View Slide

let resultText = result.text
for block in result.blocks {
let blockText = block.text
let blockConfidence = block.confidence
let blockLanguages = block.recognizedLanguages
let blockCornerPoints = block.cornerPoints
let blockFrame = block.frame
for line in block.lines {
let lineText = line.text
let lineConfidence = line.confidence
let lineLanguages = line.recognizedLanguages
let lineCornerPoints = line.cornerPoints
let lineFrame = line.frame
for element in line.elements {
let elementText = element.text
let elementConfidence = element.confidence
let elementLanguages = element.recognizedLanguages
let elementCornerPoints = element.cornerPoints
let elementFrame = element.frame
}
}
}

View Slide

MLKit on Android - Setup
dependencies {
// ...
implementation 'com.google.firebase:firebase-ml-vision:18.0.1'
implementation 'com.google.firebase:firebase-ml-vision-image-label-model:17.0.2'
implementation 'com.google.firebase:firebase-ml-vision-face-model:17.0.2'
}

...
android:name="com.google.firebase.ml.vision.DEPENDENCIES"
android:value="ocr" />


View Slide

Text recognition (Android)
val image = FirebaseVisionImage.fromBitmap(bitmap)
val textRecognizer = FirebaseVision.instance.getOnDeviceTextRecognizer();
textRecognizer.processImage(image)
.addOnSuccessListener({ result: FirebaseVisionText ->
/* Task completed successfully */
})
.addOnFailureListener({ error: Exception ->
/* Task failed with an exception */
})

View Slide

val resultText = result.getText
result.getTextBlocks.forEach {block ->
val blockText = block.text
val blockConfidence = block.confidence
val blockLanguages = block.recognizedLanguages
val blockCornerPoints = block.cornerPoints
val blockFrame = block.boundingBox
block.lines.forEach { line ->
val lineText = line.text
val lineConfidence = line.confidence
val lineLanguages = line.recognizedLanguages
val lineCornerPoints = line.cornerPoints
val lineFrame = line.boundingBox
line.elements.forEach {element ->
val elementText = element.text
val elementConfidence = element.confidence
val elementLanguages = element.recognizedLanguages
val elementCornerPoints = element.cornerPoints()
val elementFrame = element.boundingBox
}
}
}

View Slide

Dealing with camera rotation
object ORIENTATIONS: SparseIntArray() {
init {
append(Surface.ROTATION_0, 90)
}
}

View Slide

private fun calculateRotationCompensation(cameraId: String, activity: Activity): Int {
val deviceRotation = activity.windowManager.defaultDisplay.rotation
var rotationCompensation = ORIENTATIONS.get(deviceRotation)
val cameraManager = activity.getSystemService(CAMERA_SERVICE) as CameraManager
val sensorOrientation = cameraManager
.getCameraCharacteristics(cameraId)
.get(CameraCharacteristics.SENSOR_ORIENTATION)!!
rotationCompensation = (rotationCompensation + sensorOrientation + 270) % 360
// Return the corresponding FirebaseVisionImageMetadata rotation value.
return when (rotationCompensation) {
0 -> FirebaseVisionImageMetadata.ROTATION_0
else -> {
FirebaseVisionImageMetadata.ROTATION_0
}
}
}

View Slide

Face Detection

View Slide

Face Detection (Android)
// High-accuracy landmark detection and face classification
fun setupFacedetection(): FirebaseVisionFaceDetector {
val options = FirebaseVisionFaceDetectorOptions.Builder()
.setPerformanceMode(FirebaseVisionFaceDetectorOptions.ACCURATE)
.setLandmarkMode(FirebaseVisionFaceDetectorOptions.ALL_LANDMARKS)
.setClassificationMode(FirebaseVisionFaceDetectorOptions.ALL_CLASSIFICATIONS)
.build()
return FirebaseVision.getInstance().getVisionFaceDetector(options)
}

View Slide

fun detectFaces(detector: FirebaseVisionFaceDetector, image: Bitmap) {
val visionImage = FirebaseVisionImage.fromBitmap(image)
detector.detectInImage(visionImage)
.addOnSuccessListener { faces ->
for (face in faces) {
// Faces detected
}
}
.addOnFailureListener {
// TODO Error handling
}
}

View Slide

val bounds = face.boundingBox
val rotY = face.headEulerAngleY // Head is rotated to the right rotY degrees
val rotZ = face.headEulerAngleZ // Head is tilted sideways rotZ degrees
// If landmark detection was enabled (mouth, ears, eyes, cheeks, and
// nose available):
val leftEar = face.getLandmark(FirebaseVisionFaceLandmark.LEFT_EAR)
if (leftEar != null) {
val leftEarPos = leftEar.position
}

View Slide

// If contour detection was enabled:
val leftEyeContour = face.getContour(FirebaseVisionFaceContour.LEFT_EYE).points
val upperLipBottomContour = face.getContour(FirebaseVisionFaceContour.UPPER_LIP_BOTTOM).points

View Slide

// If classification was enabled:
if (face.smilingProbability != FirebaseVisionFace.UNCOMPUTED_PROBABILITY) {
val smileProb = face.smilingProbability
}
if (face.rightEyeOpenProbability != FirebaseVisionFace.UNCOMPUTED_PROBABILITY) {
val rightEyeOpenProb = face.rightEyeOpenProbability
}
// If face tracking was enabled:
if (face.trackingId != FirebaseVisionFace.INVALID_ID) {
val id = face.trackingId
}

View Slide

Cloud Image Labeling (Android)
fun createCloudLabelDetector(): FirebaseVisionCloudLabelDetector? {
val options = FirebaseVisionCloudDetectorOptions.Builder()
.setModelType(FirebaseVisionCloudDetectorOptions.LATEST_MODEL)
.setMaxResults(15)
.build()
return FirebaseVision.getInstance().getVisionCloudLabelDetector(options)
}

View Slide

fun labelImage(detector: FirebaseVisionCloudLabelDetector, bitmap: Bitmap) {
val image = FirebaseVisionImage.fromBitmap(bitmap)
detector.detectInImage(image)
.addOnSuccessListener { result ->
result.forEach { label ->
val text = label.label
val entityId = label.entityId
val confidence = label.confidence
}
}
// Error handling
}
}

View Slide

Custom TensoreFlow Lite Models

View Slide


View Slide

Determine input and output sizes
import tensorflow as tf
interpreter = tf.contrib.lite.Interpreter(model_path="my_custom_model.tflite")
interpreter.allocate_tensors()
# Print input shape and type
print(interpreter.get_input_details()[0]['shape']) # Example: [1 224 224 3]
print(interpreter.get_input_details()[0]['dtype']) # Example:
# Print output shape and type
print(interpreter.get_output_details()[0]['shape']) # Example: [1 1000]
print(interpreter.get_output_details()[0]['dtype']) # Example:

View Slide

fun configureCustomModel(modelName: String): FirebaseModelInterpreter? {
val conditions = FirebaseModelDownloadConditions.Builder()
.requireWifi()
.build()
val cloudSource = FirebaseCloudModelSource.Builder(modelName)
.enableModelUpdates(true)
.setInitialDownloadConditions(conditions)
.setUpdatesDownloadConditions(conditions)
.build()
FirebaseModelManager.getInstance().registerCloudModelSource(cloudSource)
val options = FirebaseModelOptions.Builder()
.setCloudModelName(modelName)
.setLocalModelName(modelName)
.build()
return FirebaseModelInterpreter.getInstance(options)
}

View Slide

fun runInterpreter(input: Any, firebaseInterpreter: FirebaseModelInterpreter) {
val inputs = FirebaseModelInputs.Builder()
.add(input) // add() as many input arrays as your model requires
.build()
val inputOutputOptions = FirebaseModelInputOutputOptions.Builder()
.setInputFormat(/* input format defined here */)
.setOutputFormat(/* output format defined here */)
.build()
firebaseInterpreter.run(inputs, inputOutputOptions)
.addOnSuccessListener {
// Interpret results
}
// Error handling
}
}

View Slide

Resources
→ Firebase MLKit - https://firebase.google.com/
docs/ml-kit/
→ TensorFlow Lite - https://www.tensorflow.org/lite
→ TensorFlow models - https://github.com/
tensorflow/models

View Slide

Machine Learning On Mobile With MLKit

Machine Learning On Mobile With MLKit

Erik Hellman

More Decks by Erik Hellman

Other Decks in Programming

Featured

Transcript