Upgrade to Pro
— share decks privately, control downloads, hide ads and more …
Speaker Deck
Features
Speaker Deck
PRO
Sign in
Sign up for free
Search
Search
Firebase ML Kit for iOS Developer
Search
Kajornsak Peerapathananont
October 07, 2018
Technology
0
62
Firebase ML Kit for iOS Developer
Firebase Dev Day 2018 @Bangkok, Thailand
Kajornsak Peerapathananont
October 07, 2018
Tweet
Share
More Decks by Kajornsak Peerapathananont
See All by Kajornsak Peerapathananont
Understanding your Android build
kajornsakp
0
30
iOSDevTH #21
kajornsakp
0
37
What's new in Flutter (Google I/O Extended Bangkok 22)
kajornsakp
0
52
Mobile Design System at scale
kajornsakp
0
77
What's new in Flutter 2020
kajornsakp
0
56
Mobile Machine Learning for All Skill Levels
kajornsakp
0
28
What's new in Flutter 1.9
kajornsakp
0
47
Kotlin meets Web
kajornsakp
0
19
From design to develop with Material Components
kajornsakp
0
130
Other Decks in Technology
See All in Technology
AIエージェント開発のノウハウと課題
pharma_x_tech
7
4.1k
Change Managerを活用して本番環境へのセキュアなGUIアクセスを統制する / Control Secure GUI Access to the Production Environment with Change Manager
yuj1osm
0
110
事業を差別化する技術を生み出す技術
pyama86
2
380
生成AI “再”入門 2025年春@WIRED TUESDAY EDITOR'S LOUNGE
kajikent
0
150
ディスプレイ広告(Yahoo!広告・LINE広告)におけるバックエンド開発
lycorptech_jp
PRO
0
480
プルリクエストレビューを終わらせるためのチーム体制 / The Team for Completing Pull Request Reviews
nekonenene
1
160
Aurora PostgreSQLがCloudWatch Logsに 出力するログの課金を削減してみる #jawsdays2025
non97
1
230
Snowflake ML モデルを dbt データパイプラインに組み込む
estie
0
110
LINE NEWSにおけるバックエンド開発
lycorptech_jp
PRO
0
320
手を動かしてレベルアップしよう!
maruto
0
240
Two Blades, One Journey: Engineering While Managing
ohbarye
4
2.2k
自分だけの仮想クラスタを高速かつ効率的に作る kubefork
donkomura
0
110
Featured
See All Featured
Dealing with People You Can't Stand - Big Design 2015
cassininazir
366
25k
Build The Right Thing And Hit Your Dates
maggiecrowley
34
2.5k
We Have a Design System, Now What?
morganepeng
51
7.4k
[RailsConf 2023] Rails as a piece of cake
palkan
53
5.3k
KATA
mclloyd
29
14k
Building a Scalable Design System with Sketch
lauravandoore
461
33k
How to Ace a Technical Interview
jacobian
276
23k
Helping Users Find Their Own Way: Creating Modern Search Experiences
danielanewman
29
2.5k
Building Flexible Design Systems
yeseniaperezcruz
328
38k
Unsuck your backbone
ammeep
669
57k
Intergalactic Javascript Robots from Outer Space
tanoku
270
27k
Faster Mobile Websites
deanohume
306
31k
Transcript
ML Kit for iOS developers Kajornsak Peerapathananont Agoda
Machine Learning
#FirebaseDevDay
Google Lens
Smart Reply
On-device Machine Learning
#FirebaseDevDay Doable, but hard.
#FirebaseDevDay
#FirebaseDevDay Get Image Image Classification Transform Interpret Get Result
#FirebaseDevDay Transform unsigned char *sourceBaseAddr = (unsigned char *)(CVPixelBufferGetBaseAddress(pixelBuffer)); int
image_height; unsigned char *sourceStartAddr; if (fullHeight <= image_width) { image_height = fullHeight; sourceStartAddr = sourceBaseAddr; } else { image_height = image_width; const int marginY = ((fullHeight - image_width) / 2); sourceStartAddr = (sourceBaseAddr + (marginY * sourceRowBytes)); } const int image_channels = 4; assert(image_channels >= wanted_input_channels); tensorflow::Tensor image_tensor( tensorflow::DT_FLOAT, tensorflow::TensorShape( {1, wanted_input_height, wanted_input_width, wanted_input_channels})); auto image_tensor_mapped = image_tensor.tensor<float, 4>(); tensorflow::uint8 *in = sourceStartAddr; float *out = image_tensor_mapped.data(); for (int y = 0; y < wanted_input_height; ++y) { float *out_row = out + (y * wanted_input_width * wanted_input_channels); for (int x = 0; x < wanted_input_width; ++x) { const int in_x = (y * image_width) / wanted_input_width; const int in_y = (x * image_height) / wanted_input_height; tensorflow::uint8 *in_pixel = in + (in_y * image_width * image_channels) + (in_x * image_channels); float *out_pixel = out_row + (x * wanted_input_channels); for (int c = 0; c < wanted_input_channels; ++c) { out_pixel[c] = (in_pixel[c] - input_mean) / input_std; } } }
#FirebaseDevDay Interpret if (tf_session.get()) { std::vector<tensorflow::Tensor> outputs; tensorflow::Status run_status =
tf_session->Run( {{input_layer_name, image_tensor}}, {output_layer_name}, {}, &outputs); if (!run_status.ok()) { LOG(ERROR) << "Running model failed:" << run_status; } else { tensorflow::Tensor *output = &outputs[0]; auto predictions = output->flat<float>(); NSMutableDictionary *newValues = [NSMutableDictionary dictionary]; for (int index = 0; index < predictions.size(); index += 1) { const float predictionValue = predictions(index); if (predictionValue > 0.05f) { std::string label = labels[index % predictions.size()]; NSString *labelObject = [NSString stringWithUTF8String:label.c_str()]; NSNumber *valueObject = [NSNumber numberWithFloat:predictionValue]; [newValues setObject:valueObject forKey:labelObject]; } } dispatch_async(dispatch_get_main_queue(), ^(void) { [self setPredictionValues:newValues]; }); } }
None
#FirebaseDevDay
#FirebaseDevDay Real-world Common Use Cases
#FirebaseDevDay FIRVisionImage | VisionImage NS_SWIFT_NAME(VisionImage) @interface FIRVisionImage : NSObject @property(nonatomic,
nullable) FIRVisionImageMetadata *metadata; - (instancetype)initWithImage:(UIImage *)image NS_DESIGNATED_INITIALIZER; - (instancetype)initWithBuffer:(CMSampleBufferRef)sampleBuffer NS_DESIGNATED_INITIALIZER; - (instancetype)init NS_UNAVAILABLE; @end
Text Recognition - On-device - On-cloud
#FirebaseDevDay https://firebase.google.com/docs/ml-kit/recognize-text
#FirebaseDevDay FIRVisionText | VisionText NS_SWIFT_NAME(VisionText) @interface FIRVisionText : NSObject @property(nonatomic,
readonly) NSString *text; @property(nonatomic, readonly) NSArray<FIRVisionTextBlock *> *blocks; - (instancetype)init NS_UNAVAILABLE; @end
#FirebaseDevDay On-device Usage let textRecognizer = vision.onDeviceTextRecognizer() textRecognizer.process(visionImage) { (text,
error) in guard let text = text else { return } // do something with your text }
#FirebaseDevDay On-cloud Usage let textRecognizer = vision.cloudTextRecognize() textRecognizer.process(visionImage) { (text,
error) in guard let text = text else { return } // do something with your text }
Image Labeling - On-device (400+ labels) - On-cloud (10,000+ labels)
#FirebaseDevDay https://firebase.google.com/docs/ml-kit/label-images
#FirebaseDevDay FIRVisionLabel | VisionLabel NS_SWIFT_NAME(VisionLabel) @interface FIRVisionLabel : NSObject @property(nonatomic,
readonly) CGRect frame; @property(nonatomic, readonly) float confidence; @property(nonatomic, copy, readonly) NSString *entityID; @property(nonatomic, copy, readonly) NSString *label; @end
#FirebaseDevDay On-device Usage let labelDetector = vision.labelDetector() labelDetector.detect(in: visionImage) {
(labels, error) in guard let error == nill, let labels = labels, !labels.isEmpty else { return } // do something with your labels }
#FirebaseDevDay On-cloud Usage let labelDetector = vision.cloudLabelDetector() labelDetector.detect(in: visionImage) {
(labels, error) in guard let error == nill, let labels = labels, !labels.isEmpty else { return } // do something with your labels }
Face detection - On-device
#FirebaseDevDay
#FirebaseDevDay FIRVisionFace | VisionFace NS_SWIFT_NAME(VisionFace) @interface FIRVisionFace : NSObject @property(nonatomic,
readonly) CGRect frame; @property(nonatomic, readonly) BOOL hasTrackingID; @property(nonatomic, readonly) NSInteger trackingID; @property(nonatomic, readonly) BOOL hasHeadEulerAngleY; @property(nonatomic, readonly) CGFloat headEulerAngleY; @property(nonatomic, readonly) BOOL hasHeadEulerAngleZ; @property(nonatomic, readonly) CGFloat headEulerAngleZ; @property(nonatomic, readonly) BOOL hasSmilingProbability; @property(nonatomic, readonly) CGFloat smilingProbability; @property(nonatomic, readonly) BOOL hasLeftEyeOpenProbability; @property(nonatomic, readonly) CGFloat leftEyeOpenProbability; @property(nonatomic, readonly) BOOL hasRightEyeOpenProbability; @property(nonatomic, readonly) CGFloat rightEyeOpenProbability; - (instancetype)init NS_UNAVAILABLE; - (nullable FIRVisionFaceLandmark *)landmarkOfType:(FIRFaceLandmarkType)type; #ifdef ENABLE_FACE_CONTOUR - (nullable FIRVisionFaceContour *)contourOfType:(FIRFaceContourType)type; #endif // ENABLE_FACE_CONTOUR @end
#FirebaseDevDay On-device Usage let faceDetector = vision.faceDetector() faceDetector.detect(in: visionImage) {
(faces, error) in guard let error == nill, let faces = faces, !faces.isEmpty else { return } // do something with your faces }
#FirebaseDevDay Face Contour?
Landmark recognition - On-cloud
#FirebaseDevDay
#FirebaseDevDay FIRVisionCloudLandmark | VisionCloudLandmark NS_SWIFT_NAME(VisionCloudLandmark) @interface FIRVisionCloudLandmark : NSObject @property(nonatomic,
copy, readonly, nullable) NSString *entityId; @property(nonatomic, copy, readonly, nullable) NSString *landmark; @property(nonatomic, readonly, nullable) NSNumber *confidence; @property(nonatomic, readonly) CGRect frame; @property(nonatomic, readonly, nullable) NSArray<FIRVisionLatitudeLongitude *> *locations; - (instancetype)init NS_UNAVAILABLE; @end
#FirebaseDevDay On-cloud Usage let landmarkDetector = vision.cloudLandmarkDetector() landmarkDetector.detect(in: visionImage) {
(landmarks, error) in guard let error == nill, let landmarks = landmarks, !landmarks.isEmpty else { return } // do something with your landmarks }
Barcode scanning - On-device
#FirebaseDevDay https://firebase.google.com/docs/ml-kit/label-images
#FirebaseDevDay FIRVisionBarcode | VisionBarcode NS_SWIFT_NAME(VisionBarcode) @interface FIRVisionBarcode : NSObject @property(nonatomic,
readonly) CGRect frame; @property(nonatomic, readonly, nullable) NSString *rawValue; @property(nonatomic, readonly, nullable) NSString *displayValue; @property(nonatomic, readonly) FIRVisionBarcodeFormat format; @property(nonatomic, readonly, nullable) NSArray<NSValue *> *cornerPoints; @property(nonatomic, readonly) FIRVisionBarcodeValueType valueType; @property(nonatomic, readonly, nullable) FIRVisionBarcodeEmail *email; @property(nonatomic, readonly, nullable) FIRVisionBarcodePhone *phone; @property(nonatomic, readonly, nullable) FIRVisionBarcodeSMS *sms; @property(nonatomic, readonly, nullable) FIRVisionBarcodeURLBookmark *URL; @property(nonatomic, readonly, nullable) FIRVisionBarcodeWiFi *wifi; @property(nonatomic, readonly, nullable) FIRVisionBarcodeGeoPoint *geoPoint; @property(nonatomic, readonly, nullable) FIRVisionBarcodeContactInfo *contactInfo; @property(nonatomic, readonly, nullable) FIRVisionBarcodeCalendarEvent *calendarEvent; @property(nonatomic, readonly, nullable) FIRVisionBarcodeDriverLicense *driverLicense; - (instancetype)init NS_UNAVAILABLE; @end
#FirebaseDevDay FIRVisionBarcodeCalendarEvent | VisionBarcodeCalendarEvent NS_SWIFT_NAME(VisionBarcodeCalendarEvent) @interface FIRVisionBarcodeCalendarEvent : NSObject @property(nonatomic,
readonly, nullable) NSString *eventDescription; @property(nonatomic, readonly, nullable) NSString *location; @property(nonatomic, readonly, nullable) NSString *organizer; @property(nonatomic, readonly, nullable) NSString *status; @property(nonatomic, readonly, nullable) NSString *summary; @property(nonatomic, readonly, nullable) NSDate *start; @property(nonatomic, readonly, nullable) NSDate *end; - (instancetype)init NS_UNAVAILABLE; @end
#FirebaseDevDay On-device Usage let barcodeDetector = vision.barcodeDetector() barcodeDetector.detect(in: visionImage) {
(barcodes, error) in guard let error == nill, let barcodes = barcodes, !barcodes.isEmpty else { return } // do something with your barcodes }
Custom model - Tensorflow Lite
#FirebaseDevDay let conditions = ModelDownloadConditions(isWiFiRequired: true, canDownloadInBackground: true) let cloudModelSource
= CloudModelSource( modelName: "my_cloud_model", enableModelUpdates: true, initialConditions: conditions, updateConditions: conditions ) let registrationSuccessful = ModelManager.modelManager().register(cloudModelSource)
Demo
Thank You! #FirebaseDevDay Helpful resources fb.com/FirebaseThailand fb.com/groups/FirebaseDevTH medium.com/FirebaseThailand Kajornsak Peerapathananont