Get the flake out

Transcript

1. Get the flake out Sandeep Nair Senior Software Engineer Jarek Ankowski Software Engineer NOTE: Opinions and views on products, services and/or resources expressed in this presentation are ours alone and do not necessarily reflect the views of our employer.
2. Architecture Tools Tips & Tricks
3. Robot pattern
4. @Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled() } } @PaymentDetailsRobot class PaymentDetailsRobot : ScreenRobot() { fun verifyPaymentIsDisabled() { onView(withId(R.id.payment_button)) .checkMatches(not(isEnabled())) } fun inputAmount(amount: Int) { onView(withId(R.id.amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.tip)) .type(tip) } fun verifyPaymentIsEnabled() { onView(withId(R.id.payment_button)) .checkMatches(isEnabled()) } }
5. @Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled() } } WHAT HOW @PaymentDetailsRobot class PaymentDetailsRobot : ScreenRobot() { fun verifyPaymentIsDisabled() { onView(withId(R.id.payment_button)) .checkMatches(not(isEnabled())) } fun inputAmount(amount: Int) { onView(withId(R.id.amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.tip)) .type(tip) } fun verifyPaymentIsEnabled() { onView(withId(R.id.payment_button)) .checkMatches(isEnabled()) } }
6. @Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled() } }
7. @Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled() } }
8. @Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled() } } inline fun onPaymentDetailsScreen(f: PaymentDetailsRobot.() -> Unit) = PaymentDetailsRobot().apply { f() } class PaymentDetailsRobot : ScreenRobot() { fun verifyPaymentIsDisabled() { onView(withId(R.id.payment_button)) .checkMatches(not(isEnabled())) } fun inputAmount(amount: Int) { onView(withId(R.id.amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.tip)) .type(tip) } fun verifyPaymentIsEnabled() { onView(withId(R.id.payment_button)) .checkMatches(isEnabled()) } }
9. inline fun onPaymentDetailsScreen(f: PaymentDetailsRobot.() -> Unit) = PaymentDetailsRobot().apply { f() } class PaymentDetailsRobot : ScreenRobot() { fun verifyPaymentIsDisabled() { onView(withId(R.id.payment_button)) .checkMatches(not(isEnabled())) } fun inputAmount(amount: Int) { onView(withId(R.id.amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.tip)) .type(tip) } fun verifyPaymentIsEnabled() { onView(withId(R.id.payment_button)) .checkMatches(isEnabled()) } } @Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled() } }
10. @DslMarker annotation class PaymentDetailsRobot inline fun onPaymentDetailsScreen(f: PaymentDetailsRobot.() -> Unit) = PaymentDetailsRobot().apply { f() } @Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled() } } class PaymentDetailsRobot : ScreenRobot() { fun verifyPaymentIsDisabled() { onView(withId(R.id.payment_button)) .checkMatches(not(isEnabled())) } fun inputAmount(amount: Int) { onView(withId(R.id.amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.tip)) .type(tip) } fun verifyPaymentIsEnabled() { onView(withId(R.id.payment_button)) .checkMatches(isEnabled()) } } @PaymentDetailsRobot
11. Scaling
12. @Test fun paymentFlowTest() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) pay() } onPinDetailsScreen { enterPin() pay() } onPaymentSuccessScreen { verifyPaymentSuccess() exitScreen() } onHomeScreen { verifyHomeScreenIsDisplayed() } }
13. @Test fun paymentFlowTest() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) pay() } onPinDetailsScreen { enterPin() pay() } onPaymentSuccessScreen { verifyPaymentSuccess() exitScreen() } onHomeScreen { verifyHomeScreenIsDisplayed() } }
14. @DslMarker annotation class PaymentDetailsRobot inline fun onPaymentDetailsScreen(f: PaymentDetailsRobot.() -> Unit) = PaymentDetailsRobot().apply { f() } @PaymentDetailsRobot class PaymentDetailsRobot : ScreenRobot() { fun inputAmount(amount: Int) { onView(withId(R.id.payment_details_amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.payment_details_tip)) .type(tip) } fun pay() { onView(withId(R.id.payment_button)) .perform(click()) } } @Test fun paymentFlowTest() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) pay() } onPinDetailsScreen { enterPin() pay() } onPaymentSuccessScreen { verifyPaymentSuccess() exitScreen() } onHomeScreen { verifyHomeScreenIsDisplayed() } }
15. @Test fun paymentFlowTest() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) pay() } onPinDetailsScreen { enterPin() pay() } onPaymentSuccessScreen { verifyPaymentSuccess() exitScreen() } onHomeScreen { verifyHomeScreenIsDisplayed() } } @DslMarker annotation class PaymentDetailsRobot inline fun onPaymentDetailsScreen(f: PaymentDetailsRobot.() -> Unit) = PaymentDetailsRobot().apply { f() } @PaymentDetailsRobot class PaymentDetailsRobot : ScreenRobot() { fun inputAmount(amount: Int) { onView(withId(R.id.payment_details_amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.payment_details_tip)) .type(tip) } fun pay() { onView(withId(R.id.payment_button)) .perform(click()) } } infix fun onPinDetailsScreen(f: PinEntryScreenRobot.() -> Unit) = PinEntryScreenRobot().apply { f() }
16. Robot pattern • Separating the What from the How • Framework agnostic tests • Meaningful assertions and verifications • Reusable code • Descriptive stack traces
17. Architecture Tools Tips & Tricks
18. Wiremock • Wiremock server • Screen tests & flow tests • Testing complete use cases • Configurable responses
19. class LoginFlow( private val isSuccess: Boolean ) : WiremockFlow { override fun configure(wiremock: WiremockFlowRule) { wiremock.stubFor(post(urlEqualTo("/service/path")) .willReturn(aResponse().withBodyFile( when { isSuccess -> "successResponse.json" else -> “errorResponse.json" }))) } }
20. class LoginFlow( private val isSuccess: Boolean ) : WiremockFlow { override fun configure(wiremock: WiremockFlowRule) { wiremock.stubFor(post(urlEqualTo("/service/path")) .willReturn(aResponse().withBodyFile( when { isSuccess -> "successResponse.json" else -> “errorResponse.json" }))) } }
21. class LoginFlow( private val isSuccess: Boolean ) : WiremockFlow { override fun configure(wiremock: WiremockFlowRule) { wiremock.stubFor(post(urlEqualTo("/service/path")) .willReturn(aResponse().withBodyFile( when { isSuccess -> "successResponse.json" else -> “errorResponse.json" }))) } }
22. class LoginFlowTest { @Rule @JvmField val activityRule = ActivityTestRule(MainActivity::class.java) @Rule @JvmField val wiremock = WiremockFlowRule() @Test fun whenCorrectCredentialsUsed_thenLoggedInScreenIsDisplayed() { wiremock.add(LoginFlow(isSuccess = true)) activityRule.launchActivity() onLoginScreen { typeUsername("user") typePassword("password") login() } onHomeScreen { verifyUserLoggedIn() } } }
23. Response stub assets/stubs/<region>/<feature>/<api_response>.json assets/stubs/<region>/<feature>/<api_response_variation>.json { "statusCode": "SUCCESS", "loginResponse": { //... } } { "statusCode": "ERROR", "loginResponseMessage": "Unable to login" }
24. Test Orchestrator • Running tests in isolation • Isolating crashes • Enables parallel execution
25. Firebase Test Lab • Physical and virtual devices • All Espresso tests run for every PR • Debugging test failures
26. Flank • Parallel test runner for FTL • Optimised bucketing system • Saves money
27. gcloud: project: project-name record-video: true timeout: 7m async: false # test and app are the only required args app: ../test_app/apks/app-debug.apk test: ../test_app/apks/app-debug-androidTest.apk use-orchestrator: true device: - model: NexusLowRes version: 23 flank: # The amount of groups to split the test suite into testShards: 58 # The amount of times to run the tests. repeatTests: 1
28. gcloud: project: project-name record-video: true timeout: 7m async: false # test and app are the only required args app: ../test_app/apks/app-debug.apk test: ../test_app/apks/app-debug-androidTest.apk use-orchestrator: true device: - model: NexusLowRes version: 23 flank: # The amount of groups to split the test suite into testShards: 58 # The amount of times to run the tests. repeatTests: 1
29. gcloud: project: project-name record-video: true timeout: 7m async: false # test and app are the only required args app: ../test_app/apks/app-debug.apk test: ../test_app/apks/app-debug-androidTest.apk use-orchestrator: true device: - model: NexusLowRes version: 23 flank: # The amount of groups to split the test suite into testShards: 58 # The amount of times to run the tests. repeatTests: 1
30. gcloud: project: project-name record-video: true timeout: 7m async: false # test and app are the only required args app: ../test_app/apks/app-debug.apk test: ../test_app/apks/app-debug-androidTest.apk use-orchestrator: true device: - model: NexusLowRes version: 23 flank: # The amount of groups to split the test suite into testShards: 58 # The amount of times to run the tests. repeatTests: 1
31. gcloud: project: project-name record-video: true timeout: 7m async: false # test and app are the only required args app: ../test_app/apks/app-debug.apk test: ../test_app/apks/app-debug-androidTest.apk use-orchestrator: true device: - model: NexusLowRes version: 23 flank: # The amount of groups to split the test suite into testShards: 58 # The amount of times to run the tests. repeatTests: 1
32. Architecture Tools Tips & Tricks
33. Flakiness • Multi-threading • Animations • Android OS • API calls
34. Flake out • Case by case • Scroll • Do not rely on time • IdlingResource
35. IdlingResource • Custom animations, dialogs, WebViews • It does not speed up Espresso • Do not be busy longer than necessary
36. class IdlingResourceWrapper : IdlingResource { fun busy(): Boolean { // ... } fun idle(): Boolean { // ... } override fun registerIdleTransitionCallback(callback: ResourceCallback) { // ... } override fun getName(): String { // ... } override fun isIdleNow(): Boolean { // ... } }
37. class IdlingResourceWrapper : IdlingResource { fun busy(): Boolean { // ... } fun idle(): Boolean { // ... } override fun registerIdleTransitionCallback(callback: ResourceCallback) { // ... } override fun getName(): String { // ... } override fun isIdleNow(): Boolean { // ... } }
38. class IdlingResourceWrapper : IdlingResource { fun busy(): Boolean { // ... } fun idle(): Boolean { // ... } override fun registerIdleTransitionCallback(callback: ResourceCallback) { // ... } override fun getName(): String { // ... } override fun isIdleNow(): Boolean { // ... } }
39. class IdlingResourceWrapper : IdlingResource { fun busy(): Boolean { // ... } fun idle(): Boolean { // ... } override fun registerIdleTransitionCallback(callback: ResourceCallback) { // ... } // ... } class AppAndroidJunitRunner : AndroidJUnitRunner() { @Inject lateinit var idlingResourceWrapper: IdlingResourceWrapper override fun onCreate(arguments: Bundle) { IdlingRegistry.getInstance() .register(idlingResourceWrapper) } }
40. class WebViewClient( private val idleResourceWrapper: IdlingResourceWrapper ) : WebViewClient() { override fun onPageStarted(view: WebView, url: String, favicon: Bitmap) { idleResourceWrapper.busy() // ... } override fun onPageFinished(view: WebView, url: String) { idleResourceWrapper.idle() // ... } }
41. ReadWriteLock • Pause/resume mechanism fun pauseToCheck(lock: ReadWriteLock) { lock.writeLock().lock() // Do some checking lock.writeLock().unlock() }
42. ViewAction
43. fun customViewAction(viewAction: ViewAction): ViewAction { return object : ViewAction { override fun perform(uiController: UiController, view: View) { // ... viewAction.perform(uiController, view) // ... } override fun getConstraints(): Matcher<View> = viewAction.constraints override fun getDescription(): String = viewAction.description } }
44. fun customViewAction(viewAction: ViewAction): ViewAction { return object : ViewAction { override fun perform(uiController: UiController, view: View) { // ... viewAction.perform(uiController, view) // ... } } interface UiController { fun injectMotionEvent(event: MotionEvent): Boolean fun injectKeyEvent(event: KeyEvent): Boolean fun injectString(str: String): Boolean fun loopMainThreadUntilIdle() fun loopMainThreadForAtLeast(millisDelay: Long) }
45. TL;DR • Let robots take over your job • Flank your UI tests • Do not @Ignore flakiness
46. Sources • Jake Wharton - Testing Robots: https://academy.realm.io/posts/kau-jake-wharton-testing- robots/ • Flank https://github.com/TestArmada/flank
47. Thank you