Share
Embed
Get the flake out
Jarek
January 22, 2019
Transcript
47 slides
-
Get the flake out Sandeep Nair Senior Software Engineer Jarek
Ankowski Software Engineer NOTE: Opinions and views on products, services and/or resources expressed in this presentation are ours alone and do not necessarily reflect the views of our employer. -
@Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled()
} } @PaymentDetailsRobot class PaymentDetailsRobot : ScreenRobot() { fun verifyPaymentIsDisabled() { onView(withId(R.id.payment_button)) .checkMatches(not(isEnabled())) } fun inputAmount(amount: Int) { onView(withId(R.id.amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.tip)) .type(tip) } fun verifyPaymentIsEnabled() { onView(withId(R.id.payment_button)) .checkMatches(isEnabled()) } } -
@Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled()
} } WHAT HOW @PaymentDetailsRobot class PaymentDetailsRobot : ScreenRobot() { fun verifyPaymentIsDisabled() { onView(withId(R.id.payment_button)) .checkMatches(not(isEnabled())) } fun inputAmount(amount: Int) { onView(withId(R.id.amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.tip)) .type(tip) } fun verifyPaymentIsEnabled() { onView(withId(R.id.payment_button)) .checkMatches(isEnabled()) } } -
@Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled()
} } inline fun onPaymentDetailsScreen(f: PaymentDetailsRobot.() -> Unit) = PaymentDetailsRobot().apply { f() } class PaymentDetailsRobot : ScreenRobot() { fun verifyPaymentIsDisabled() { onView(withId(R.id.payment_button)) .checkMatches(not(isEnabled())) } fun inputAmount(amount: Int) { onView(withId(R.id.amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.tip)) .type(tip) } fun verifyPaymentIsEnabled() { onView(withId(R.id.payment_button)) .checkMatches(isEnabled()) } } -
inline fun onPaymentDetailsScreen(f: PaymentDetailsRobot.() -> Unit) = PaymentDetailsRobot().apply { f()
} class PaymentDetailsRobot : ScreenRobot() { fun verifyPaymentIsDisabled() { onView(withId(R.id.payment_button)) .checkMatches(not(isEnabled())) } fun inputAmount(amount: Int) { onView(withId(R.id.amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.tip)) .type(tip) } fun verifyPaymentIsEnabled() { onView(withId(R.id.payment_button)) .checkMatches(isEnabled()) } } @Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled() } } -
@DslMarker annotation class PaymentDetailsRobot inline fun onPaymentDetailsScreen(f: PaymentDetailsRobot.() -> Unit)
= PaymentDetailsRobot().apply { f() } @Test fun whenAmountAndTipProvided_enablePayment() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) verifyPaymentIsEnabled() } } class PaymentDetailsRobot : ScreenRobot() { fun verifyPaymentIsDisabled() { onView(withId(R.id.payment_button)) .checkMatches(not(isEnabled())) } fun inputAmount(amount: Int) { onView(withId(R.id.amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.tip)) .type(tip) } fun verifyPaymentIsEnabled() { onView(withId(R.id.payment_button)) .checkMatches(isEnabled()) } } @PaymentDetailsRobot -
@Test fun paymentFlowTest() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) pay()
} onPinDetailsScreen { enterPin() pay() } onPaymentSuccessScreen { verifyPaymentSuccess() exitScreen() } onHomeScreen { verifyHomeScreenIsDisplayed() } } -
@Test fun paymentFlowTest() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) pay()
} onPinDetailsScreen { enterPin() pay() } onPaymentSuccessScreen { verifyPaymentSuccess() exitScreen() } onHomeScreen { verifyHomeScreenIsDisplayed() } } -
@DslMarker annotation class PaymentDetailsRobot inline fun onPaymentDetailsScreen(f: PaymentDetailsRobot.() -> Unit)
= PaymentDetailsRobot().apply { f() } @PaymentDetailsRobot class PaymentDetailsRobot : ScreenRobot() { fun inputAmount(amount: Int) { onView(withId(R.id.payment_details_amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.payment_details_tip)) .type(tip) } fun pay() { onView(withId(R.id.payment_button)) .perform(click()) } } @Test fun paymentFlowTest() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) pay() } onPinDetailsScreen { enterPin() pay() } onPaymentSuccessScreen { verifyPaymentSuccess() exitScreen() } onHomeScreen { verifyHomeScreenIsDisplayed() } } -
@Test fun paymentFlowTest() { onPaymentDetailsScreen { verifyPaymentIsDisabled() inputAmount(213) inputTip(42) pay()
} onPinDetailsScreen { enterPin() pay() } onPaymentSuccessScreen { verifyPaymentSuccess() exitScreen() } onHomeScreen { verifyHomeScreenIsDisplayed() } } @DslMarker annotation class PaymentDetailsRobot inline fun onPaymentDetailsScreen(f: PaymentDetailsRobot.() -> Unit) = PaymentDetailsRobot().apply { f() } @PaymentDetailsRobot class PaymentDetailsRobot : ScreenRobot() { fun inputAmount(amount: Int) { onView(withId(R.id.payment_details_amount)) .type(amount) } fun inputTip(tip: Int) { onView(withId(R.id.payment_details_tip)) .type(tip) } fun pay() { onView(withId(R.id.payment_button)) .perform(click()) } } infix fun onPinDetailsScreen(f: PinEntryScreenRobot.() -> Unit) = PinEntryScreenRobot().apply { f() } -
Robot pattern • Separating the What from the How •
Framework agnostic tests • Meaningful assertions and verifications • Reusable code • Descriptive stack traces -
Wiremock • Wiremock server • Screen tests & flow tests
• Testing complete use cases • Configurable responses -
class LoginFlow( private val isSuccess: Boolean ) : WiremockFlow {
override fun configure(wiremock: WiremockFlowRule) { wiremock.stubFor(post(urlEqualTo("/service/path")) .willReturn(aResponse().withBodyFile( when { isSuccess -> "successResponse.json" else -> “errorResponse.json" }))) } } -
class LoginFlow( private val isSuccess: Boolean ) : WiremockFlow {
override fun configure(wiremock: WiremockFlowRule) { wiremock.stubFor(post(urlEqualTo("/service/path")) .willReturn(aResponse().withBodyFile( when { isSuccess -> "successResponse.json" else -> “errorResponse.json" }))) } } -
class LoginFlow( private val isSuccess: Boolean ) : WiremockFlow {
override fun configure(wiremock: WiremockFlowRule) { wiremock.stubFor(post(urlEqualTo("/service/path")) .willReturn(aResponse().withBodyFile( when { isSuccess -> "successResponse.json" else -> “errorResponse.json" }))) } } -
class LoginFlowTest { @Rule @JvmField val activityRule = ActivityTestRule(MainActivity::class.java) @Rule
@JvmField val wiremock = WiremockFlowRule() @Test fun whenCorrectCredentialsUsed_thenLoggedInScreenIsDisplayed() { wiremock.add(LoginFlow(isSuccess = true)) activityRule.launchActivity() onLoginScreen { typeUsername("user") typePassword("password") login() } onHomeScreen { verifyUserLoggedIn() } } } -
Response stub assets/stubs/<region>/<feature>/<api_response>.json assets/stubs/<region>/<feature>/<api_response_variation>.json { "statusCode": "SUCCESS", "loginResponse": { //...
} } { "statusCode": "ERROR", "loginResponseMessage": "Unable to login" } -
Test Orchestrator • Running tests in isolation • Isolating crashes
• Enables parallel execution -
Firebase Test Lab • Physical and virtual devices • All
Espresso tests run for every PR • Debugging test failures -
Flank • Parallel test runner for FTL • Optimised bucketing
system • Saves money -
gcloud: project: project-name record-video: true timeout: 7m async: false #
test and app are the only required args app: ../test_app/apks/app-debug.apk test: ../test_app/apks/app-debug-androidTest.apk use-orchestrator: true device: - model: NexusLowRes version: 23 flank: # The amount of groups to split the test suite into testShards: 58 # The amount of times to run the tests. repeatTests: 1 -
gcloud: project: project-name record-video: true timeout: 7m async: false #
test and app are the only required args app: ../test_app/apks/app-debug.apk test: ../test_app/apks/app-debug-androidTest.apk use-orchestrator: true device: - model: NexusLowRes version: 23 flank: # The amount of groups to split the test suite into testShards: 58 # The amount of times to run the tests. repeatTests: 1 -
gcloud: project: project-name record-video: true timeout: 7m async: false #
test and app are the only required args app: ../test_app/apks/app-debug.apk test: ../test_app/apks/app-debug-androidTest.apk use-orchestrator: true device: - model: NexusLowRes version: 23 flank: # The amount of groups to split the test suite into testShards: 58 # The amount of times to run the tests. repeatTests: 1 -
gcloud: project: project-name record-video: true timeout: 7m async: false #
test and app are the only required args app: ../test_app/apks/app-debug.apk test: ../test_app/apks/app-debug-androidTest.apk use-orchestrator: true device: - model: NexusLowRes version: 23 flank: # The amount of groups to split the test suite into testShards: 58 # The amount of times to run the tests. repeatTests: 1 -
gcloud: project: project-name record-video: true timeout: 7m async: false #
test and app are the only required args app: ../test_app/apks/app-debug.apk test: ../test_app/apks/app-debug-androidTest.apk use-orchestrator: true device: - model: NexusLowRes version: 23 flank: # The amount of groups to split the test suite into testShards: 58 # The amount of times to run the tests. repeatTests: 1 -
Flake out • Case by case • Scroll • Do
not rely on time • IdlingResource -
IdlingResource • Custom animations, dialogs, WebViews • It does not
speed up Espresso • Do not be busy longer than necessary -
class IdlingResourceWrapper : IdlingResource { fun busy(): Boolean { //
... } fun idle(): Boolean { // ... } override fun registerIdleTransitionCallback(callback: ResourceCallback) { // ... } override fun getName(): String { // ... } override fun isIdleNow(): Boolean { // ... } } -
class IdlingResourceWrapper : IdlingResource { fun busy(): Boolean { //
... } fun idle(): Boolean { // ... } override fun registerIdleTransitionCallback(callback: ResourceCallback) { // ... } override fun getName(): String { // ... } override fun isIdleNow(): Boolean { // ... } } -
class IdlingResourceWrapper : IdlingResource { fun busy(): Boolean { //
... } fun idle(): Boolean { // ... } override fun registerIdleTransitionCallback(callback: ResourceCallback) { // ... } override fun getName(): String { // ... } override fun isIdleNow(): Boolean { // ... } } -
class IdlingResourceWrapper : IdlingResource { fun busy(): Boolean { //
... } fun idle(): Boolean { // ... } override fun registerIdleTransitionCallback(callback: ResourceCallback) { // ... } // ... } class AppAndroidJunitRunner : AndroidJUnitRunner() { @Inject lateinit var idlingResourceWrapper: IdlingResourceWrapper override fun onCreate(arguments: Bundle) { IdlingRegistry.getInstance() .register(idlingResourceWrapper) } } -
class WebViewClient( private val idleResourceWrapper: IdlingResourceWrapper ) : WebViewClient() {
override fun onPageStarted(view: WebView, url: String, favicon: Bitmap) { idleResourceWrapper.busy() // ... } override fun onPageFinished(view: WebView, url: String) { idleResourceWrapper.idle() // ... } } -
ReadWriteLock • Pause/resume mechanism fun pauseToCheck(lock: ReadWriteLock) { lock.writeLock().lock() //
Do some checking lock.writeLock().unlock() } -
fun customViewAction(viewAction: ViewAction): ViewAction { return object : ViewAction {
override fun perform(uiController: UiController, view: View) { // ... viewAction.perform(uiController, view) // ... } override fun getConstraints(): Matcher<View> = viewAction.constraints override fun getDescription(): String = viewAction.description } } -
fun customViewAction(viewAction: ViewAction): ViewAction { return object : ViewAction {
override fun perform(uiController: UiController, view: View) { // ... viewAction.perform(uiController, view) // ... } } interface UiController { fun injectMotionEvent(event: MotionEvent): Boolean fun injectKeyEvent(event: KeyEvent): Boolean fun injectString(str: String): Boolean fun loopMainThreadUntilIdle() fun loopMainThreadForAtLeast(millisDelay: Long) } -
TL;DR • Let robots take over your job • Flank
your UI tests • Do not @Ignore flakiness -
Sources • Jake Wharton - Testing Robots: https://academy.realm.io/posts/kau-jake-wharton-testing- robots/ •
Flank https://github.com/TestArmada/flank