Upgrade to Pro — share decks privately, control downloads, hide ads and more …

State of Android Testing in 2022

State of Android Testing in 2022

Android’s testing story has come a long way since the early days (old timers might recall not being able to run unit tests on the local JVM, or struggling to write UI tests in the low on caffeine pre-Espresso world), and nowadays there’s a multitude of tools, both 1st and 3rd party, to help you assemble a world-class testing pipeline. In this talk we’ll go down the test pyramid and present an overview of tools and techniques to help you write efficient tests in each category. We’ll discuss technologies that are heavily used at (and some even built by) Cash App. The set of topics includes, but is not limited to:

- What kinds of tests should you focus on?
- Writing end-to-end tests with Espresso and running them on real devices in the Firebase Test Lab.
- Using Paparazzi to write screenshot tests and catch UI regressions.
- Turbine & other useful tools for testing Kotlin’s coroutines & Flows.

This talk should give developers a holistic overview and a wealth of practical advice on writing all types of tests for your Android app!

Egor Andreevich

October 07, 2022
Tweet

More Decks by Egor Andreevich

Other Decks in Programming

Transcript

  1. Agenda • Brief history of Android testing • The testing

    pyramid • Testing: what, why and how • Tools & techniques Photo by Girl with red hat on Unsplash
  2. Unit tests • Hermetic • Safety net • Fast feedback

    loop • API testbed Photo by Fikri Rasyid on Unsplash
  3. Po rt folioRepository interface PortfolioRepository { suspend fun getPortfolio(): Portfolio

    suspend fun filterPortfolio(filterTerm: String): Portfolio } internal class RealPortfolioRepository( private val portfolioApi: PortfolioApi, portfolioDb: Database, ) : PortfolioRepository { private val stockQueries = portfolioDb.stockQueries override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain() } catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain()
  4. interface PortfolioRepository { suspend fun getPortfolio(): Portfolio suspend fun filterPortfolio(filterTerm:

    String): Portfolio } internal class RealPortfolioRepository( private val portfolioApi: PortfolioApi, portfolioDb: Database, ) : PortfolioRepository { private val stockQueries = portfolioDb.stockQueries override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain() } catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain() } } override suspend fun filterPortfolio( filterTerm: String ): Portfolio {
  5. } internal class RealPortfolioRepository( private val portfolioApi: PortfolioApi, portfolioDb: Database,

    ) : PortfolioRepository { private val stockQueries = portfolioDb.stockQueries override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain() } catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain() } } override suspend fun filterPortfolio( filterTerm: String ): Portfolio { return stockQueries.selectMatching(filterTerm) .executeAsList().toDomain() } }
  6. override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain()

    } catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain() } } override suspend fun filterPortfolio( filterTerm: String ): Portfolio { return stockQueries.selectMatching(filterTerm) .executeAsList().toDomain() } }
  7. Po rt folioRepositoryTest class PortfolioRepositoryTest { private val portfolioApi =

    FakePortfolioApi() private val portfolioDb = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) .let { driver -> DbModule.providePortfolioDb(driver).also { Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi = portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain())
  8. class FakePortfolioApi : PortfolioApi { val portfolio = Turbine<Portfolio>() val

    exception = Turbine<Exception>() private val exceptionChannel = exception.asChannel() override suspend fun getPortfolio(): Portfolio { return if (!exceptionChannel.isEmpty) { throw exception.awaitItem() } else { portfolio.awaitItem() } } }
  9. public interface Turbine<T> • Coroutine-safe datastore • Based on Channel

    • Add items synchronously • Suspend to take out items
  10. class FakePortfolioApi : PortfolioApi { val portfolio = Turbine<Portfolio>() val

    exception = Turbine<Exception>() private val exceptionChannel = exception.asChannel() override suspend fun getPortfolio(): Portfolio { return if (!exceptionChannel.isEmpty) { throw exception.awaitItem() } else { portfolio.awaitItem() } } }
  11. class FakePortfolioApi : PortfolioApi { val portfolio = Turbine<Portfolio>() val

    exception = Turbine<Exception>() private val exceptionChannel = exception.asChannel() override suspend fun getPortfolio(): Portfolio { return if (!exceptionChannel.isEmpty) { throw exception.awaitItem() } else { portfolio.awaitItem() } } }
  12. class FakePortfolioApi : PortfolioApi { val portfolio = Turbine<Portfolio>() val

    exception = Turbine<Exception>() private val exceptionChannel = exception.asChannel() override suspend fun getPortfolio(): Portfolio { return if (!exceptionChannel.isEmpty) { throw exception.awaitItem() } else { portfolio.awaitItem() } } }
  13. class PortfolioRepositoryTest { private val portfolioApi = FakePortfolioApi() private val

    portfolioDb = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) .let { driver -> DbModule.providePortfolioDb(driver).also { Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi = portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) }
  14. class PortfolioRepositoryTest { private val portfolioApi = FakePortfolioApi() private val

    portfolioDb = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) .let { driver -> DbModule.providePortfolioDb(driver).also { Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi = portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio
  15. Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi =

    portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(networkPortfolio.toDomain()) portfolioApi.exception += IOException() assertThat(portfolioRepository.getPortfolio())
  16. Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi =

    portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(networkPortfolio.toDomain()) portfolioApi.exception += IOException() assertThat(portfolioRepository.getPortfolio())
  17. /** * Add an item to the underlying [Channel] without

    blocking. */ public fun add(item: T) public operator fun <T> Turbine<T>.plusAssign(value: T) { add(value) }
  18. Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi =

    portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(networkPortfolio.toDomain()) portfolioApi.exception += IOException() assertThat(portfolioRepository.getPortfolio())
  19. ) @Test fun `portfolio loaded from network`() = runBlocking {

    portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(networkPortfolio.toDomain()) portfolioApi.exception += IOException() assertThat(portfolioRepository.getPortfolio()) .isEqualTo(dbPortfolio.toDomain()) } }
  20. Po rt folioPresenter @Composable internal fun PortfolioPresenter( portfolioRepository: PortfolioRepository, currencyFormatter:

    NumberFormat, dateTimeFormatter: DateFormat, ): PortfolioModel { var state by remember { mutableStateOf(State(isLoading = true)) } LaunchedEffect("get-portfolio") { val portfolio = portfolioRepository.getPortfolio() state = state.copy(portfolio = portfolio, isLoading = false) } return when { state.isLoading -> PortfolioModel.Loading state.portfolio != null -> state.portfolio!!.toModel( currencyFormatter = currencyFormatter, dateTimeFormatter = dateTimeFormatter, ) else -> error("Unexpected state: $state")
  21. @Composable internal fun PortfolioPresenter( portfolioRepository: PortfolioRepository, currencyFormatter: NumberFormat, dateTimeFormatter: DateFormat,

    ): PortfolioModel { var state by remember { mutableStateOf(State(isLoading = true)) } LaunchedEffect("get-portfolio") { val portfolio = portfolioRepository.getPortfolio() state = state.copy(portfolio = portfolio, isLoading = false) } return when { state.isLoading -> PortfolioModel.Loading state.portfolio != null -> state.portfolio!!.toModel( currencyFormatter = currencyFormatter, dateTimeFormatter = dateTimeFormatter, ) else -> error("Unexpected state: $state") } }
  22. @Composable internal fun PortfolioPresenter( portfolioRepository: PortfolioRepository, currencyFormatter: NumberFormat, dateTimeFormatter: DateFormat,

    ): PortfolioModel { var state by remember { mutableStateOf(State(isLoading = true)) } LaunchedEffect("get-portfolio") { val portfolio = portfolioRepository.getPortfolio() state = state.copy(portfolio = portfolio, isLoading = false) } return when { state.isLoading -> PortfolioModel.Loading state.portfolio != null -> state.portfolio!!.toModel( currencyFormatter = currencyFormatter, dateTimeFormatter = dateTimeFormatter, ) else -> error("Unexpected state: $state") } }
  23. currencyFormatter: NumberFormat, dateTimeFormatter: DateFormat, ): PortfolioModel { var state by

    remember { mutableStateOf(State(isLoading = true)) } LaunchedEffect("get-portfolio") { val portfolio = portfolioRepository.getPortfolio() state = state.copy(portfolio = portfolio, isLoading = false) } return when { state.isLoading -> PortfolioModel.Loading state.portfolio != null -> state.portfolio!!.toModel( currencyFormatter = currencyFormatter, dateTimeFormatter = dateTimeFormatter, ) else -> error("Unexpected state: $state") } }
  24. Po rt folioPresenterTest class PortfolioPresenterTest { private val portfolioRepository =

    FakePortfolioRepository() @Test fun `loads portfolio`() = runBlocking { portfolioRepository.portfolio += Portfolio( positions = listOf( Position( stock = Stock( ticker = "TWTR", name = "Twitter, Inc.", currency = Currency.getInstance("USD"), currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25, ),
  25. class PortfolioPresenterTest { private val portfolioRepository = FakePortfolioRepository() @Test fun

    `loads portfolio`() = runBlocking { portfolioRepository.portfolio += Portfolio( positions = listOf( Position( stock = Stock( ticker = "TWTR", name = "Twitter, Inc.", currency = Currency.getInstance("USD"), currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25, ), ), )
  26. currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25,

    ), ), ) makePresenter().test { assertThat(awaitItem()).isEqualTo(PortfolioModel.Loading) assertThat(awaitItem()).isEqualTo( PortfolioModel.Loaded( positions = listOf( PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ), ), ), ) } }
  27. package app.cash.turbine public suspend fun <T> Flow<T>.test( timeout: Duration? =

    null, validate: suspend ReceiveTurbine<T>.() -> Unit, )
  28. currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25,

    ), ), ) makePresenter().test { assertThat(awaitItem()).isEqualTo(PortfolioModel.Loading) assertThat(awaitItem()).isEqualTo( PortfolioModel.Loaded( positions = listOf( PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ), ), ), ) } }
  29. currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25,

    ), ), ) makePresenter().test { assertThat(awaitItem()).isEqualTo(PortfolioModel.Loading) assertThat(awaitItem()).isEqualTo( PortfolioModel.Loaded( positions = listOf( PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ), ), ), ) } }
  30. quantity = 25, ), ), ), ) } } private

    fun makePresenter(): Flow<PortfolioModel> { return PresenterModule.providePortfolioPresenter( clock = Immediate, portfolioRepository = portfolioRepository, locale = Locale.US, timeZone = TimeZone.getTimeZone("America/New_York"), ) } }
  31. Integration tests • Not hermetic • Test integration assumptions •

    Slower feedback Photo by Taylor Vick on Unsplash
  32. Po rt folioApiTest @RunWith(TestParameterInjector::class) class PortfolioApiTest( @TestParameter private val fixture:

    Fixture, ) { @Test fun test() = runBlocking { val server = MockWebServer() server.enqueue( MockResponse() .setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path = "/").toString(), ) try {
  33. enum class Fixture( val responsePath: String, val expectedResponse: Portfolio?, val

    expectedExceptionType: KClass<out Throwable>?, ) { DEFAULT( responsePath = "/portfolio.json", expectedResponse = Portfolio(stocks = Network.all), expectedExceptionType = null, ), EMPTY( responsePath = "/portfolio-empty.json", expectedResponse = Portfolio(stocks = emptyList()), expectedExceptionType = null, ), MALFORMED( responsePath = "/portfolio-malformed.json", expectedResponse = null, expectedExceptionType = JsonEncodingException::class, ) }
  34. @RunWith(TestParameterInjector::class) class PortfolioApiTest( @TestParameter private val fixture: Fixture, ) {

    @Test fun test() = runBlocking { val server = MockWebServer() server.enqueue( MockResponse() .setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path = "/").toString(), ) try { val response = portfolioApi.getPortfolio() assertThat(response).isEqualTo(fixture.expectedResponse) } catch (t: Throwable) { assertThat(t).isInstanceOf(fixture.expectedExceptionType?.java)
  35. @Test fun test() = runBlocking { val server = MockWebServer()

    server.enqueue( MockResponse() .setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path = "/").toString(), ) try { val response = portfolioApi.getPortfolio() assertThat(response).isEqualTo(fixture.expectedResponse) } catch (t: Throwable) { assertThat(t).isInstanceOf(fixture.expectedExceptionType?.java) } }
  36. .setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path

    = "/").toString(), ) try { val response = portfolioApi.getPortfolio() assertThat(response).isEqualTo(fixture.expectedResponse) } catch (t: Throwable) { assertThat(t).isInstanceOf(fixture.expectedExceptionType?.java) } }
  37. PositionViewTest @RunWith(TestParameterInjector::class) class PositionViewTest( @TestParameter private val theme: Theme, )

    { @get:Rule val paparazzi = Paparazzi(deviceConfig = PIXEL_5) @Test fun default() { val model = PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ) paparazzi.snapshot { StonksTestingTheme(darkTheme = theme == Theme.DARK) { Surface(
  38. @RunWith(TestParameterInjector::class) class PositionViewTest( @TestParameter private val theme: Theme, ) {

    @get:Rule val paparazzi = Paparazzi(deviceConfig = PIXEL_5) @Test fun default() { val model = PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ) paparazzi.snapshot { StonksTestingTheme(darkTheme = theme == Theme.DARK) { Surface( modifier = Modifier.fillMaxSize(), color = MaterialTheme.colors.background, ) { PositionView(model) } }
  39. @Test fun default() { val model = PortfolioModel.PositionModel( ticker =

    "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ) paparazzi.snapshot { StonksTestingTheme(darkTheme = theme == Theme.DARK) { Surface( modifier = Modifier.fillMaxSize(), color = MaterialTheme.colors.background, ) { PositionView(model) } } } }
  40. End-to-end tests • Test complete fl ows • Run on

    real hardware • Slowest feedback • Multiple sources of fl akiness Photo by Adi Goldstein on Unsplash
  41. Po rt folioRobot class PortfolioRobot( private val testRule: ComposeContentTestRule )

    { fun filter(filterTerm: String) { testRule.waitUntilExists(isFilterInputField()) testRule.onFilterInputField().performTextInput(filterTerm) } private fun SemanticsNodeInteractionsProvider.onFilterInputField() = onNode(isFilterInputField()) private fun isFilterInputField() = hasSetTextAction() and hasText("Type to filter") fun checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() }
  42. class PortfolioRobot( private val testRule: ComposeContentTestRule ) { fun filter(filterTerm:

    String) { testRule.waitUntilExists(isFilterInputField()) testRule.onFilterInputField().performTextInput(filterTerm) } private fun SemanticsNodeInteractionsProvider.onFilterInputField() = onNode(isFilterInputField()) private fun isFilterInputField() = hasSetTextAction() and hasText("Type to filter") fun checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() } } fun ComposeContentTestRule.portfolio(
  43. } private fun SemanticsNodeInteractionsProvider.onFilterInputField() = onNode(isFilterInputField()) private fun isFilterInputField() =

    hasSetTextAction() and hasText("Type to filter") fun checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() } } fun ComposeContentTestRule.portfolio( body: PortfolioRobot.() -> Unit ) = PortfolioRobot(this).body()
  44. private fun isFilterInputField() = hasSetTextAction() and hasText("Type to filter") fun

    checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() } } fun ComposeContentTestRule.portfolio( body: PortfolioRobot.() -> Unit ) = PortfolioRobot(this).body()
  45. @RunWith(AndroidJUnit4::class) class FilterPortfolioTest { @get:Rule val activityRule = createAndroidComposeRule<MainActivity>() @Test

    fun filterPortfolioByStockTicker() = with(activityRule) { portfolio { filter("JNJ") checkHasEntry("Johnson & Johnson") } } @Test fun filterPortfolioByStockName() = with(activityRule) { portfolio { filter("Under Armour") checkHasEntry("UA") } } FilterPo rt folioTest
  46. @RunWith(AndroidJUnit4::class) class FilterPortfolioTest { @get:Rule val activityRule = createAndroidComposeRule<MainActivity>() @Test

    fun filterPortfolioByStockTicker() = with(activityRule) { portfolio { filter("JNJ") checkHasEntry("Johnson & Johnson") } } @Test fun filterPortfolioByStockName() = with(activityRule) { portfolio { filter("Under Armour") checkHasEntry("UA") } } }
  47. fladle { serviceAccountCredentials.set( project.layout.projectDirectory.file( “fladle-auth.json" ) ) variant.set("debug") performanceMetrics.set(false) devices.set(

    listOf( mapOf("model" to "bluejay", "version" to "32"), // Pixel 6a mapOf("model" to "Nexus5X", "version" to "24"), ) ) environmentVariables.set( mapOf( "failureScreenshots" to "true", ) ) }
  48. firebase: name: Run UI tests in Firebase Test Lab runs-on:

    ubuntu-latest steps: - name: Checkout the repo. … - name: Set up JDK 11. … - name: Setup Gradle. … - name: Create Fladle authentication JSON file env: GCLOUD_AUTH: ${{ secrets.GCLOUD_AUTH }} run: echo $GCLOUD_AUTH >> app/fladle-auth.json - name: Run Instrumentation Tests in Firebase Test Lab run: | ./gradlew app:assembleDebug app:assembleDebugAndroidTest ./gradlew runFlank
  49. Conclusion • Testing tooling is awesome! • Prioritize smaller tests,

    but write all. • Use open source! Photo by Girl with red hat on Unsplash