Share
State of Android Testing in 2022
Android’s testing story has come a long way since the early days (old timers might recall not being able to run unit tests on the local JVM, or struggling to write UI tests in the low on caffeine pre-Espresso world), and nowadays there’s a multitude of tools, both 1st and 3rd party, to help you assemble a world-class testing pipeline. In this talk we’ll go down the test pyramid and present an overview of tools and techniques to help you write efficient tests in each category. We’ll discuss technologies that are heavily used at (and some even built by) Cash App. The set of topics includes, but is not limited to:
- What kinds of tests should you focus on?
- Writing end-to-end tests with Espresso and running them on real devices in the Firebase Test Lab.
- Using Paparazzi to write screenshot tests and catch UI regressions.
- Turbine & other useful tools for testing Kotlin’s coroutines & Flows.
This talk should give developers a holistic overview and a wealth of practical advice on writing all types of tests for your Android app!
Egor Andreevich
Transcript
88 slides
-
Agenda • Brief history of Android testing • The testing
pyramid • Testing: what, why and how • Tools & techniques Photo by Girl with red hat on Unsplash -
Sep ‘08 Android 1.0 Nov ‘07 Aug ‘13 3.2” 320x480
528 MHz ARM 11 192 MB RAM 1150 mAh -
Testing pyramid Unit Integration E2E Integration Less More Less Cost
Less More Less Flakiness Less More Less -
Unit tests • Hermetic • Safety net • Fast feedback
loop • API testbed Photo by Fikri Rasyid on Unsplash -
Po rt folioRepository interface PortfolioRepository { suspend fun getPortfolio(): Portfolio
suspend fun filterPortfolio(filterTerm: String): Portfolio } internal class RealPortfolioRepository( private val portfolioApi: PortfolioApi, portfolioDb: Database, ) : PortfolioRepository { private val stockQueries = portfolioDb.stockQueries override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain() } catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain() -
interface PortfolioRepository { suspend fun getPortfolio(): Portfolio suspend fun filterPortfolio(filterTerm:
String): Portfolio } internal class RealPortfolioRepository( private val portfolioApi: PortfolioApi, portfolioDb: Database, ) : PortfolioRepository { private val stockQueries = portfolioDb.stockQueries override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain() } catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain() } } override suspend fun filterPortfolio( filterTerm: String ): Portfolio { -
} internal class RealPortfolioRepository( private val portfolioApi: PortfolioApi, portfolioDb: Database,
) : PortfolioRepository { private val stockQueries = portfolioDb.stockQueries override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain() } catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain() } } override suspend fun filterPortfolio( filterTerm: String ): Portfolio { return stockQueries.selectMatching(filterTerm) .executeAsList().toDomain() } } -
override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain()
} catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain() } } override suspend fun filterPortfolio( filterTerm: String ): Portfolio { return stockQueries.selectMatching(filterTerm) .executeAsList().toDomain() } } -
Po rt folioRepositoryTest class PortfolioRepositoryTest { private val portfolioApi =
FakePortfolioApi() private val portfolioDb = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) .let { driver -> DbModule.providePortfolioDb(driver).also { Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi = portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) -
class FakePortfolioApi : PortfolioApi { val portfolio = Turbine<Portfolio>() val
exception = Turbine<Exception>() private val exceptionChannel = exception.asChannel() override suspend fun getPortfolio(): Portfolio { return if (!exceptionChannel.isEmpty) { throw exception.awaitItem() } else { portfolio.awaitItem() } } } -
public interface Turbine<T> • Coroutine-safe datastore • Based on Channel
• Add items synchronously • Suspend to take out items -
class FakePortfolioApi : PortfolioApi { val portfolio = Turbine<Portfolio>() val
exception = Turbine<Exception>() private val exceptionChannel = exception.asChannel() override suspend fun getPortfolio(): Portfolio { return if (!exceptionChannel.isEmpty) { throw exception.awaitItem() } else { portfolio.awaitItem() } } } -
class FakePortfolioApi : PortfolioApi { val portfolio = Turbine<Portfolio>() val
exception = Turbine<Exception>() private val exceptionChannel = exception.asChannel() override suspend fun getPortfolio(): Portfolio { return if (!exceptionChannel.isEmpty) { throw exception.awaitItem() } else { portfolio.awaitItem() } } } -
class FakePortfolioApi : PortfolioApi { val portfolio = Turbine<Portfolio>() val
exception = Turbine<Exception>() private val exceptionChannel = exception.asChannel() override suspend fun getPortfolio(): Portfolio { return if (!exceptionChannel.isEmpty) { throw exception.awaitItem() } else { portfolio.awaitItem() } } } -
class PortfolioRepositoryTest { private val portfolioApi = FakePortfolioApi() private val
portfolioDb = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) .let { driver -> DbModule.providePortfolioDb(driver).also { Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi = portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } -
class PortfolioRepositoryTest { private val portfolioApi = FakePortfolioApi() private val
portfolioDb = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) .let { driver -> DbModule.providePortfolioDb(driver).also { Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi = portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio -
Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi =
portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(networkPortfolio.toDomain()) portfolioApi.exception += IOException() assertThat(portfolioRepository.getPortfolio()) -
Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi =
portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(networkPortfolio.toDomain()) portfolioApi.exception += IOException() assertThat(portfolioRepository.getPortfolio()) -
/** * Add an item to the underlying [Channel] without
blocking. */ public fun add(item: T) public operator fun <T> Turbine<T>.plusAssign(value: T) { add(value) } -
Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi =
portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(networkPortfolio.toDomain()) portfolioApi.exception += IOException() assertThat(portfolioRepository.getPortfolio()) -
) @Test fun `portfolio loaded from network`() = runBlocking {
portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(networkPortfolio.toDomain()) portfolioApi.exception += IOException() assertThat(portfolioRepository.getPortfolio()) .isEqualTo(dbPortfolio.toDomain()) } } -
Po rt folioPresenter @Composable internal fun PortfolioPresenter( portfolioRepository: PortfolioRepository, currencyFormatter:
NumberFormat, dateTimeFormatter: DateFormat, ): PortfolioModel { var state by remember { mutableStateOf(State(isLoading = true)) } LaunchedEffect("get-portfolio") { val portfolio = portfolioRepository.getPortfolio() state = state.copy(portfolio = portfolio, isLoading = false) } return when { state.isLoading -> PortfolioModel.Loading state.portfolio != null -> state.portfolio!!.toModel( currencyFormatter = currencyFormatter, dateTimeFormatter = dateTimeFormatter, ) else -> error("Unexpected state: $state") -
@Composable internal fun PortfolioPresenter( portfolioRepository: PortfolioRepository, currencyFormatter: NumberFormat, dateTimeFormatter: DateFormat,
): PortfolioModel { var state by remember { mutableStateOf(State(isLoading = true)) } LaunchedEffect("get-portfolio") { val portfolio = portfolioRepository.getPortfolio() state = state.copy(portfolio = portfolio, isLoading = false) } return when { state.isLoading -> PortfolioModel.Loading state.portfolio != null -> state.portfolio!!.toModel( currencyFormatter = currencyFormatter, dateTimeFormatter = dateTimeFormatter, ) else -> error("Unexpected state: $state") } } -
@Composable internal fun PortfolioPresenter( portfolioRepository: PortfolioRepository, currencyFormatter: NumberFormat, dateTimeFormatter: DateFormat,
): PortfolioModel { var state by remember { mutableStateOf(State(isLoading = true)) } LaunchedEffect("get-portfolio") { val portfolio = portfolioRepository.getPortfolio() state = state.copy(portfolio = portfolio, isLoading = false) } return when { state.isLoading -> PortfolioModel.Loading state.portfolio != null -> state.portfolio!!.toModel( currencyFormatter = currencyFormatter, dateTimeFormatter = dateTimeFormatter, ) else -> error("Unexpected state: $state") } } -
currencyFormatter: NumberFormat, dateTimeFormatter: DateFormat, ): PortfolioModel { var state by
remember { mutableStateOf(State(isLoading = true)) } LaunchedEffect("get-portfolio") { val portfolio = portfolioRepository.getPortfolio() state = state.copy(portfolio = portfolio, isLoading = false) } return when { state.isLoading -> PortfolioModel.Loading state.portfolio != null -> state.portfolio!!.toModel( currencyFormatter = currencyFormatter, dateTimeFormatter = dateTimeFormatter, ) else -> error("Unexpected state: $state") } } -
Po rt folioPresenterTest class PortfolioPresenterTest { private val portfolioRepository =
FakePortfolioRepository() @Test fun `loads portfolio`() = runBlocking { portfolioRepository.portfolio += Portfolio( positions = listOf( Position( stock = Stock( ticker = "TWTR", name = "Twitter, Inc.", currency = Currency.getInstance("USD"), currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25, ), -
class PortfolioPresenterTest { private val portfolioRepository = FakePortfolioRepository() @Test fun
`loads portfolio`() = runBlocking { portfolioRepository.portfolio += Portfolio( positions = listOf( Position( stock = Stock( ticker = "TWTR", name = "Twitter, Inc.", currency = Currency.getInstance("USD"), currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25, ), ), ) -
currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25,
), ), ) makePresenter().test { assertThat(awaitItem()).isEqualTo(PortfolioModel.Loading) assertThat(awaitItem()).isEqualTo( PortfolioModel.Loaded( positions = listOf( PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ), ), ), ) } } -
package app.cash.turbine public suspend fun <T> Flow<T>.test( timeout: Duration? =
null, validate: suspend ReceiveTurbine<T>.() -> Unit, ) -
currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25,
), ), ) makePresenter().test { assertThat(awaitItem()).isEqualTo(PortfolioModel.Loading) assertThat(awaitItem()).isEqualTo( PortfolioModel.Loaded( positions = listOf( PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ), ), ), ) } } -
currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25,
), ), ) makePresenter().test { assertThat(awaitItem()).isEqualTo(PortfolioModel.Loading) assertThat(awaitItem()).isEqualTo( PortfolioModel.Loaded( positions = listOf( PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ), ), ), ) } } -
quantity = 25, ), ), ), ) } } private
fun makePresenter(): Flow<PortfolioModel> { return PresenterModule.providePortfolioPresenter( clock = Immediate, portfolioRepository = portfolioRepository, locale = Locale.US, timeZone = TimeZone.getTimeZone("America/New_York"), ) } } -
Integration tests • Not hermetic • Test integration assumptions •
Slower feedback Photo by Taylor Vick on Unsplash -
Po rt folioApiTest @RunWith(TestParameterInjector::class) class PortfolioApiTest( @TestParameter private val fixture:
Fixture, ) { @Test fun test() = runBlocking { val server = MockWebServer() server.enqueue( MockResponse() .setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path = "/").toString(), ) try { -
enum class Fixture( val responsePath: String, val expectedResponse: Portfolio?, val
expectedExceptionType: KClass<out Throwable>?, ) { DEFAULT( responsePath = "/portfolio.json", expectedResponse = Portfolio(stocks = Network.all), expectedExceptionType = null, ), EMPTY( responsePath = "/portfolio-empty.json", expectedResponse = Portfolio(stocks = emptyList()), expectedExceptionType = null, ), MALFORMED( responsePath = "/portfolio-malformed.json", expectedResponse = null, expectedExceptionType = JsonEncodingException::class, ) } -
@RunWith(TestParameterInjector::class) class PortfolioApiTest( @TestParameter private val fixture: Fixture, ) {
@Test fun test() = runBlocking { val server = MockWebServer() server.enqueue( MockResponse() .setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path = "/").toString(), ) try { val response = portfolioApi.getPortfolio() assertThat(response).isEqualTo(fixture.expectedResponse) } catch (t: Throwable) { assertThat(t).isInstanceOf(fixture.expectedExceptionType?.java) -
@Test fun test() = runBlocking { val server = MockWebServer()
server.enqueue( MockResponse() .setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path = "/").toString(), ) try { val response = portfolioApi.getPortfolio() assertThat(response).isEqualTo(fixture.expectedResponse) } catch (t: Throwable) { assertThat(t).isInstanceOf(fixture.expectedExceptionType?.java) } } -
.setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path
= "/").toString(), ) try { val response = portfolioApi.getPortfolio() assertThat(response).isEqualTo(fixture.expectedResponse) } catch (t: Throwable) { assertThat(t).isInstanceOf(fixture.expectedExceptionType?.java) } } -
PositionViewTest @RunWith(TestParameterInjector::class) class PositionViewTest( @TestParameter private val theme: Theme, )
{ @get:Rule val paparazzi = Paparazzi(deviceConfig = PIXEL_5) @Test fun default() { val model = PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ) paparazzi.snapshot { StonksTestingTheme(darkTheme = theme == Theme.DARK) { Surface( -
@RunWith(TestParameterInjector::class) class PositionViewTest( @TestParameter private val theme: Theme, ) {
@get:Rule val paparazzi = Paparazzi(deviceConfig = PIXEL_5) @Test fun default() { val model = PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ) paparazzi.snapshot { StonksTestingTheme(darkTheme = theme == Theme.DARK) { Surface( modifier = Modifier.fillMaxSize(), color = MaterialTheme.colors.background, ) { PositionView(model) } } -
@Test fun default() { val model = PortfolioModel.PositionModel( ticker =
"TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ) paparazzi.snapshot { StonksTestingTheme(darkTheme = theme == Theme.DARK) { Surface( modifier = Modifier.fillMaxSize(), color = MaterialTheme.colors.background, ) { PositionView(model) } } } } -
End-to-end tests • Test complete fl ows • Run on
real hardware • Slowest feedback • Multiple sources of fl akiness Photo by Adi Goldstein on Unsplash -
Po rt folioRobot class PortfolioRobot( private val testRule: ComposeContentTestRule )
{ fun filter(filterTerm: String) { testRule.waitUntilExists(isFilterInputField()) testRule.onFilterInputField().performTextInput(filterTerm) } private fun SemanticsNodeInteractionsProvider.onFilterInputField() = onNode(isFilterInputField()) private fun isFilterInputField() = hasSetTextAction() and hasText("Type to filter") fun checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() } -
class PortfolioRobot( private val testRule: ComposeContentTestRule ) { fun filter(filterTerm:
String) { testRule.waitUntilExists(isFilterInputField()) testRule.onFilterInputField().performTextInput(filterTerm) } private fun SemanticsNodeInteractionsProvider.onFilterInputField() = onNode(isFilterInputField()) private fun isFilterInputField() = hasSetTextAction() and hasText("Type to filter") fun checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() } } fun ComposeContentTestRule.portfolio( -
} private fun SemanticsNodeInteractionsProvider.onFilterInputField() = onNode(isFilterInputField()) private fun isFilterInputField() =
hasSetTextAction() and hasText("Type to filter") fun checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() } } fun ComposeContentTestRule.portfolio( body: PortfolioRobot.() -> Unit ) = PortfolioRobot(this).body() -
private fun isFilterInputField() = hasSetTextAction() and hasText("Type to filter") fun
checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() } } fun ComposeContentTestRule.portfolio( body: PortfolioRobot.() -> Unit ) = PortfolioRobot(this).body() -
@RunWith(AndroidJUnit4::class) class FilterPortfolioTest { @get:Rule val activityRule = createAndroidComposeRule<MainActivity>() @Test
fun filterPortfolioByStockTicker() = with(activityRule) { portfolio { filter("JNJ") checkHasEntry("Johnson & Johnson") } } @Test fun filterPortfolioByStockName() = with(activityRule) { portfolio { filter("Under Armour") checkHasEntry("UA") } } FilterPo rt folioTest -
@RunWith(AndroidJUnit4::class) class FilterPortfolioTest { @get:Rule val activityRule = createAndroidComposeRule<MainActivity>() @Test
fun filterPortfolioByStockTicker() = with(activityRule) { portfolio { filter("JNJ") checkHasEntry("Johnson & Johnson") } } @Test fun filterPortfolioByStockName() = with(activityRule) { portfolio { filter("Under Armour") checkHasEntry("UA") } } } -
fladle { serviceAccountCredentials.set( project.layout.projectDirectory.file( “fladle-auth.json" ) ) variant.set("debug") performanceMetrics.set(false) devices.set(
listOf( mapOf("model" to "bluejay", "version" to "32"), // Pixel 6a mapOf("model" to "Nexus5X", "version" to "24"), ) ) environmentVariables.set( mapOf( "failureScreenshots" to "true", ) ) } -
firebase: name: Run UI tests in Firebase Test Lab runs-on:
ubuntu-latest steps: - name: Checkout the repo. … - name: Set up JDK 11. … - name: Setup Gradle. … - name: Create Fladle authentication JSON file env: GCLOUD_AUTH: ${{ secrets.GCLOUD_AUTH }} run: echo $GCLOUD_AUTH >> app/fladle-auth.json - name: Run Instrumentation Tests in Firebase Test Lab run: | ./gradlew app:assembleDebug app:assembleDebugAndroidTest ./gradlew runFlank -
Conclusion • Testing tooling is awesome! • Prioritize smaller tests,
but write all. • Use open source! Photo by Girl with red hat on Unsplash