State of Android Testing in 2022

Slide 1

Slide 1 text

@egorand State of Android Testing Edition ‘22

Slide 2

Slide 2 text

Agenda • Brief history of Android testing • The testing pyramid • Testing: what, why and how • Tools & techniques Photo by Girl with red hat on Unsplash

Slide 3

Slide 3 text

Brief history of Android testing

Slide 4

Slide 4 text

Nov ‘07 Android: fi rst public beta Sep ‘08

Slide 5

Slide 5 text

Sep ‘08 Android 1.0 Nov ‘07 Aug ‘13 3.2” 320x480 528 MHz ARM 11 192 MB RAM 1150 mAh

Slide 6

Slide 6 text

Aug ‘13 Android Studio & Gradle Sep ‘08 Oct ‘13

Slide 7

Slide 7 text

Oct ‘13 Espresso Aug ‘13 Feb ‘15

Slide 8

Slide 8 text

Oct ‘13 Local unit tests Feb ‘15 May ‘17

Slide 9

Slide 9 text

Kotlin Feb ‘15 May ‘17 May ‘18

Slide 10

Slide 10 text

Android Jetpack May ‘18 May ‘17 May ‘21

Slide 11

Slide 11 text

Jetpack Compose May ‘21 May ‘18

Slide 12

Slide 12 text

Testing pyramid

Slide 13

Slide 13 text

Testing pyramid Unit Integration E2E Integration Less More Less Cost Less More Less Flakiness Less More Less

Slide 14

Slide 14 text

The “Stonks” app

Slide 15

Slide 15 text

No content

Slide 16

Slide 16 text

app ui presenters ui-models data domain

Slide 17

Slide 17 text

Unit tests • Hermetic • Safety net • Fast feedback loop • API testbed Photo by Fikri Rasyid on Unsplash

Slide 18

Slide 18 text

Po rt folioRepository interface PortfolioRepository { suspend fun getPortfolio(): Portfolio suspend fun filterPortfolio(filterTerm: String): Portfolio } internal class RealPortfolioRepository( private val portfolioApi: PortfolioApi, portfolioDb: Database, ) : PortfolioRepository { private val stockQueries = portfolioDb.stockQueries override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain() } catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain()

Slide 19

Slide 19 text

interface PortfolioRepository { suspend fun getPortfolio(): Portfolio suspend fun filterPortfolio(filterTerm: String): Portfolio } internal class RealPortfolioRepository( private val portfolioApi: PortfolioApi, portfolioDb: Database, ) : PortfolioRepository { private val stockQueries = portfolioDb.stockQueries override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain() } catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain() } } override suspend fun filterPortfolio( filterTerm: String ): Portfolio {

Slide 20

Slide 20 text

} internal class RealPortfolioRepository( private val portfolioApi: PortfolioApi, portfolioDb: Database, ) : PortfolioRepository { private val stockQueries = portfolioDb.stockQueries override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain() } catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain() } } override suspend fun filterPortfolio( filterTerm: String ): Portfolio { return stockQueries.selectMatching(filterTerm) .executeAsList().toDomain() } }

Slide 21

Slide 21 text

override suspend fun getPortfolio(): Portfolio { return try { portfolioApi.getPortfolio().also(this::persist).toDomain() } catch (e: Exception) { stockQueries.selectAll().executeAsList().toDomain() } } override suspend fun filterPortfolio( filterTerm: String ): Portfolio { return stockQueries.selectMatching(filterTerm) .executeAsList().toDomain() } }

Slide 22

Slide 22 text

Po rt folioRepositoryTest class PortfolioRepositoryTest { private val portfolioApi = FakePortfolioApi() private val portfolioDb = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) .let { driver -> DbModule.providePortfolioDb(driver).also { Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi = portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain())

Slide 23

Slide 23 text

Fakes over Mocks 🛠 🔮

Slide 24

Slide 24 text

class FakePortfolioApi : PortfolioApi { val portfolio = Turbine() val exception = Turbine() private val exceptionChannel = exception.asChannel() override suspend fun getPortfolio(): Portfolio { return if (!exceptionChannel.isEmpty) { throw exception.awaitItem() } else { portfolio.awaitItem() } } }

Slide 25

Slide 25 text

No content

Slide 26

Slide 26 text

public interface Turbine • Coroutine-safe datastore • Based on Channel • Add items synchronously • Suspend to take out items

Slide 27

Slide 27 text

Slide 28

Slide 28 text

Slide 29

Slide 29 text

Slide 30

Slide 30 text

class PortfolioRepositoryTest { private val portfolioApi = FakePortfolioApi() private val portfolioDb = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) .let { driver -> DbModule.providePortfolioDb(driver).also { Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi = portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) }

Slide 31

Slide 31 text

Slide 32

Slide 32 text

Schema.create(driver) } } private val portfolioRepository = RealPortfolioRepository( portfolioApi = portfolioApi, portfolioDb = portfolioDb, ) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(networkPortfolio.toDomain()) portfolioApi.exception += IOException() assertThat(portfolioRepository.getPortfolio())

Slide 33

Slide 33 text

Slide 34

Slide 34 text

/** * Add an item to the underlying [Channel] without blocking. */ public fun add(item: T) public operator fun Turbine.plusAssign(value: T) { add(value) }

Slide 35

Slide 35 text

Slide 36

Slide 36 text

) @Test fun `portfolio loaded from network`() = runBlocking { portfolioApi.portfolio += Network.portfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(Network.portfolio.toDomain()) } @Test fun `persisted portfolio loaded from DB`() = runBlocking { val networkPortfolio = Network.portfolio .copy(stocks = Network.all - Network.TWTR) val dbPortfolio = Db.all - Db.TWTR portfolioApi.portfolio += networkPortfolio assertThat(portfolioRepository.getPortfolio()) .isEqualTo(networkPortfolio.toDomain()) portfolioApi.exception += IOException() assertThat(portfolioRepository.getPortfolio()) .isEqualTo(dbPortfolio.toDomain()) } }

Slide 37

Slide 37 text

Po rt folioPresenter @Composable internal fun PortfolioPresenter( portfolioRepository: PortfolioRepository, currencyFormatter: NumberFormat, dateTimeFormatter: DateFormat, ): PortfolioModel { var state by remember { mutableStateOf(State(isLoading = true)) } LaunchedEffect("get-portfolio") { val portfolio = portfolioRepository.getPortfolio() state = state.copy(portfolio = portfolio, isLoading = false) } return when { state.isLoading -> PortfolioModel.Loading state.portfolio != null -> state.portfolio!!.toModel( currencyFormatter = currencyFormatter, dateTimeFormatter = dateTimeFormatter, ) else -> error("Unexpected state: $state")

Slide 38

Slide 38 text

No content

Slide 39

Slide 39 text

No content

Slide 40

Slide 40 text

@Composable internal fun PortfolioPresenter( portfolioRepository: PortfolioRepository, currencyFormatter: NumberFormat, dateTimeFormatter: DateFormat, ): PortfolioModel { var state by remember { mutableStateOf(State(isLoading = true)) } LaunchedEffect("get-portfolio") { val portfolio = portfolioRepository.getPortfolio() state = state.copy(portfolio = portfolio, isLoading = false) } return when { state.isLoading -> PortfolioModel.Loading state.portfolio != null -> state.portfolio!!.toModel( currencyFormatter = currencyFormatter, dateTimeFormatter = dateTimeFormatter, ) else -> error("Unexpected state: $state") } }

Slide 41

Slide 41 text

Slide 42

Slide 42 text

currencyFormatter: NumberFormat, dateTimeFormatter: DateFormat, ): PortfolioModel { var state by remember { mutableStateOf(State(isLoading = true)) } LaunchedEffect("get-portfolio") { val portfolio = portfolioRepository.getPortfolio() state = state.copy(portfolio = portfolio, isLoading = false) } return when { state.isLoading -> PortfolioModel.Loading state.portfolio != null -> state.portfolio!!.toModel( currencyFormatter = currencyFormatter, dateTimeFormatter = dateTimeFormatter, ) else -> error("Unexpected state: $state") } }

Slide 43

Slide 43 text

Po rt folioPresenterTest class PortfolioPresenterTest { private val portfolioRepository = FakePortfolioRepository() @Test fun `loads portfolio`() = runBlocking { portfolioRepository.portfolio += Portfolio( positions = listOf( Position( stock = Stock( ticker = "TWTR", name = "Twitter, Inc.", currency = Currency.getInstance("USD"), currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25, ),

Slide 44

Slide 44 text

class PortfolioPresenterTest { private val portfolioRepository = FakePortfolioRepository() @Test fun `loads portfolio`() = runBlocking { portfolioRepository.portfolio += Portfolio( positions = listOf( Position( stock = Stock( ticker = "TWTR", name = "Twitter, Inc.", currency = Currency.getInstance("USD"), currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25, ), ), )

Slide 45

Slide 45 text

currentPriceCents = 3833, currentPriceTime = Instant.fromEpochSeconds(1636657688), ), quantity = 25, ), ), ) makePresenter().test { assertThat(awaitItem()).isEqualTo(PortfolioModel.Loading) assertThat(awaitItem()).isEqualTo( PortfolioModel.Loaded( positions = listOf( PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ), ), ), ) } }

Slide 46

Slide 46 text

package app.cash.turbine public suspend fun Flow.test( timeout: Duration? = null, validate: suspend ReceiveTurbine.() -> Unit, )

Slide 47

Slide 47 text

Slide 48

Slide 48 text

Slide 49

Slide 49 text

quantity = 25, ), ), ), ) } } private fun makePresenter(): Flow { return PresenterModule.providePortfolioPresenter( clock = Immediate, portfolioRepository = portfolioRepository, locale = Locale.US, timeZone = TimeZone.getTimeZone("America/New_York"), ) } }

Slide 50

Slide 50 text

No content

Slide 51

Slide 51 text

Integration tests • Not hermetic • Test integration assumptions • Slower feedback Photo by Taylor Vick on Unsplash

Slide 52

Slide 52 text

Po rt folioApi interface PortfolioApi { @GET("portfolio.json") suspend fun getPortfolio(): Portfolio }

Slide 53

Slide 53 text

Po rt folioApiTest @RunWith(TestParameterInjector::class) class PortfolioApiTest( @TestParameter private val fixture: Fixture, ) { @Test fun test() = runBlocking { val server = MockWebServer() server.enqueue( MockResponse() .setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path = "/").toString(), ) try {

Slide 54

Slide 54 text

No content

Slide 55

Slide 55 text

enum class Fixture( val responsePath: String, val expectedResponse: Portfolio?, val expectedExceptionType: KClass?, ) { DEFAULT( responsePath = "/portfolio.json", expectedResponse = Portfolio(stocks = Network.all), expectedExceptionType = null, ), EMPTY( responsePath = "/portfolio-empty.json", expectedResponse = Portfolio(stocks = emptyList()), expectedExceptionType = null, ), MALFORMED( responsePath = "/portfolio-malformed.json", expectedResponse = null, expectedExceptionType = JsonEncodingException::class, ) }

Slide 56

Slide 56 text

No content

Slide 57

Slide 57 text

No content

Slide 58

Slide 58 text

@RunWith(TestParameterInjector::class) class PortfolioApiTest( @TestParameter private val fixture: Fixture, ) { @Test fun test() = runBlocking { val server = MockWebServer() server.enqueue( MockResponse() .setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path = "/").toString(), ) try { val response = portfolioApi.getPortfolio() assertThat(response).isEqualTo(fixture.expectedResponse) } catch (t: Throwable) { assertThat(t).isInstanceOf(fixture.expectedExceptionType?.java)

Slide 59

Slide 59 text

@Test fun test() = runBlocking { val server = MockWebServer() server.enqueue( MockResponse() .setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path = "/").toString(), ) try { val response = portfolioApi.getPortfolio() assertThat(response).isEqualTo(fixture.expectedResponse) } catch (t: Throwable) { assertThat(t).isInstanceOf(fixture.expectedExceptionType?.java) } }

Slide 60

Slide 60 text

.setBody(javaClass.getResource(fixture.responsePath)!!.readText()), ) server.start() val portfolioApi = NetworkModule.providePortfolioApi( baseUrl = server.url(path = "/").toString(), ) try { val response = portfolioApi.getPortfolio() assertThat(response).isEqualTo(fixture.expectedResponse) } catch (t: Throwable) { assertThat(t).isInstanceOf(fixture.expectedExceptionType?.java) } }

Slide 61

Slide 61 text

PositionView

Slide 62

Slide 62 text

No content

Slide 63

Slide 63 text

PositionViewTest @RunWith(TestParameterInjector::class) class PositionViewTest( @TestParameter private val theme: Theme, ) { @get:Rule val paparazzi = Paparazzi(deviceConfig = PIXEL_5) @Test fun default() { val model = PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ) paparazzi.snapshot { StonksTestingTheme(darkTheme = theme == Theme.DARK) { Surface(

Slide 64

Slide 64 text

@RunWith(TestParameterInjector::class) class PositionViewTest( @TestParameter private val theme: Theme, ) { @get:Rule val paparazzi = Paparazzi(deviceConfig = PIXEL_5) @Test fun default() { val model = PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ) paparazzi.snapshot { StonksTestingTheme(darkTheme = theme == Theme.DARK) { Surface( modifier = Modifier.fillMaxSize(), color = MaterialTheme.colors.background, ) { PositionView(model) } }

Slide 65

Slide 65 text

@Test fun default() { val model = PortfolioModel.PositionModel( ticker = "TWTR", name = "Twitter, Inc.", currentPrice = "$38.33", currentPriceTime = "Nov 11, 2021, 2:08 PM", quantity = 25, ) paparazzi.snapshot { StonksTestingTheme(darkTheme = theme == Theme.DARK) { Surface( modifier = Modifier.fillMaxSize(), color = MaterialTheme.colors.background, ) { PositionView(model) } } } }

Slide 66

Slide 66 text

./gradlew ui:recordPaparazziDebug

Slide 67

Slide 67 text

No content

Slide 68

Slide 68 text

./gradlew ui:verifyPaparazziDebug

Slide 69

Slide 69 text

No content

Slide 70

Slide 70 text

End-to-end tests • Test complete fl ows • Run on real hardware • Slowest feedback • Multiple sources of fl akiness Photo by Adi Goldstein on Unsplash

Slide 71

Slide 71 text

Po rt folio fi ltering

Slide 72

Slide 72 text

Espresso

Slide 73

Slide 73 text

No content

Slide 74

Slide 74 text

Test robots

Slide 75

Slide 75 text

Po rt folioRobot class PortfolioRobot( private val testRule: ComposeContentTestRule ) { fun filter(filterTerm: String) { testRule.waitUntilExists(isFilterInputField()) testRule.onFilterInputField().performTextInput(filterTerm) } private fun SemanticsNodeInteractionsProvider.onFilterInputField() = onNode(isFilterInputField()) private fun isFilterInputField() = hasSetTextAction() and hasText("Type to filter") fun checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() }

Slide 76

Slide 76 text

class PortfolioRobot( private val testRule: ComposeContentTestRule ) { fun filter(filterTerm: String) { testRule.waitUntilExists(isFilterInputField()) testRule.onFilterInputField().performTextInput(filterTerm) } private fun SemanticsNodeInteractionsProvider.onFilterInputField() = onNode(isFilterInputField()) private fun isFilterInputField() = hasSetTextAction() and hasText("Type to filter") fun checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() } } fun ComposeContentTestRule.portfolio(

Slide 77

Slide 77 text

} private fun SemanticsNodeInteractionsProvider.onFilterInputField() = onNode(isFilterInputField()) private fun isFilterInputField() = hasSetTextAction() and hasText("Type to filter") fun checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() } } fun ComposeContentTestRule.portfolio( body: PortfolioRobot.() -> Unit ) = PortfolioRobot(this).body()

Slide 78

Slide 78 text

private fun isFilterInputField() = hasSetTextAction() and hasText("Type to filter") fun checkHasEntry(entryText: String) { testRule.onNode(hasText(entryText)).assertIsDisplayed() } } fun ComposeContentTestRule.portfolio( body: PortfolioRobot.() -> Unit ) = PortfolioRobot(this).body()

Slide 79

Slide 79 text

@RunWith(AndroidJUnit4::class) class FilterPortfolioTest { @get:Rule val activityRule = createAndroidComposeRule() @Test fun filterPortfolioByStockTicker() = with(activityRule) { portfolio { filter("JNJ") checkHasEntry("Johnson & Johnson") } } @Test fun filterPortfolioByStockName() = with(activityRule) { portfolio { filter("Under Armour") checkHasEntry("UA") } } FilterPo rt folioTest

Slide 80

Slide 80 text

Slide 81

Slide 81 text

Firebase Test Lab

Slide 82

Slide 82 text

No content

Slide 83

Slide 83 text

No content

Slide 84

Slide 84 text

fladle { serviceAccountCredentials.set( project.layout.projectDirectory.file( “fladle-auth.json" ) ) variant.set("debug") performanceMetrics.set(false) devices.set( listOf( mapOf("model" to "bluejay", "version" to "32"), // Pixel 6a mapOf("model" to "Nexus5X", "version" to "24"), ) ) environmentVariables.set( mapOf( "failureScreenshots" to "true", ) ) }

Slide 85

Slide 85 text

firebase: name: Run UI tests in Firebase Test Lab runs-on: ubuntu-latest steps: - name: Checkout the repo. … - name: Set up JDK 11. … - name: Setup Gradle. … - name: Create Fladle authentication JSON file env: GCLOUD_AUTH: ${{ secrets.GCLOUD_AUTH }} run: echo $GCLOUD_AUTH >> app/fladle-auth.json - name: Run Instrumentation Tests in Firebase Test Lab run: | ./gradlew app:assembleDebug app:assembleDebugAndroidTest ./gradlew runFlank

Slide 86

Slide 86 text

No content

Slide 87

Slide 87 text

Conclusion • Testing tooling is awesome! • Prioritize smaller tests, but write all. • Use open source! Photo by Girl with red hat on Unsplash

Slide 88

Slide 88 text

@egorand Thanks!