inventory', () => { context('When user purchase ONE item', () => { beforeEach(function(){ /* * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! * ! re-seed DB to restore testing data each time! * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ cy.exec('node ../seed/product-seeder') // login before each test cy.loginByForm('
[email protected]', 'test1234') cy.server() cy.route('GET', '/prod').as('prodPage') }) it('Then he can checkout purchased item successfully', function(){ // buy a game cy.visit('/') cy.contains('div', 'Inversion').find('a.cart').click() // checkout cy.checkout('QA5') // wait process & do assertion cy.wait('@prodPage') cy.get('div#success').contains('Successfully bought') cy.contains('div', 'Inversion').find('div.stock').should('contain', 'Stock: 4') }) }) }) describe('Given all products have sufficient inventory', () => { context('When user purchase ONE item', () => { beforeEach(function(){ // login before each test cy.loginByForm('
[email protected]', 'test1234') cy.server() cy.route('GET', '/prod').as('prodPage') }) it('Then he can checkout purchased item successfully', function(){ // buy a game cy.visit('/') cy.contains('div', 'Inversion').find('a.cart').click() // checkout cy.checkout('QA5') // wait process & do assertion cy.wait('@prodPage') cy.get('div#success').contains('Successfully bought') cy.contains('div', 'Inversion').find('div.stock').should('contain', 'Stock: 4') }) /* * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! * ! run the same spec multiple times will fail * ! -> not isolated from itself * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ }) }) Isolation from itself: - Each test case is repeatable (run multiple times) - Run multiple times against the same env - Easy debuging, don't need to cleanup/restore manually describe('Given all products have sufficient inventory', () => { context('When user purchase ONE item', () => { beforeEach(function () { /* * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! * ! re-seed DB to restore testing data each time! * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ cy.exec('node ../seed/product-seeder') // login before each test cy.loginByForm('
[email protected]', 'test1234') cy.server() cy.route('GET', '/prod').as('prodPage') }) it('can checkout one purchase successfully', function () { // buy a game cy.visit('/') cy.contains('div', 'Inversion').find('a.cart').click() // checkout cy.checkout('QA5') // wait process & do assertion cy.wait('@prodPage') cy.get('div#success').contains('Successfully bought') cy.contains('div', 'Inversion').find('div.stock').should('contain', 'Stock: 4') }) }) }) describe('Given all products have sufficient inventory', () => { context('When user purchase TWO item', () => { beforeEach(function () { cy.exec('node ../seed/product-seeder') // login before each test cy.loginByForm('
[email protected]', 'test1234') ... .... }) it('can checkout all items successfully', function () { // buy a game cy.visit('/') // buy one cy.contains('div', 'Inversion').find('a.cart').click() // buy 2nd one cy.contains('div', 'Borderlands').find('a.cart').click() cy.visit('/shopping-cart') // checkout cy.checkout('QA5') // wait process & do assertion cy.wait('@prodPage') cy.get('div#success').contains('Successfully bought') cy.contains('div', 'Inversion').find('div.stock').should(($div) => { const text = $div.text() expect(text).to.eq('Stock: 4') }) cy.contains('div', 'Borderlands').find('div.stock').should(($div) => { const text = $div.text() expect(text).to.eq('Stock: 4') }) }) }) }) Isolation from others: - NO dependency between tests and their results - NOT depends on the running order - Test against the same entity might impact each other - Reset database seems to be a solution ??? - Reset data is destructive - Prohibit parallel execution - Provision multiple set of environment? - cost $$ - need to merge test result describe('Given all products have sufficient inventory', () => { context('When user purchase TWO item', () => { const firstProduct = new Product('Test Product NO1') const secondProduct = new Product('Test Product NO2') let testContext = new TestContext() testContext.products.push(firstProduct, secondProduct) before(function () { testContext.products.forEach((prod) => { cy.task('createDocument', {collectionName: 'products', filter: prod}) }) }) after(function () { // since create diff product each run, you can leave it or not, it depends~ cy.task('deleteDocuments', {collectionName: 'products', filter: {title: {$regex: '^Test Product'}}}) }) it('can checkout all items successfully', function () { cy.visit('/') cy.contains('div', testContext.products[0].title).find('a.cart').click() // buy 2nd one cy.contains('div', testContext.products[1].title).find('a.cart').click() // checkout cy.checkout(testContext.userName) cy.get('div#success').contains('Successfully bought') cy.contains('div', testContext.products[0].title).find('div.stock').should(($div) => { const text = $div.text() expect(text).to.eq('Stock: 49') }) ... ... }) }) }) import ShortUniqueId from 'short-unique-id' class Product { constructor(name) { this.title = `${name}_${new ShortUniqueId().randomUUID(6)}`, //<= alias name this.description = `Test description for ${this.title} product! rlVyLUWQr9EU2oO3SZKe-ihpfDrsokUb8nwVmeUU7-oS2S9kzBGw' this.imagePath = 'images/test_the_test.png', this.price = 22, //<== assign default values this.stock = 50 //<== assign default values } } class TestContext { constructor() { this.products = [], //<== assign default values // for user this.userName = 'defaultUser', //<== assign default values this.userEmail = '
[email protected]', //<== assign default values this.userPassword = 'test1234' //<== assign default values } } TestContext: - Find 'functional entities', for every test-case - Shopping -> create new product - Github -> create new account & repo - Event -> crete new campaign - Give alias name for the func entity - TC create/teardown it's own testing data - Easy parallel execution context('Test purchasing when short of inventory', () => { let testContext = new TestContext() let outOfStock = new Product('No Enough Inventory') outOfStock.stock = 0 // <== clearly shows the purpose that you want to TEST! testContext.products.push(outOfStock) testContext.userName = 'ValidUser' // <== shows the important variation of this test case! testContext.userPassword = 'GoodPassword' ... ... it('add to cart should shows error message', function(){ // buy a game cy.visit('/') cy.contains('div', testContext.products[0].title).find('a.cart').click() // checkout // cy.checkout(testContext.userName) // wait process & do assertion cy.wait('@prodPage') cy.get('div#success').contains('short of inventory') }) }) TestContext with Default Object: - Clearly show test intentions! - Test case shows intention is important! - Team will result in similar coding pattern / structure context('Stub Response Data', () => { it('cy.route() - route responses to matching requests', () => { // https://on.cypress.io/route cy.server() cy.fixture('example.json').as('fakeResp') // Stub a response to GET /prod cy.route({ method: 'GET', url: '/prod', response: '@fakeResp' }).as('getComment') cy.visit('http://localhost:3000') cy.wait('@getComment') // UI displayed correctly according api response cy.get('div.price').should('have.length', 6) // linkage attributes are correctly set cy.get('a.cart').first().invoke('attr', 'href') .should('contain', '5c4a83c471d09c3125654816') }) }) account: ? Johnny? ==> ? Johnny_4534031? book: ? DevOps 101? ==> ? DevOps 101_1234567? BRYAN LIU | June 25, 2020 Test Automation Test Isolation and TestContext Goals of Test Automation Git Repo: https://git.linecorp.com/TW-QA/test-isolation 3 Levels of test isolation 1. Isolating test cases from themselves (repeatable) 2. Isolating test cases from each other 3. Isolating the System under test Run 20 times before asking PR merge - (*) Stability 1st - 20 times in a row Run Smoke / AC before manual regression (all env) - Don't waste time on testing unmature delivery Reduce manual regression efforts - Reduce # of test case automation - Focus on major functionalities and ROI (confidence) - Discuss automation needed for defect & bug found - Build tools to seepd up manual testing - (*) Single source of test cases - (*) Automated coverage rate of AC / high priority test cases Daily / weekly run for other integration regression - Save time on failure build checking - Or just complete this regression before release Run AC in each commit (higher goal) - Run smoke test before full acceptance tests More process & monitoring automation - Deployment pipeline - Flacky analysis - Performance / stress in delivery pipeline All Automated Tests AC Smoke Tests All Test Cases Per Commit Daily/Weekly Per Release Frequency My Service (SUT) Event Bus Test Suite Assertion Assertion output Stub 3rd Pty Service Tests send prepared input msgs of System A & B System A System B Insert test data Stub / Mock / Fake BFF / Express Vue App Mock (Talkback JS) X X Browser My Application http://abc.xzy.com iFrame iFrame cy.route() Cypress (Tests) Application X Database Full Set Database . . . In browser mock/stub, ex: Polly.js Cypress.io Service Virtualization Tools (corss-platform): Ruby: VCR GO: Hoverfly JS: Talkback, mountebank Java: WireMock, betamax, Moco TestConteners, ex: DB: MySQL, Mongo Message Queue: Kafka Test Runtime: browser, Selenium System / Component Level Test Isolations - TestContainers 3 2 1 1 3 2 https://www.testcontainers.org/
bryan0817
rmakiyama
marcoroth
kaspth
yosuke_furukawa
coe401_
prof18
medley
hacarus
mkdev10
genetec
yossydev
mitsuhiko
mthomps
samanthasiow
aleyda
garrettdimon
samlambert
kneath
sstephenson
marcduiker
eileencodes
dougneiner
productmarketing
myddelton
