Next Level Testing James Saryerwinnie @jsaryer PyCon 2017 

Property based testing Fuzz testing Stress testing Mutation testing Agenda 

What it is, why you’d use it Real world examples Tips on project integration 

Property based testing Fuzz testing Stress testing Mutation testing 

abs.py def test_abs_always_positive(): assert abs(1) == 1 def test_abs_negative(): assert abs(-1) == 1 def test_abs_zero_is_zero(): assert abs(0) == 0 Property example 

abs.py def test_abs_always_positive(): assert abs(1) == 1 def test_abs_negative(): assert abs(-1) == 1 def test_abs_zero_is_zero(): assert abs(0) == 0 Property example Can we do better? For all integers i, abs(i) should always be greater than or equal to 0. 

abs.py def test_abs_always_positive(): assert abs(1) == 1 def test_abs_negative(): assert abs(-1) == 1 def test_abs_zero_is_zero(): assert abs(0) == 0 Property example import random def test_abs(): for _ in range(1000): n = random.randint( -sys.maxint, sys.maxint) assert abs(n) >= 0 

Property Based Testing 1. Write assertions about the properties of a function 2. Generate random input data that violates these assertions 3. Minimize the example to be as simple as possible 

Hypothesis • Integrates with unittest/pytest • Powerful test data generation • Generates minimal test cases on failure • pip install hypothesis 

abs.py import random def test_abs(): for _ in range(1000): n = random.randint( -sys.maxint, sys.maxint) assert abs(n) >= 0 Property example from hypothesis import given import hypothesis.strategies as s @given(s.integers()) def test_abs(x): assert abs(x) >= 0 

abs.py import random def test_abs(): for _ in range(1000): n = random.randint( -sys.maxint, sys.maxint) assert abs(n) >= 0 Property example from hypothesis import given import hypothesis.strategies as s @given(s.integers()) def test_abs(x): assert abs(x) >= 0 

Strategies >>> import hypothesis.strategies as s >>> test_data = s.integers() >>> test_data.example() -214460024625629886 >>> test_data.example() 288486085571772 >>> test_data.example() -2199980509 >>> test_data.example() -207377623894 >>> test_data.example() 4588868 

Strategies >>> test_data = s.lists( ... (s.text() | s.integers() | s.booleans()), ... max_size=3) >>> test_data.example() [] >>> test_data.example() [True, -1185002, True] >>> test_data.example() [False, False, 0] >>> test_data.example() [32284634116830856916480955L, u''] >>> test_data.example() [3453158272709441024718420603051L, u'\n\x05\xc3\x123\ue59f', True] >>> test_data.example() [False, -122986972202643871446411L, True] 

Strategies >>> test_data = s.lists( ... (s.text() | s.integers() | s.booleans()), ... max_size=3) >>> test_data.example() [] >>> test_data.example() [True, -1185002, True] >>> test_data.example() [False, False, 0] >>> test_data.example() [32284634116830856916480955L, u''] >>> test_data.example() [3453158272709441024718420603051L, u'\n\x05\xc3\x123\ue59f', True] >>> test_data.example() [False, -122986972202643871446411L, True] 

Strategies >>> test_data = s.lists( ... (s.text() | s.integers() | s.booleans()), ... max_size=3) >>> test_data.example() [] >>> test_data.example() [True, -1185002, True] >>> test_data.example() [False, False, 0] >>> test_data.example() [32284634116830856916480955L, u''] >>> test_data.example() [3453158272709441024718420603051L, u'\n\x05\xc3\x123\ue59f', True] >>> test_data.example() [False, -122986972202643871446411L, True] 

A Query Language for JSON import jmespath # .search(expression, input_data) jmespath.search(‘a.b', {'a': {'b': {'c': 'd'}}}) # {'c': 'd'} $ aws ec2 describe-instances --query 'Reservations[].Instances[].[InstanceId, State.Name]' - name: "Display all cluster names" debug: var=item with_items: "{{domain_definition|json_query('domain.cluster[*].name')}}" AWS CLI Ansible Python API 

jmespath.py from jmespath import lexer list(lexer.Lexer().tokenize('foo.bar')) JMESPath API [{'start': 0, 'end': 3, type': 'unquoted_identifier', 'value': 'foo'}, {'start': 3, 'end': 4, type': 'dot', 'value': '.'}, {'start': 4, 'end': 7, type': 'unquoted_identifier', 'value': 'bar'}, {'start': 7, 'end': 7, type': 'eof', 'value': ''}] 

test_lexer.py # For all of these tests they verify these properties: # either the operation succeeds or it raises a JMESPathError. # If any other exception is raised then we error out. @given(st.text()) def test_lexer_api(expr): try: tokens = list(lexer.Lexer().tokenize(expr)) except exceptions.JMESPathError as e: return except Exception as e: raise AssertionError("Non JMESPathError raised: %s" % e) Property tests for JMESPath Lexer 

test_lexer.py # For all of these tests they verify these properties: # either the operation succeeds or it raises a JMESPathError. # If any other exception is raised then we error out. @given(st.text()) def test_lexer_api(expr): try: tokens = list(lexer.Lexer().tokenize(expr)) except exceptions.JMESPathError as e: return except Exception as e: raise AssertionError("Non JMESPathError raised: %s" % e) Property tests for JMESPath Lexer Properties: The lexer must return a list of tokens or it must raise a JMESPathError exception 

test_lexer.py # For all of these tests they verify these properties: # either the operation succeeds or it raises a JMESPathError. # If any other exception is raised then we error out. @given(st.text()) def test_lexer_api(expr): … assert isinstance(tokens, list) # Token starting positions must be unique, can't have two # tokens with the same start position. start_locations = [t['start'] for t in tokens] assert len(set(start_locations)) == len(start_locations), ( "Tokens must have unique starting locations.") # Starting positions must be increasing (i.e sorted). assert sorted(start_locations) == start_locations, ( "Tokens must have increasing start locations.") # Last token is always EOF. assert tokens[-1]['type'] == 'eof' Property tests for JMESPath Lexer 

test_lexer.py # For all of these tests they verify these properties: # either the operation succeeds or it raises a JMESPathError. # If any other exception is raised then we error out. @given(st.text()) def test_lexer_api(expr): … assert isinstance(tokens, list) # Token starting positions must be unique, can't have two # tokens with the same start position. start_locations = [t['start'] for t in tokens] assert len(set(start_locations)) == len(start_locations), ( "Tokens must have unique starting locations.") # Starting positions must be increasing (i.e sorted). assert sorted(start_locations) == start_locations, ( "Tokens must have increasing start locations.") # Last token is always EOF. assert tokens[-1]['type'] == 'eof' Property tests for JMESPath Lexer 

test_lexer.py # For all of these tests they verify these properties: # either the operation succeeds or it raises a JMESPathError. # If any other exception is raised then we error out. @given(st.text()) def test_lexer_api(expr): … assert isinstance(tokens, list) # Token starting positions must be unique, can't have two # tokens with the same start position. start_locations = [t['start'] for t in tokens] assert len(set(start_locations)) == len(start_locations), ( "Tokens must have unique starting locations.") # Starting positions must be increasing (i.e sorted). assert sorted(start_locations) == start_locations, ( "Tokens must have increasing start locations.") # Last token is always EOF. assert tokens[-1]['type'] == 'eof' Property tests for JMESPath Lexer 

jmespath/lexer.py Lexer Bug Traceback (most recent call last): File "", line 1, in File "jmespath/__init__.py", line 8, in compile return parser.Parser().parse(expression) File "jmespath/parser.py", line 87, in parse parsed_result = self._do_parse(expression) File "jmespath/parser.py", line 95, in _do_parse return self._parse(expression) File "jmespath/parser.py", line 110, in _parse parsed = self._expression(binding_power=0) File "jmespath/parser.py", line 125, in _expression while binding_power < self.BINDING_POWER[current_token]: KeyError: 'unknown' 

jmespath/lexer.py if self._current == '=': yield self._match_or_else(expected='=', match_type='eq', else_type='unknown') Lexer Bug Traceback (most recent call last): File "", line 1, in File "jmespath/__init__.py", line 8, in compile return parser.Parser().parse(expression) File "jmespath/parser.py", line 87, in parse parsed_result = self._do_parse(expression) File "jmespath/parser.py", line 95, in _do_parse return self._parse(expression) File "jmespath/parser.py", line 110, in _parse parsed = self._expression(binding_power=0) File "jmespath/parser.py", line 125, in _expression while binding_power < self.BINDING_POWER[current_token]: KeyError: 'unknown' 

jmespath/lexer.py if self._current == '=': yield self._match_or_else(expected='=', match_type='eq', else_type='unknown') Lexer Bug Traceback (most recent call last): File "", line 1, in File "jmespath/__init__.py", line 8, in compile return parser.Parser().parse(expression) File "jmespath/parser.py", line 87, in parse parsed_result = self._do_parse(expression) File "jmespath/parser.py", line 95, in _do_parse return self._parse(expression) File "jmespath/parser.py", line 110, in _parse parsed = self._expression(binding_power=0) File "jmespath/parser.py", line 125, in _expression while binding_power < self.BINDING_POWER[current_token]: KeyError: 'unknown' Not a valid token type 

jmespath/lexer.py if self._current == '=': yield self._match_or_else(expected='=', match_type='eq', else_type='unknown') Lexer Bug Traceback (most recent call last): File "", line 1, in File "jmespath/__init__.py", line 8, in compile return parser.Parser().parse(expression) File "jmespath/parser.py", line 87, in parse parsed_result = self._do_parse(expression) File "jmespath/parser.py", line 95, in _do_parse return self._parse(expression) File "jmespath/parser.py", line 110, in _parse parsed = self._expression(binding_power=0) File "jmespath/parser.py", line 125, in _expression while binding_power < self.BINDING_POWER[current_token]: KeyError: 'unknown' Not a valid token type 

if self._current == '=': yield self._match_or_else(expected='=', match_type='eq', else_type='unknown') jmespath/lexer.py Lexer Bug if self._next() == '=': yield {'type': 'eq', 'value': '==', 'start': self._position - 1, 'end': self._position} self._next() else: raise LexerError( lexer_position=self._position - 1, lexer_value='=', message="Unknown token '='") 

Tip: Design Feedback • Nothing explicitly linking token types from the lexer to parser • {'type': tokens.NOT_EQUALS, ...} 

.travis.yml script: - cd tests/ && nosetests --with-coverage --cover-package jmespath . - if [[ $TRAVIS_PYTHON_VERSION != '2.6' ]]; \ then JP_MAX_EXAMPLES=10000 nosetests ../extra/test_hypothesis.py; fi CI Integration MAX_EXAMPLES = int(os.environ.get('JP_MAX_EXAMPLES', 1000)) BASE_SETTINGS = { 'max_examples': MAX_EXAMPLES, 'suppress_health_check': [HealthCheck.too_slow], } @settings(**BASE_SETTINGS) @given(st.text()) def test_lexer_api(expr): ... 

Property based testing Fuzz testing Stress testing Mutation testing 

fuzzing.py while True: fuzzing_input = generate_random_input() try: code_under_test(fuzzing_input) except AllowedExceptions: pass except: # An unexpected exception was raised. report_fuzzing_failure(fuzzing_input) Simplified fuzzing 

bug.py def bug(x): if not isinstance(x, str): return if not len(x) == 5: return if x[0] == 'b': if x[1] == 'u': if x[2] == 'g': if x[3] == 'g': if x[4] == 'y': raise RuntimeError("BUGGY!") AFL Fuzzing Example 

bug.py def brute_force_fuzz(): # This will try all sequences of itertools.product # of length 1, then length 2, etc: # 0, 1, 2, ..., a, b, c, ..., x, y, z # 00, 01, 02, ..., aa, ab, ac, .., zx, zy, zz for i in range(1, 100): for letters in itertools.product(string.printable, repeat=i): test_string = ''.join(letters) bug(test_string) def bug(x): if not isinstance(x, str): return if not len(x) == 5: return if x[0] == 'b': if x[1] == 'u': if x[2] == 'g': if x[3] == 'g': if x[4] == 'y': raise RuntimeError(“BUGGY!") AFL Fuzzing Example 

def brute_force_fuzz(): # This will try all sequences of itertools.product # of length 1, then length 2, etc: # 0, 1, 2, ..., a, b, c, ..., x, y, z # 00, 01, 02, ..., aa, ab, ac, .., zx, zy, zz for i in range(1, 100): for letters in itertools.product(string.printable, repeat=i): test_string = ''.join(letters) bug(test_string) def bug(x): if not isinstance(x, str): return if not len(x) == 5: return if x[0] == 'b': if x[1] == 'u': if x[2] == 'g': if x[3] == 'g': if x[4] == 'y': raise RuntimeError(“BUGGY!") bug.py AFL Fuzzing Example Brute force. real 8m27.541s user 8m27.524s sys 0m0.016s 

AFL - American Fuzzy Lop • Coverage guided genetic fuzzer • Fast • Simple to use 

python-afl • Create a python program that reads input from stdin • Raise an exception on error cases (“crashes”) • Create a set of sample input files, one sample input per file 

bug.py def bug(x): if not isinstance(x, str): return if not len(x) == 5: return if x[0] == 'b': if x[1] == 'u': if x[2] == 'g': if x[3] == 'g': if x[4] == 'y': raise RuntimeError(“BUGGY!") def main(): bug(sys.stdin.read()) import afl while afl.loop(): main() AFL Fuzzing Example 

$ cat corpus/a a $ py-afl-fuzz -o results/ -i corpus/ -- $(which python) bug.py Running python-afl Sample input Crashes 

$ tree results/ results/ ├── crashes │ ├── id:000000,sig:10,src:000005,op:arith8,pos:4,val:+23 │ └── README.txt ├── fuzz_bitmap ├── fuzzer_stats ├── hangs ├── plot_data └── queue ├── id:000000,orig:a ├── id:000001,src:000000,op:havoc,rep:16,+cov ├── id:000002,src:000001,op:havoc,rep:16,+cov ├── id:000003,src:000002,op:arith8,pos:1,val:+19,+cov ├── id:000004,src:000003,op:arith8,pos:2,val:+5,+cov └── id:000005,src:000004,op:arith8,pos:3,val:+5,+cov AFL Fuzzing Example 

$ tree results/ results/ ├── crashes │ ├── id:000000,sig:10,src:000005,op:arith8,pos:4,val:+23 │ └── README.txt ├── fuzz_bitmap ├── fuzzer_stats ├── hangs ├── plot_data └── queue ├── id:000000,orig:a ├── id:000001,src:000000,op:havoc,rep:16,+cov ├── id:000002,src:000001,op:havoc,rep:16,+cov ├── id:000003,src:000002,op:arith8,pos:1,val:+19,+cov ├── id:000004,src:000003,op:arith8,pos:2,val:+5,+cov └── id:000005,src:000004,op:arith8,pos:3,val:+5,+cov AFL Fuzzing Example buggy 

$ tree results/ results/ ├── crashes │ ├── id:000000,sig:10,src:000005,op:arith8,pos:4,val:+23 │ └── README.txt ├── fuzz_bitmap ├── fuzzer_stats ├── hangs ├── plot_data └── queue ├── id:000000,orig:a ├── id:000001,src:000000,op:havoc,rep:16,+cov ├── id:000002,src:000001,op:havoc,rep:16,+cov ├── id:000003,src:000002,op:arith8,pos:1,val:+19,+cov ├── id:000004,src:000003,op:arith8,pos:2,val:+5,+cov └── id:000005,src:000004,op:arith8,pos:3,val:+5,+cov AFL Fuzzing Example bbbbb bubbb bugbb buggb buggy 

AWS Command Line Interface aws ec2 run-instances --image-id ami-abc12345 --count 1 \ --instance-type t2.micro --key-name MyKeyPair \ --subnet-id subnet-6e7f829e \ --tag-specifications \ 'ResourceType=instance,Tags=[{Key=webserver,Value=prod}]' \ 'ResourceType=volume,Tags=[{Key=cost-center,Value=cc123}]' 

AWS Command Line Interface aws ec2 run-instances --image-id ami-abc12345 --count 1 \ --instance-type t2.micro --key-name MyKeyPair \ --subnet-id subnet-6e7f829e \ --tag-specifications \ 'ResourceType=instance,Tags=[{Key=webserver,Value=prod}]' \ 'ResourceType=volume,Tags=[{Key=cost-center,Value=cc123}]' Shorthand Syntax 

from awscli.shorthand import ShorthandParser from awscli.shorthand import ShorthandParseError def main(): try: ShorthandParser().parse(sys.stdin.read()) except ShorthandParseError: pass import afl while afl.loop(): main() Bugs from python-afl sh-fuzz 

foo=[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ Bugs from python-afl ... File "/usr/local/lib/python2.7/site-packages/awscli/shorthand.py", line 242, in _explicit_list self._expect('[', consume_whitespace=True) File "/usr/local/lib/python2.7/site-packages/awscli/shorthand.py", line 313, in _expect self._consume_whitespace() RuntimeError: maximum recursion depth exceeded jp-fuzz 

$ py-afl-fuzz -m 100 -o results/ -i corpus/ -M fuzzer01 -- $(which python) sh-fuzz & $ py-afl-fuzz -m 100 -o results/ -i corpus/ -S fuzzer02 -- $(which python) sh-fuzz & $ py-afl-fuzz -m 100 -o results/ -i corpus/ -S fuzzer03 -- $(which python) sh-fuzz & $ py-afl-fuzz -m 100 -o results/ -i corpus/ -S fuzzer04 -- $(which python) sh-fuzz & Tips: python-afl Multi-core -m Max memory allowed for program -o Sync directory -i Sample inputs -M Main process -S Secondary process (random tweaks) import afl while afl.loop(): main() Persistent mode 

Property based testing Fuzz testing Stress testing Mutation testing 

Stress Testing • Same input, difference execution order due to threading • Assert properties and invariants about the code • Catch synchronization issues 

import boto3 s3 = boto3.client('s3') s3.download_fileobj('bucket', 'key', stream) S3 streaming downloads $ aws s3 cp s3://bucket/key - | gzip … 

Sliding Window Semaphore 

Sliding Window Semaphore sem.release('a.txt', 13) 

Sliding Window Semaphore sem.release('a.txt', 12) 

Sliding Window Semaphore sem.release('a.txt', 21) 

Sliding Window Semaphore sem.release('a.txt', 11) 

Sliding Window Semaphore sem.release('a.txt', 10) sem.acquire(‘a.txt’) 

Sliding Window Semaphore 

Sliding Window Semaphore sem.acquire(‘a.txt’) 33 

Sliding Window Semaphore sem.acquire(‘a.txt’) 34 

Sliding Window Semaphore sem.acquire(‘a.txt’) 35 

test_s3.py def test_blocking_stress(self): sem = SlidingWindowSemaphore(5) num_threads = 10 num_iterations = 50 def acquire(): for _ in range(num_iterations): num = sem.acquire('a', blocking=True) time.sleep(0.001) sem.release('a', num) for i in range(num_threads): t = threading.Thread(target=acquire) self.threads.append(t) self.start_threads() self.join_threads() # Should have all the available resources freed. self.assertEqual(sem.current_count(), 5) # Should have acquired num_threads * num_iterations self.assertEqual(sem.acquire('a', blocking=False), num_threads * num_iterations) Multithreaded Stress Test 

test_s3.py def test_blocking_stress(self): sem = SlidingWindowSemaphore(5) num_threads = 10 num_iterations = 50 def acquire(): for _ in range(num_iterations): num = sem.acquire('a', blocking=True) time.sleep(0.001) sem.release('a', num) for i in range(num_threads): t = threading.Thread(target=acquire) self.threads.append(t) self.start_threads() self.join_threads() # Should have all the available resources freed. self.assertEqual(sem.current_count(), 5) # Should have acquired num_threads * num_iterations self.assertEqual(sem.acquire('a', blocking=False), num_threads * num_iterations) Multithreaded Stress Test 

test_s3.py def test_blocking_stress(self): sem = SlidingWindowSemaphore(5) num_threads = 10 num_iterations = 50 def acquire(): for _ in range(num_iterations): num = sem.acquire('a', blocking=True) time.sleep(0.001) sem.release('a', num) for i in range(num_threads): t = threading.Thread(target=acquire) self.threads.append(t) self.start_threads() self.join_threads() # Should have all the available resources freed. self.assertEqual(sem.current_count(), 5) # Should have acquired num_threads * num_iterations self.assertEqual(sem.acquire('a', blocking=False), num_threads * num_iterations) Multithreaded Stress Test 

test_s3.py def test_blocking_stress(self): sem = SlidingWindowSemaphore(5) num_threads = 10 num_iterations = 50 def acquire(): for _ in range(num_iterations): num = sem.acquire('a', blocking=True) time.sleep(0.001) sem.release('a', num) for i in range(num_threads): t = threading.Thread(target=acquire) self.threads.append(t) self.start_threads() self.join_threads() # Should have all the available resources freed. self.assertEqual(sem.current_count(), 5) # Should have acquired num_threads * num_iterations self.assertEqual(sem.acquire('a', blocking=False), num_threads * num_iterations) Multithreaded Stress Test 

test_s3.py def test_blocking_stress(self): sem = SlidingWindowSemaphore(5) num_threads = 10 num_iterations = 50 def acquire(): for _ in range(num_iterations): num = sem.acquire('a', blocking=True) time.sleep(0.001) sem.release('a', num) for i in range(num_threads): t = threading.Thread(target=acquire) self.threads.append(t) self.start_threads() self.join_threads() # Should have all the available resources freed. self.assertEqual(sem.current_count(), 5) # Should have acquired num_threads * num_iterations self.assertEqual(sem.acquire('a', blocking=False), num_threads * num_iterations) Multithreaded Stress Test 

test_s3.py def test_blocking_stress(self): sem = SlidingWindowSemaphore(5) num_threads = 10 num_iterations = 50 def acquire(): for _ in range(num_iterations): num = sem.acquire('a', blocking=True) time.sleep(0.001) sem.release('a', num) for i in range(num_threads): t = threading.Thread(target=acquire) self.threads.append(t) self.start_threads() self.join_threads() # Should have all the available resources freed. self.assertEqual(sem.current_count(), 5) # Should have acquired num_threads * num_iterations self.assertEqual(sem.acquire('a', blocking=False), num_threads * num_iterations) Multithreaded Stress Test 

Property based testing Fuzz testing Stress testing Mutation testing 

example.py def function(x, add=True): if add: return {'add-result': x + 1} return {'subtract-result': x - 1} def test_add(): result = function(5, add=True) assert result == {'add-result': 6} def test_add_is_false(): result = function(5, add=False) assert 'subtract-result' in result Mutation testing 

example.py def function(x, add=True): if add: return {'add-result': x + 1} return {'subtract-result': x - 1} def test_add(): result = function(5, add=True) assert result == {'add-result': 6} def test_add_is_false(): result = function(5, add=False) assert 'subtract-result' in result Mutation testing • 100% line coverage • 100% branch coverage $ py.test --cov example --cov-report term-missing example.py collected 2 items example.py .. Name Stmts Miss Branch BrPart Cover Missing -------------------------------------------------------- example.py 10 0 2 0 100% ================= 2 passed in 0.02 seconds ================= 

example.py def function(x, add=True): if add: return {'add-result': x + 1} return {'subtract-result': x - 1} def test_add(): result = function(5, add=True) assert result == {'add-result': 6} def test_add_is_false(): result = function(5, add=False) assert 'subtract-result' in result Mutation testing • 100% line coverage • 100% branch coverage Doesn’t properly test add=False 

example.py def function(x, add=True): if add: return {'add-result': x + 1} return {'subtract-result': x - 1} def test_add(): result = function(5, add=True) assert result == {'add-result': 6} def test_add_is_false(): result = function(5, add=False) assert 'subtract-result' in result Mutation testing • 100% line coverage • 100% branch coverage Doesn’t properly test add=False LE T ’S I N T R O D U C E A B U G 

example.py def function(x, add=True): if add: return {'add-result': x + 1} return {'subtract-result': x --1} def test_add(): result = function(5, add=True) assert result == {'add-result': 6} def test_add_is_false(): result = function(5, add=False) assert 'subtract-result' in result Mutation testing Bug 

example.py def function(x, add=True): if add: return {'add-result': x + 1} return {'subtract-result': x --1} def test_add(): result = function(5, add=True) assert result == {'add-result': 6} def test_add_is_false(): result = function(5, add=False) assert 'subtract-result' in result Mutation testing The tests still pass $ py.test --cov example --cov-report term-missing example.py collected 2 items example.py .. Name Stmts Miss Branch BrPart Cover Missing -------------------------------------------------------- example.py 10 0 2 0 100% ================= 2 passed in 0.02 seconds ================= 

Mutation Testing • Modify the program in a small way • Run your test suite • If a test fails, success • If no tests pass, mutation survives. 

mutatation-test.sh $ pip install cosmic-ray $ cosmic-ray init --baseline=10 mysession jmespath -- tests/ $ cosmic-ray --verbose exec mysession # This will take a while … INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '21', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '22', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '23', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '24', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '25', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '26', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '27', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '28', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '29', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '30', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '31', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '32', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '33', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '34', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '35', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '36', 'unittest', '--', 'tests/'] INFO:cosmic_ray.tasks.worker:executing: ['cosmic-ray', 'worker', 'jmespath.parser', 'mutate_binary_operator', '37', 'unittest', '--', ‘tests/'] … $ cosmic-ray report mysession Using cosmic-ray for Mutation Testing 

mutatation-test.sh command: cosmic-ray worker jmespath.lexer mutate_binary_operator 48 nose -- tests/ --- mutation diff --- --- a/home/ec2-user/jmespath.py/jmespath/lexer.py +++ b/home/ec2-user/jmespath.py/jmespath/lexer.py @@ -30,7 +30,7 @@ next_char = self._next() if (next_char == ']'): self._next() - (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start + 2)}) + (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start - 2)}) elif (next_char == '?'): self._next() (yield {'type': 'filter', 'value': '[?', 'start': start, 'end': (start + 2)}) Using Mutation Testing 

mutatation-test.sh command: cosmic-ray worker jmespath.lexer mutate_binary_operator 48 nose -- tests/ --- mutation diff --- --- a/home/ec2-user/jmespath.py/jmespath/lexer.py +++ b/home/ec2-user/jmespath.py/jmespath/lexer.py @@ -30,7 +30,7 @@ next_char = self._next() if (next_char == ']'): self._next() - (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start + 2)}) + (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start - 2)}) elif (next_char == '?'): self._next() (yield {'type': 'filter', 'value': '[?', 'start': start, 'end': (start + 2)}) Using Mutation Testing ˘ 

mutatation-test.sh command: cosmic-ray worker jmespath.lexer mutate_binary_operator 48 nose -- tests/ --- mutation diff --- --- a/home/ec2-user/jmespath.py/jmespath/lexer.py +++ b/home/ec2-user/jmespath.py/jmespath/lexer.py @@ -30,7 +30,7 @@ next_char = self._next() if (next_char == ']'): self._next() - (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start + 2)}) + (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start - 2)}) elif (next_char == '?'): self._next() (yield {'type': 'filter', 'value': '[?', 'start': start, 'end': (start + 2)}) Using Mutation Testing ˘ Original code Mutation 

mutatation-test.sh command: cosmic-ray worker jmespath.lexer mutate_binary_operator 48 nose -- tests/ --- mutation diff --- --- a/home/ec2-user/jmespath.py/jmespath/lexer.py +++ b/home/ec2-user/jmespath.py/jmespath/lexer.py @@ -30,7 +30,7 @@ next_char = self._next() if (next_char == ']'): self._next() - (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start + 2)}) + (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start - 2)}) elif (next_char == '?'): self._next() (yield {'type': 'filter', 'value': '[?', 'start': start, 'end': (start + 2)}) Using Mutation Testing ˘ Original code Mutation But the tests still pass! 

mutatation-test.sh command: cosmic-ray worker jmespath.lexer mutate_binary_operator 51 nose -- tests/ --- mutation diff --- --- a/home/ec2-user/jmespath.py/jmespath/lexer.py +++ b/home/ec2-user/jmespath.py/jmespath/lexer.py @@ -30,7 +30,7 @@ next_char = self._next() if (next_char == ']'): self._next() - (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start + 2)}) + (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start // 2)}) elif (next_char == '?'): self._next() (yield {'type': 'filter', 'value': '[?', 'start': start, 'end': (start + 2)}) Using Mutation Testing ˘ Still passing… 

mutatation-test.sh command: cosmic-ray worker jmespath.lexer mutate_binary_operator 53 nose -- tests/ --- mutation diff --- --- a/home/ec2-user/jmespath.py/jmespath/lexer.py +++ b/home/ec2-user/jmespath.py/jmespath/lexer.py @@ -30,7 +30,7 @@ next_char = self._next() if (next_char == ']'): self._next() - (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start + 2)}) + (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start ** 2)}) elif (next_char == '?'): self._next() (yield {'type': 'filter', 'value': '[?', 'start': start, 'end': (start + 2)}) Using Mutation Testing ˘ 

mutatation-test.sh command: cosmic-ray worker jmespath.lexer mutate_binary_operator 53 nose -- tests/ --- mutation diff --- --- a/home/ec2-user/jmespath.py/jmespath/lexer.py +++ b/home/ec2-user/jmespath.py/jmespath/lexer.py @@ -30,7 +30,7 @@ next_char = self._next() if (next_char == ']'): self._next() - (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start + 2)}) + (yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': (start ** 2)}) elif (next_char == '?'): self._next() (yield {'type': 'filter', 'value': '[?', 'start': start, 'end': (start + 2)}) Using Mutation Testing ˘ I think there’s a missing test… 

mutatation-test.sh - if ('.' in arg): + if ('.' != arg): - if (len(allowed_subtypes) == 1): + if (len(allowed_subtypes) <= 1): - if ((self._lookahead(0) == 'colon') or (self._lookahead(1) == 'colon')): + if ((self._lookahead(0) in 'colon') or (self._lookahead(1) == 'colon')): - prev_t = self._lookahead_token((-2)) + prev_t = self._lookahead_token(2) - self._index = 0 + self._index = 1 - for key in random.sample(self._CACHE.keys(), int((self._MAX_SIZE / 2))): + for key in random.sample(self._CACHE.keys(), int((self._MAX_SIZE / 3))): Using Mutation Testing 

Cosmic Ray Tips • Python 3 only • Long execution time, offers distributed test execution • Run periodically as test audit, results need manual review • Helps to have fast test suite 

Property based testing Fuzz testing Stress testing Mutation testing 

Thanks! • American Fuzzy Lop - http://lcamtuf.coredump.cx/afl/ • Hypothesis - http://hypothesis.works/ • Cosmic Ray - http://cosmic-ray.readthedocs.io/en/latest/ • JMESPath - http://jmespath.org/ • @jsaryer