Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Let the computer write the tests

dcrosta
November 04, 2018

Let the computer write the tests

The Hypothesis library by David McIver is a property testing library for Python. Property testing is related to fuzz testing, a technique commonly applied in fields where correctness is paramount. With a little guidance from you, the programmer, Hypothesis can generate a wide variety of valid -- and invalid -- inputs to your functions, and test scenarios you might never have considered. It can generate instances of your custom objects, and isn't limited just to simple functions. On top of all of this, when Hypothesis finds a failure, it will simplify the failing case, which aids tremendously in diagnosing what is actually wrong in your code.

dcrosta

November 04, 2018
Tweet

More Decks by dcrosta

Other Decks in Programming

Transcript

  1. def test_list_mean(): lst = list(range(10)) mean = list_mean(lst) # Even

    though we could predict the exact # mean for this simple case, when testing # with Hypothesis, we relax the assertion # so that it works for many values assert min(lst) <= mean <= max(lst) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  2. def utf8_encode(text): return text.encode("utf8") def utf8_decode(bytes): return bytes.decode("utf8") def test_decode_encode():

    # Definitely probably UTF-8 value = b"f\xc3\xbc\xc3\xb1ky d\xc3\xbc\xc3\xa7k" roundtripped = utf8_encode(utf8_decode(value)) assert roundtripped == value Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  3. @contextmanager def does_not_raise(): try: yield except Exception as e: pytest.fail("Unexpected

    exception") def test_it_doesnt_fail(): with does_not_raise(): my_function(123) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  4. class MediaMetadata: def __init__(self, medium, rotation, width, height): self.medium =

    medium self.rotation = rotation self.width = width self.height = height def get_display_size(metadata): """Get the display width and height of a video, accounting for rotation. A video with rotation of either 90 or 270 degress returns (height, width) while a rotation of either 0 or 180 returns (width, height).""" Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  5. def get_display_size(metadata): """...""" assert metadata.medium in ("audio", "video") if metadata.medium

    == "video": assert metadata.width and metadata.height assert metadata.rotation % 90 == 0 if metadata.rotation % 360 in (90, 270): return (metadata.height, metadata.width) else: return (metadata.width, metadata.height) else: return None Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  6. def test_get_display_size(): # make sure rotation is handled properly metadata

    = MediaMetadata( medium="video", rotation=90, width=1920, height=1080, ) assert (1080, 1920) == get_display_size(metadata) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  7. from hypothesis import given, strategies as st @given( medium=st.just("video"), rotation=st.just(90),

    width=st.just(1920), height=st.just(1080), ) def test_get_display_size(medium, rotation, width, height): # make sure rotation is handled properly metadata = MediaMetadata( medium, rotation, width, height, ) assert (height, width) == get_display_size(metadata) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  8. Strategies • just() / one_of() / sampled_from() • characters() /

    text() / binary() / from_regex() • booleans() / integers() / floats() • lists(...) / dictionaries(...) / frozensets(...) • dates() / times() / datetimes() Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  9. from hypothesis import given, strategies as st @given( medium=st.just("video"), rotation=st.just(90),

    width=st.just(1920), height=st.just(1080), ) def test_get_display_size(medium, rotation, width, height): # make sure rotation is handled properly metadata = MediaMetadata( medium, rotation, width, height, ) assert (height, width) == get_display_size(metadata) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  10. @given( medium=st.text(), rotation=st.just(90), width=st.just(1920), height=st.just(1080), ) def test_get_display_size(medium, rotation, width,

    height): # make sure rotation is handled properly metadata = MediaMetadata( medium, rotation, width, height, ) assert (height, width) == get_display_size(metadata) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  11. Traceback (most recent call last): File ".../test_media.py", line 30, in

    test_get_display_size medium=st.text(), File ".../site-packages/hypothesis/core.py", line 960, in wrapped_test raise the_error_hypothesis_found File ".../test_media.py", line 41, in test_get_display_size assert (height, width) == get_display_size(metadata) File ".../test_media.py", line 15, in get_display_size assert metadata.medium in ("audio", "video") AssertionError: assert '' in ('audio', 'video') + where '' = <MediaMetadata object at 0x1057c3080>.medium Falsifying example: test_get_display_size(medium='', rotation=90, width=1920, height=1080) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  12. @given( medium=st.sampled_from(["audio", "video"]), rotation=st.just(90), width=st.just(1920), height=st.just(1080), ) def test_get_display_size(medium, rotation,

    width, height): # make sure rotation is handled properly metadata = MediaMetadata( medium, rotation, width, height, ) assert (height, width) == get_display_size(metadata) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  13. Traceback (most recent call last): File ".../test_media.py", line 30, in

    test_get_display_size medium=st.sampled_from(["audio", "video"]), File ".../site-packages/hypothesis/core.py", line 960, in wrapped_test raise the_error_hypothesis_found File ".../test_media.py", line 41, in test_get_display_size assert (height, width) == get_display_size(metadata) AssertionError: assert (1080, 1920) == None + where None = get_display_size(<MediaMetadata object at 0x10ef47048>) Falsifying example: test_get_display_size(medium='audio', rotation=90, width=1920, height=1080) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  14. @given( medium=st.sampled_from(["audio", "video"]), rotation=st.just(90), width=st.just(1920), height=st.just(1080), ) def test_get_display_size(medium, rotation,

    width, height): # make sure rotation is handled properly metadata = MediaMetadata(medium, rotation, width, height) if medium == "audio": assert get_display_size(metadata) is None else: assert get_display_size(metadata) == (height, width) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  15. @given( medium=st.sampled_from(["audio", "video"]), rotation=st.just(90), width=st.integers(min_value=1), height=st.integers(min_value=1), ) def test_get_display_size(medium, rotation,

    width, height): # make sure rotation is handled properly metadata = MediaMetadata(medium, rotation, width, height) if medium == "audio": assert get_display_size(metadata) is None else: assert get_display_size(metadata) == (height, width) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  16. @given( medium=st.sampled_from(["audio", "video"]), rotation=st.integers().map(lambda i: i * 90), width=st.integers(min_value=1), height=st.integers(min_value=1),

    ) def test_get_display_size(medium, rotation, width, height): # make sure rotation is handled properly metadata = MediaMetadata(medium, rotation, width, height) if medium == "audio": assert get_display_size(metadata) is None else: w, h = get_display_size(metadata) assert w, h in [(width, height), (height, width)] Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  17. def mediums(): """ Generate valid mediums for MediaMetadata objects. """

    return st.sampled_from(["audio", "video"]) @given( medium=mediums(), rotation=st.integers().map(lambda i: i * 90), width=st.integers(min_value=1), height=st.integers(min_value=1), ) def test_get_display_size(medium, rotation, width, height): # ... Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  18. def video_sizes(): return st.integers(min_value=1) def video_rotations(): return st.integers().map(lambda i: i

    * 90) @given( medium=mediums(), rotation=video_rotations(), width=video_sizes(), height=video_sizes(), ) def test_get_display_size(medium, rotation, width, height): # ... Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  19. def media_metadatas(): """ Generate valid, plausible MediaMetadata objects. """ return

    st.builds( MediaMetadata, medium=mediums(), rotation=video_rotations(), width=video_sizes(), height=video_sizes(), ) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  20. @given(media_metadatas()) def test_get_display_size(metadata): # make sure rotation is handled properly

    if metadata.medium == "audio": assert get_display_size(metadata) is None else: w, h = get_display_size(metadata) assert w, h in [ (metadata.width, metadata.height), (metadata.height, metadata.width), ] Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  21. def media_metadatas(): """ Generate valid, plausible MediaMetadata objects. """ return

    st.builds( MediaMetadata, medium=mediums(), rotation=video_rotations(), width=video_sizes(), height=video_sizes(), ) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  22. @st.composite def media_metadatas(draw): """ Generate valid, plausible MediaMetadata objects. """

    medium = draw(mediums()) if medium == "video": rotation = draw(video_rotations()) width = draw(video_sizes()) height = draw(video_sizes()) else: rotation = width = height = 0 return MediaMetadata(medium, rotation, width, height) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  23. def audio_metadatas(): return st.just(MediaMetadata("audio", 0, 0, 0)) def video_metadatas(): return

    st.builds( MediaMetadata, medium=st.just("video"), rotation=video_rotations(), width=video_sizes(), height=video_sizes(), ) def media_metadatas(): return st.one_of([audio_metadatas(), video_metadatas()]) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  24. @given( medium=st.text().filter(lambda t: t not in ("audio", "video")), rotation=st.integers().filter(lambda i:

    i % 90 != 0), width=st.just(0), height=st.just(0), ) def test_get_display_size_failures(medium, rotation, width, height): metadata = MediaMetadata(medium, rotation, width, height) with pytest.raises(AssertionError): get_display_size(metadata) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  25. @given(...) @example(medium="video", rotation=90, width=0, height=1) def test_get_display_size_failures(medium, rotation, width, height):

    metadata = MediaMetadata(medium, rotation, width, height) with pytest.raises(AssertionError): get_display_size(metadata) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  26. def am_i_unlucky(num): if num == 1234: raise Exception("you are very

    unlucky!") @given(st.integers()) @example(1234) def test_always_fails(num): am_i_unlucky(num) @given(st.integers()) def test_never_fails(num): assume(num != 1234) am_i_unlucky(num) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  27. def test_list_mean(): lst = list(range(10)) assert list_mean(lst) == 4.5 Concepts

    → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  28. >>> data = os.urandom(8) >>> data b'zP\xb4\xca\r\xff{\x96' >>> struct.unpack(">Q", data)

    (8813743250675301270,) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  29. >>> data = os.urandom(8) >>> data b'zP\xb4\xca\r\xff{\x96'L >>> struct.unpack(">Q", data)

    (8813743250675301270,) >>> struct.unpack(">Q", b'\0' + data[1:]) (22716778048093078,) >>> struct.unpack(">Q", b'\0\0' + data[2:]) (198779911240598,) # ... >>> struct.unpack(">Q", b'\0\0\0\0\0\0\0' + data[7:]) (150,) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  30. def audio_metadatas(): return st.just(MediaMetadata("audio", 0, 0, 0)) def video_metadatas(): return

    st.builds( MediaMetadata, medium=st.just("video"), rotation=video_rotations(), width=video_sizes(), height=video_sizes(), ) def media_metadatas(): return st.one_of([audio_metadatas(), video_metadatas()]) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  31. Trying example: ...(medium='\x17/ \x0c', rotation=-1941619424369918455, Trying example: ...(medium='\x17/ \x0c', rotation=-1941619424369918455,

    Trying example: ...(medium='///ı', rotation=-128, width=0, height=0) Trying example: ...(medium='', rotation=64, width=0, height=0) Trying example: ...(medium='', rotation=32, width=0, height=0) Trying example: ...(medium='', rotation=16, width=0, height=0) Trying example: ...(medium='', rotation=8, width=0, height=0) Trying example: ...(medium='', rotation=4, width=0, height=0) Trying example: ...(medium='', rotation=3, width=0, height=0) Trying example: ...(medium='', rotation=2, width=0, height=0) Trying example: ...(medium='', rotation=1, width=0, height=0) Falsifying example: test_get_display_size_failures(medium='', rotation=1, width=0, height=0) Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  32. from hypothesis import given, settings, Verbosity from hypothesis.strategies import integers

    @settings(verbosity=Verbosity.verbose) @given(integers()) def test_to_demonstrate_verbosity(an_int): # ... Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits
  33. # conftest.py from hypothesis import settings, Verbosity settings.register_profile( "local", verbosity=Verbosity.verbose,

    ) settings.register_profile( "ci", max_examples=50, perform_health_check=False, ) # pip install hypothesis-pytest # pytest --hypothesis-profile=ci ... Concepts → Using Hypothesis → Custom Strategies → Other Tests You Need → Shrinking → Tidbits