Property-based testing is a mindset

Transcript

1. I S A M I N D S E T
   P R O P E R T Y - B A S E D T E S T I N G
   

   View Slide

2. @whatyouhide
   

   View Slide

3. View Slide

4. View Slide

5. https://weedmaps.com/careers
   

   View Slide

6. T E S T I N G
   

   View Slide

7. why do we test?
   no tests
   yes tests
   

   View Slide

8. unit tests
   

   View Slide

9. test "sorting" do
   assert sort([]) == []
   assert sort([1, 2, 3]) == [1, 2, 3]
   assert sort([2, 1, 3]) == [1, 2, 3]
   end
   example-based
   

   View Slide

10. table-based
    Input Output
    [] []
    [1, 2, 3] [1, 2, 3]
    [2, 1, 3] [1, 2, 3]
    

    View Slide

11. unit tests
    , but also
    
    

    View Slide

12. P R O P E R T I E S
    

    View Slide

13. hard to write , but...
    

    View Slide

14. valid inputs
    properties of output
    testing framework
    

    View Slide

15. github.com/whatyouhide/stream_data
    

    View Slide

16. example time:
    sorting lists
    

    View Slide

17. test "sorting" do
    assert sort([]) == []
    assert sort([1, 2, 3]) == [1, 2, 3]
    assert sort([2, 1, 3]) == [1, 2, 3]
    end
    

    View Slide

18. lists of integers
    

    View Slide

19. it's a list
    has the same elements
    it's ordered
    

    View Slide

20. check all list sorted = sort(list)
    assert is_list(sorted)
    assert same_elements?(list, sorted)
    assert ordered?(sorted)
    end
    

    View Slide

21. check all list sorted = sort(list)
    assert is_list(sorted)
    assert same_elements?(list, sorted)
    assert ordered?(sorted)
    end
    

    View Slide

22. check all list sorted = sort(list)
    assert is_list(sorted)
    assert same_elements?(list, sorted)
    assert ordered?(sorted)
    end
    

    View Slide

23. check all list sorted = sort(list)
    assert is_list(sorted)
    assert same_elements?(list, sorted)
    assert ordered?(sorted)
    end
    

    View Slide

24. check all list sorted = sort(list)
    assert is_list(sorted)
    assert same_elements?(list, sorted)
    assert ordered?(sorted)
    end
    

    View Slide

25. def sort(list), do: list
    

    View Slide

26. [32, 2, 44, -12]
    [1, 0]
    
    

    View Slide

27. G E N E R A T O R S
    

    View Slide

28. iex> Enum.take(integer(), 4)
    [1, 0, -3, 1]
    

    View Slide

29. Composability
    

    View Slide

30. number = one_of([integer(), float()])
    

    View Slide

31. StreamData.map(integer(), &abs/1)
    

    View Slide

32. def string(:ascii) do
    integer(?\s..?~)
    |> list_of()
    |> map(&List.to_string/1)
    end
    Example
    

    View Slide

33. ⚠
    Keep shrinkability
    

    View Slide

34. constant(term)
    bind_filter(gen, fun)
    +
    

    View Slide

35. iex> Enum.take(constant(:foo), 4)
    [:foo, :foo, :foo, :foo]
    constant always generates a term
    

    View Slide

36. bind_filter(integer(), fn i ->
    if i < 0 do
    :skip
    else
    gen = map(list_of(integer()), &(&1 + i))
    {:cont, gen}
    end
    end)
    bind_filter (possibly) creates
    generators from generated terms
    

    View Slide

37. P A T T E R N S
    

    View Slide

38. circular code
    

    View Slide

39. decode(encode(term)) == term
    

    View Slide

40. property "unicode escaping" do
    check all string encoded = encode(string, escape: :unicode)
    assert decode(encoded) == string
    end
    end
    JSON encoding
    

    View Slide

41. oracle model
    

    View Slide

42. my_code() == oracle_code()
    

    View Slide

43. older system
    less performant implementation
    

    View Slide

44. property "gives same results as Erlang impl" do
    check all bin assert Huffman.encode(bin) ==
    :huffman.encode(bin)
    end
    end
    

    View Slide

45. smoke tests
    https://www.youtube.com/watch?v=jvwfDdgg93E
    

    View Slide

46. API: 200, 201, 400, 404
    https://www.youtube.com/watch?v=jvwfDdgg93E
    

    View Slide

47. https://www.youtube.com/watch?v=jvwfDdgg93E
    property "only expected codes are returned" do
    check all request response = HTTP.perform(request)
    assert response.status in [200, 201, 400, 404]
    end
    end
    

    View Slide

48. locally , CI
    

    View Slide

49. if ci?() do
    config :stream_data, max_runs: 500
    else
    config :stream_data, max_runs: 25
    end
    

    View Slide

50. unit + properties
    

    View Slide

51. property "String.contains?/2" do
    check all left right assert String.contains?(left <> right, left)
    assert String.contains?(left <> right, right)
    end
    end
    test "String.contains?/2" do
    assert String.contains?("foobar", "foo")
    assert String.contains?("foobar", "bar")
    assert String.contains?("foobar", "ob")
    end
    +
    

    View Slide

52. S T A T E F U L
    T E S T I N G
    

    View Slide

53. model
    valid commands
    +
    

    View Slide

54. model: state + state transformations
    

    View Slide

55. commands: calls + preconditions
    

    View Slide

56. getting/setting keys
    in Redis
    

    View Slide

57. model
    system
    Redis %{}
    

    View Slide

58. •get(key)
    •set(key, value)
    commands
    

    View Slide

59. def get(model, key),
    do: Map.get(model, key)
    def set(model, key, value),
    do: Map.put(model, key, value)
    

    View Slide

60. keys = Map.keys(model)
    one_of([
    command(:get, [one_of(keys)]),
    command(:set, [binary(), binary()]),
    command(:set, [one_of(keys), binary()])
    ])
    

    View Slide

61. {:ok, result} = Redix.command!(conn, ["GET", key])
    assert Map.fetch(model, key) == {:ok, result}
    Redix.command!(conn, ["SET", key, value])
    Map.replace!(model, key, value)
    get
    set_existing
    

    View Slide

62. LevelDB
    

    View Slide

63. 17 (seventeen) calls
    33 (thirty three) calls
    

    View Slide

64. R E S E A R C H
    

    View Slide

65. trees are hard
    

    View Slide

66. defmodule Tree do
    def tree() do
    StreamData.tree(:leaf, fn leaf -> {leaf, leaf} end)
    end
    def size({l, r}), do: 1 + size(l) + size(r)
    def size(_leaf), do: 1
    def depth({l, r}), do: 1 + max(depth(l), depth(r))
    def depth(_leaf), do: 1
    end
    

    View Slide

67. Generation size: 10
    Avg size: 4.9
    Avg max depth: 2.473
    Generation size: 100
    Avg size: 10.892
    Avg max depth: 3.466
    Generation size: 1000
    Avg size: 22.732
    Avg max depth: 4.507
    

    View Slide

68. stream_data
    +
    dialyzer
    

    View Slide

69. Generators from type
    

    View Slide

70. @type timeout() :: :infinity | non_neg_integer()
    

    View Slide

71. from_type(timeout())
    one_of([:infinity, map(integer(), &abs/1)])
    

    View Slide

72. Automatic typespec property
    checking
    

    View Slide

73. @spec my_fun(timeout()) :: :ok | :error
    check all timeout assert my_fun(timeout) in [:ok, :error]
    end
    

    View Slide

74. C O N C L U S I O N
    

    View Slide

75. find obscure bugs
    reduce to minimal failing input
    find specification errors
    cover vast input space
    

    View Slide

76. v1.7
    

    View Slide

77. use stream_data
    

    View Slide

78. use property-based testing
    

    View Slide

79. @whatyouhide
    github.com/whatyouhide/stream_data
    

    View Slide