Property Based Testing

A short overview of property based testing using jsverify as the implementation.

Transcript

1. Property Based Testing Hardy Jones
2. What is the definition of a test?
3. What is the purpose of a test?
4. Functionality can be simple, or complex.
5. Functionality tests can be simple, or complex.
6. Functionality tests can be unit tests, or complex.
7. Functionality tests can be unit tests, or integration tests.
8. Unit Test
9. Unit Test Result of a function
10. Unit Test Result of a function Visible effects of a procedure
11. Unit Test Result of a function Visible effects of a procedure Message sent from an object
12. Integration Test
13. Integration Test Logging to a file
14. Integration Test Logging to a file Clicking a button and getting a POST request
15. Integration Test Logging to a file Clicking a button and getting a POST request Ensuring two objects communicate
16. What benefits do tests provide?
17. But, what are the downsides?
18. But, what are the downsides? “...program testing can be a very effective way to show the presence of bugs, but is hopelessly inadequate for showing their absence.” - E.W. Disjkstra
19. But, what are the downsides? “...program testing can be a very effective way to show the presence of bugs, but is hopelessly inadequate for showing their absence.” - E.W. Disjkstra Well, sort of...
20. Example Test Numbers
21. Example Test Numbers Addition
22. Example Test Numbers Addition Associative property
23. Example Test Numbers Addition Associative property Identity property
24. Associative Property 1 + (2 + 3) = (1 + 2) + 3 (1 + 2) + (3 + 4) = ((1 + 2) + 3) + 4
25. Identity Property 1 + 0 = 1 0 + 3 = 3 0 + 0 = 0
26. var assert = require('assert'); describe('Addition for numbers', function() { describe('should not care about the order', function() { it('1 + (2 + 3) = (1 + 2) + 3', function() { assert(1 + (2 + 3) == (1 + 2) + 3); }); it('(1 + 2) + (3 + 4) = ((1 + 2) + 3) + 4', function() { assert((1 + 2) + (3 + 4) == ((1 + 2) + 3) + 4); }); }); …
27. var assert = require('assert'); describe('Addition for numbers', function() { describe('should not care about the order', function() { it('1 + (2 + 3) = (1 + 2) + 3', function() { assert(1 + (2 + 3) == (1 + 2) + 3); }); it('(1 + 2) + (3 + 4) = ((1 + 2) + 3) + 4', function() { assert((1 + 2) + (3 + 4) == ((1 + 2) + 3) + 4); }); }); …
28. var assert = require('assert'); describe('Addition for numbers', function() { describe('should not care about the order', function() { it('1 + (2 + 3) = (1 + 2) + 3', function() { assert(1 + (2 + 3) == (1 + 2) + 3); }); it('(1 + 2) + (3 + 4) = ((1 + 2) + 3) + 4', function() { assert((1 + 2) + (3 + 4) == ((1 + 2) + 3) + 4); }); }); …
29. var assert = require('assert'); describe('Addition for numbers', function() { describe('should not care about the order', function() { it('1 + (2 + 3) = (1 + 2) + 3', function() { assert(1 + (2 + 3) == (1 + 2) + 3); }); it('(1 + 2) + (3 + 4) = ((1 + 2) + 3) + 4', function() { assert((1 + 2) + (3 + 4) == ((1 + 2) + 3) + 4); }); }); …
30. var assert = require('assert'); describe('Addition for numbers', function() { describe('should not care about the order', function() { it('1 + (2 + 3) = (1 + 2) + 3', function() { assert(1 + (2 + 3) == (1 + 2) + 3); }); it('(1 + 2) + (3 + 4) = ((1 + 2) + 3) + 4', function() { assert((1 + 2) + (3 + 4) == ((1 + 2) + 3) + 4); }); }); …
31. var assert = require('assert'); describe('Addition for numbers', function() { describe('should not care about the order', function() { it('1 + (2 + 3) = (1 + 2) + 3', function() { assert(1 + (2 + 3) == (1 + 2) + 3); }); it('(1 + 2) + (3 + 4) = ((1 + 2) + 3) + 4', function() { assert((1 + 2) + (3 + 4) == ((1 + 2) + 3) + 4); }); }); …
32. var assert = require('assert'); describe('Addition for numbers', function() { describe('should not care about the order', function() { it('1 + (2 + 3) = (1 + 2) + 3', function() { assert(1 + (2 + 3) == (1 + 2) + 3); }); it('(1 + 2) + (3 + 4) = ((1 + 2) + 3) + 4', function() { assert((1 + 2) + (3 + 4) == ((1 + 2) + 3) + 4); }); }); …
33. … describe('should have an identity', function() { it('1 + 0 = 1', function() {assert(1 + 0 == 1);}); it('0 + 3 = 3', function() {assert(0 + 3 == 3);}); it('0 + 0 = 0', function() {assert(0 + 0 == 0);}); }); });
34. … describe('should have an identity', function() { it('1 + 0 = 1', function() {assert(1 + 0 == 1);}); it('0 + 3 = 3', function() {assert(0 + 3 == 3);}); it('0 + 0 = 0', function() {assert(0 + 0 == 0);}); }); });
35. … describe('should have an identity', function() { it('1 + 0 = 1', function() {assert(1 + 0 == 1);}); it('0 + 3 = 3', function() {assert(0 + 3 == 3);}); it('0 + 0 = 0', function() {assert(0 + 0 == 0);}); }); });
36. … describe('should have an identity', function() { it('1 + 0 = 1', function() {assert(1 + 0 == 1);}); it('0 + 3 = 3', function() {assert(0 + 3 == 3);}); it('0 + 0 = 0', function() {assert(0 + 0 == 0);}); }); });
37. … describe('should have an identity', function() { it('1 + 0 = 1', function() {assert(1 + 0 == 1);}); it('0 + 3 = 3', function() {assert(0 + 3 == 3);}); it('0 + 0 = 0', function() {assert(0 + 0 == 0);}); }); });
38. … describe('should have an identity', function() { it('1 + 0 = 1', function() {assert(1 + 0 == 1);}); it('0 + 3 = 3', function() {assert(0 + 3 == 3);}); it('0 + 0 = 0', function() {assert(0 + 0 == 0);}); }); });
39. Does it work for all valid inputs?
40. Does it work for all valid inputs? How can we be sure?
41. “...program testing can be a very effective way to show the presence of bugs, but is hopelessly inadequate for showing their absence.” - E.W. Disjkstra
42. “...program testing can be a very effective way to show the presence of bugs, but is hopelessly inadequate for showing their absence.” - E.W. Disjkstra Proofs!
43. Proofs
44. Proofs Direct proof
45. Proofs Direct proof Proof by Contradiction
46. Proofs Direct proof Proof by Contradiction Proof by Induction
47. Proofs Direct proof Proof by Contradiction Proof by Induction Proof by Enumeration
48. QuickCheck
49. QuickCheck Paper by Koen Claessen & John Hughes (2000)
50. QuickCheck Paper by Koen Claessen & John Hughes (2000) Haskell library circa 2006
51. QuickCheck Paper by Koen Claessen & John Hughes (2000) Haskell library circa 2006 node-quickcheck
52. QuickCheck Paper by Koen Claessen & John Hughes (2000) Haskell library circa 2006 node-quickcheck JSCheck
53. QuickCheck Paper by Koen Claessen & John Hughes (2000) Haskell library circa 2006 node-quickcheck JSCheck Claire
54. QuickCheck Paper by Koen Claessen & John Hughes (2000) Haskell library circa 2006 node-quickcheck JSCheck Claire JSVerify
55. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
56. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
57. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
58. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
59. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
60. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
61. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
62. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
63. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
64. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
65. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
66. var jsverify = require('jsverify'); var assert = jsverify.assert; var forall = jsverify.forall; var nat = jsverify.nat; describe('Addition for numbers', function() { describe('should not care about the order', function() { var assoc = forall(nat, nat, nat, function(a, b, c) { return a + (b + c) == (a + b) + c; }); it('a + (b + c) = (a + b) + c', function() { assert(assoc); }); …
67. … var identity = forall(nat, function(a) { return a + 0 == a && 0 + a == a; }); it('a + 0 = a && 0 + a = a', function() { assert(identity); }); }); });
68. … var identity = forall(nat, function(a) { return a + 0 == a && 0 + a == a; }); it('a + 0 = a && 0 + a = a', function() { assert(identity); }); }); });
69. … var identity = forall(nat, function(a) { return a + 0 == a && 0 + a == a; }); it('a + 0 = a && 0 + a = a', function() { assert(identity); }); }); });
70. … var identity = forall(nat, function(a) { return a + 0 == a && 0 + a == a; }); it('a + 0 = a && 0 + a = a', function() { assert(identity); }); }); });
71. … var identity = forall(nat, function(a) { return a + 0 == a && 0 + a == a; }); it('a + 0 = a && 0 + a = a', function() { assert(identity); }); }); });
72. Real World Example
73. Username Validation
74. Username Validation At least 8 characters
75. Username Validation At least 8 characters One lower case
76. Username Validation At least 8 characters One lower case One upper case
77. Username Validation At least 8 characters One lower case One upper case One number
78. Two cases
79. Two cases Valid input (SallyMae12, th3Us3rNam3)
80. Two cases Valid input (SallyMae12, th3Us3rNam3) Invalid input (user1, myname, US3RNAM3)
81. Make our own arbitraries
82. Make our own arbitraries var elements = jsverify.elements; var lower = elements('abcdefghijklmnopqrstuvwxyz'.split('')); var upper = elements('ABCDEFGHIJKLMNOPQRSTUVWXYZ'.split('')); var digits = elements('0123456789'.split(''));
83. Make our own arbitraries var elements = jsverify.elements; var lower = elements('abcdefghijklmnopqrstuvwxyz'.split('')); var upper = elements('ABCDEFGHIJKLMNOPQRSTUVWXYZ'.split('')); var digits = elements('0123456789'.split(''));
84. Make our own arbitraries var elements = jsverify.elements; var lower = elements('abcdefghijklmnopqrstuvwxyz'.split('')); var upper = elements('ABCDEFGHIJKLMNOPQRSTUVWXYZ'.split('')); var digits = elements('0123456789'.split(''));
85. Make our own arbitraries var elements = jsverify.elements; var lower = elements('abcdefghijklmnopqrstuvwxyz'.split('')); var upper = elements('ABCDEFGHIJKLMNOPQRSTUVWXYZ'.split('')); var digits = elements('0123456789'.split(''));
86. Make our own arbitraries var tuple = jsverify.tuple; var proper = tuple([lower, upper, digits, lower, upper, digits, lower, upper ]).smap(function(chars) { return chars.join(''); }, function(str) { return str.split(''); });
87. Make our own arbitraries var tuple = jsverify.tuple; var proper = tuple([lower, upper, digits, lower, upper, digits, lower, upper ]).smap(function(chars) { return chars.join(''); }, function(str) { return str.split(''); });
88. Make our own arbitraries var tuple = jsverify.tuple; var proper = tuple([lower, upper, digits, lower, upper, digits, lower, upper ]).smap(function(chars) { return chars.join(''); }, function(str) { return str.split(''); });
89. Make our own arbitraries var tuple = jsverify.tuple; var proper = tuple([lower, upper, digits, lower, upper, digits, lower, upper ]).smap(function(chars) { return chars.join(''); }, function(str) { return str.split(''); });
90. Make our own arbitraries var tuple = jsverify.tuple; var proper = tuple([lower, upper, digits, lower, upper, digits, lower, upper ]).smap(function(chars) { return chars.join(''); }, function(str) { return str.split(''); });
91. Make our own arbitraries var tuple = jsverify.tuple; var proper = tuple([lower, upper, digits, lower, upper, digits, lower, upper ]).smap(function(chars) { return chars.join(''); }, function(str) { return str.split(''); });
92. Make our own arbitraries var tuple = jsverify.tuple; var proper = tuple([lower, upper, digits, lower, upper, digits, lower, upper ]).smap(function(chars) { return chars.join(''); }, function(str) { return str.split(''); });
93. Make our own arbitraries var short = proper.smap(function(name) { return name.slice(-5); }, function(name) { return name; });
94. Make our own arbitraries var short = proper.smap(function(name) { return name.slice(-5); }, function(name) { return name; });
95. Make our own arbitraries var short = proper.smap(function(name) { return name.slice(-5); }, function(name) { return name; });
96. Make our own arbitraries var short = proper.smap(function(name) { return name.slice(-5); }, function(name) { return name; });
97. Make our own arbitraries var short = proper.smap(function(name) { return name.slice(-5); }, function(name) { return name; });
98. describe('Validator', function() { describe('#username', function() { var properName = forall(proper, function(name) { return Validator.username(name); }); it('should validate a proper username', function() { assert(properName); }); …
99. describe('Validator', function() { describe('#username', function() { var properName = forall(proper, function(name) { return Validator.username(name); }); it('should validate a proper username', function() { assert(properName); }); …
100. describe('Validator', function() { describe('#username', function() { var properName = forall(proper, function(name) { return Validator.username(name); }); it('should validate a proper username', function() { assert(properName); }); …
101. describe('Validator', function() { describe('#username', function() { var properName = forall(proper, function(name) { return Validator.username(name); }); it('should validate a proper username', function() { assert(properName); }); …
102. … var badName = forall(short, function(name) { return !Validator.username(name); }); it('should not validate a short username', function() { assert(badName); }); }); });
103. … var badName = forall(short, function(name) { return !Validator.username(name); }); it('should not validate a short username', function() { assert(badName); }); }); });
104. … var badName = forall(short, function(name) { return !Validator.username(name); }); it('should not validate a short username', function() { assert(badName); }); }); });
105. … var badName = forall(short, function(name) { return !Validator.username(name); }); it('should not validate a short username', function() { assert(badName); }); }); });
106. Where do we go from here?
107. Where do we go from here? Model with Property-based testing
108. Where do we go from here? Model with Property-based testing Strong static types (GHCjs, PureScript)
109. Where do we go from here? Model with Property-based testing Strong static types (GHCjs, PureScript) Computer verified proofs (Idris)
110. Thanks