Benchmarking Ruby

Testing is firmly ingrained in our culture, and many of us rely on a network of services to test, evaluate and profile our code. But what about benchmarking?

Learn tips and tricks about how to approach benchmarking and the variety of gems that are available. Learn about tools to help determine algorithmic complexity of your code, as well as how this information can help you make development choices. Learn how to properly set up your benchmarking experiments to ensure that the results you receive are accurate. More importantly, discover that benchmarking can be both fun and easy.

Transcript

1. #FODINBSLJOH
   3VCZ 3VCZ$POG
   
2. )J
3. !EBWZTUFWFOTPO
   PO
   5XJUUFS *`N
   %BWZ
   4UFWFOTPO
4. -FU`T
   UBML
   BCPVU
   #FODINBSLJOH
5. 3VCZ
   *NQMFNFOUBUJPOT
   PS
   'SBNFXPSLT
6. 3VCZ
   *NQMFNFOUBUJPOT
   PS
   'SBNFXPSLT
7. JTSVCZGBTUZFUDPN
   ! .BEF
   CZ
   .BSLFU
   
   
   
   
   
   
   3VCZ
   8FC
   #FODINBSL
   3FQPSU
8. )PX
9. )PX
   1JUGBMMT
10. )PX
    1JUGBMMT
    'VO
11. 8IZ
    TIPVME
    :06
    CFODINBSL
12. 8IZ
    EP
    ZPV
    XSJUF
    UFTUT
13. $FSUBJOUZ
    JO
    GVODUJPOBMJUZ
14. 8IZ
    TIPVME
    ZPV
    CFODINBSL
15. $FSUBJOUZ
    JO
    QFSGPSNBODF
16. :PVS
    DPEF
17. :PVS
    DPEF
    
    HFN
    DPEF
18. :PVS
    DPEF
    
    HFN
    DPEF
    
    3VCZ
    DPEF
19. )PX
    UP
    #FODINBSL
    3VCZ
    $PEF
20. require 'benchmark' n = 100_000 size = 10_000 array = (0...size).to_a.shuffle ! Benchmark.bm do |x| x.report("#at") { n.times { array.at rand(size) } } ! x.report("#index") { n.times { array.index rand(size) } } end
21. require 'benchmark' n = 100_000 size = 10_000 array = (0...size).to_a.shuffle ! Benchmark.bm do |x| x.report("#at") { n.times { array.at rand(size) } } ! x.report("#index") { n.times { array.index rand(size) } } end
22. require 'benchmark' n = 100_000 size = 10_000 array = (0...size).to_a.shuffle ! Benchmark.bm do |x| x.report("#at") { n.times { array.at rand(size) } } ! x.report("#index") { n.times { array.index rand(size) } } end
23. require 'benchmark' n = 100_000 size = 10_000 array = (0...size).to_a.shuffle ! Benchmark.bm do |x| x.report("#at") { n.times { array.at rand(size) } } ! x.report("#index") { n.times { array.index rand(size) } } end
24. $ ruby examples/bm.rb user system total real #at 0.020000 0.000000 0.020000 ( 0.023444) #index 2.380000 0.000000 2.380000 ( 2.391208)
25. $ ruby examples/bm.rb user system total real #at 0.020000 0.000000 0.020000 ( 0.023444) #index 2.380000 0.000000 2.380000 ( 2.391208)
26. 1SPT
    
27. 1SPT
    3FBMMZ
    &BTZ
28. 1SPT
    *O
    4UE
    -JC
29. $POT
    
30. $POT
    7BSJBCMF
    'JEEMJOH
31. $POT
    %JGpDVMU
    0VUQVU
32. $POT
    #PJMFSQMBUF
    3FRVJSFE
33. require 'benchmark' n = 100_000 size = 10_000 array = (0...size).to_a.shuffle ! Benchmark.bm do |x| x.report("#at") { n.times { array.at rand(size) } } ! x.report("#index") { n.times { array.index rand(size) } } ! end
34. require 'benchmark/ips' ! size = 10_000 array = (0...size).to_a.shuffle ! Benchmark.ips do |x| x.report("#at") { array.at rand(size) } ! x.report("#index") { array.index rand(size) } x.compare! end
35. $ ruby examples/ips.rb ! Calculating ------------------------------------- #at 77.544k i/100ms #index 4.036k i/100ms ------------------------------------------------- #at 3.205M (± 7.9%) i/s - 15.897M #index 41.660k (± 8.7%) i/s - 205.836k ! Comparison: #at: 3204843.0 i/s #index: 41659.6 i/s - 76.93x slower
36. $ ruby examples/ips.rb ! Calculating ------------------------------------- #at 77.544k i/100ms #index 4.036k i/100ms ------------------------------------------------- #at 3.205M (± 7.9%) i/s - 15.897M #index 41.660k (± 8.7%) i/s - 205.836k ! Comparison: #at: 3204843.0 i/s #index: 41659.6 i/s - 76.93x slower
37. $ ruby examples/ips.rb ! Calculating ------------------------------------- #at 77.544k i/100ms #index 4.036k i/100ms ------------------------------------------------- #at 3.205M (± 7.9%) i/s - 15.897M #index 41.660k (± 8.7%) i/s - 205.836k ! Comparison: #at: 3204843.0 i/s #index: 41659.6 i/s - 76.93x slower
38. $ ruby examples/ips.rb ! Calculating ------------------------------------- #at 77.544k i/100ms #index 4.036k i/100ms ------------------------------------------------- #at 3.205M (± 7.9%) i/s - 15.897M #index 41.660k (± 8.7%) i/s - 205.836k ! Comparison: #at: 3204843.0 i/s #index: 41659.6 i/s - 76.93x slower
39. $ ruby examples/ips.rb ! Calculating ------------------------------------- #at 77.544k i/100ms #index 4.036k i/100ms ------------------------------------------------- #at 3.205M (± 7.9%) i/s - 15.897M #index 41.660k (± 8.7%) i/s - 205.836k ! Comparison: #at: 3204843.0 i/s #index: 41659.6 i/s - 76.93x slower
40. $ ruby examples/ips.rb ! Calculating ------------------------------------- #at 77.544k i/100ms #index 4.036k i/100ms ------------------------------------------------- #at 3.205M (± 7.9%) i/s - 15.897M #index 41.660k (± 8.7%) i/s - 205.836k ! Comparison: #at: 3204843.0 i/s #index: 41659.6 i/s - 76.93x slower
41. 1SPT
    
42. 1SPT
    -FTT
    pEEMZ
43. 1SPT
    #JHHFS
    
    #FUUFS
44. 1SPT
    4BNF
    4ZOUBY
45. 1SPT
    $PNQBSF
    'FBUVSF
46. $POT
    
47. $POT
    4FQBSBUF
    (FN
48. $POT
    4OBQTIPU
    7JFX
49. size = 10_000 $ ruby examples/ips.rb ! Comparison: #at: 3204843.0 i/s #index: 41659.6 i/s - 76.93x slower ! ! ! ! ! ! ! ! !
50. size = 10_000 $ ruby examples/ips.rb ! Comparison: #at: 3204843.0 i/s #index: 41659.6 i/s - 76.93x slower ! size = 100_000 $ ruby examples/ips.rb ! Comparison: #at: 3280139.5 i/s #index: 4066.3 i/s - 806.67x slower !
51. require 'benchmark/ips' size = 10_000 array = (0...size).to_a.shuffle ! Benchmark.ips do |x| x.report("#at") { array.at rand(size) } ! x.report("#index") { array.index rand(size) } x.compare! end
52. require 'benchmark/bigo' ! Benchmark.bigo do |x| x.generate :array ! x.report("#at") {|array, size| array.at rand(size) } ! x.report("#index") {|array, size| array.index rand(size) } x.chart! 'chart.html' x.termplot! end
53. require 'benchmark/bigo' ! Benchmark.bigo do |x| x.generate :array ! x.report("#at") {|array, size| array.at rand(size) } ! x.report("#index") {|array, size| array.index rand(size) } x.chart! 'chart.html' x.termplot! end
54. require 'benchmark/bigo' ! Benchmark.bigo do |x| x.generator {|size| (0...size).to_a.shuffle } ! x.report("#at") {|array, size| array.at rand(size) } ! x.report("#index") {|array, size| array.index rand(size) } x.chart! 'chart.html' x.termplot! end
55. require 'benchmark/bigo' ! Benchmark.bigo do |x| x.generate :array ! x.report("#at") {|array, size| array.at rand(size) } ! x.report("#index") {|array, size| array.index rand(size) } x.chart! 'chart.html' x.termplot! end
56. require 'benchmark/bigo' ! Benchmark.bigo do |x| x.generate :array ! x.report("#at") {|array, size| array.at rand(size) } ! x.report("#index") {|array, size| array.index rand(size) } x.chart! 'chart.html' x.termplot! end
57. require 'benchmark/bigo' ! Benchmark.bigo do |x| x.generate :array ! x.report("#at") {|array, size| array.at rand(size) } ! x.report("#index") {|array, size| array.index rand(size) } x.chart! 'chart.html' x.termplot! end
58. Calculating ------------------------------------- #at 100 70.107k i/100ms #at 200 68.002k i/100ms #at 300 70.241k i/100ms #at 400 70.798k i/100ms #at 500 68.512k i/100ms ... ------------------------------------------------- #at 100 1.389M (±12.7%) i/s - 6.870M #at 200 1.403M (±13.5%) i/s - 6.936M #at 300 1.384M (±11.2%) i/s - 6.884M #at 400 1.908M (± 7.9%) i/s - 9.487M #at 500 1.902M (± 8.4%) i/s - 9.455M #at 600 1.872M (± 6.7%) i/s - 9.327M #at 700 1.901M (± 8.3%) i/s - 9.437M #at 800 1.923M (± 5.9%) i/s - 9.582M #at 900 1.887M (± 8.9%) i/s - 9.382M ... #index 600 284.390k (± 8.0%) i/s - 1.428M #index 700 245.863k (± 6.7%) i/s - 1.231M #index 800 225.728k (± 5.9%) i/s - 1.137M #index 900 206.441k (± 5.6%) i/s - 1.032M #index 1000 186.174k (± 8.2%) i/s - 928.076k
60. 3 ++------+------+-------+-------+------+-------+-------+------+------++ + #at ****** + + + + + + ### | #index ###### #### | 2.5 ++ ##### ++ | ###### | | #### | 2 ++ ###### ++ | ##### | | #### | 1.5 ++ ###### ++ | ######## | | ##### | | #### | 1 ++ ###### ++ #### | | | 0.5 ********************************************************************** | | + + + + + + + + + + 0 ++------+------+-------+-------+------+-------+-------+------+------++ 100 200 300 400 500 600 700 800 900 1000
61. 1SPT
    
62. 1SPT
    3BOHF
    PG
    *OQVUT
63. 1SPT
    $IBSUT
64. 1SPT
    "4$**
    $)"354
65. $POT
    
66. $POT
    4FQBSBUF
    (FN
67. $POT
    -POHFS
    3VO
    5JNF
68. $POT
    /PU
    "MXBZT
    "QQMJDBCMF
69. )PX
    UP
    #FODINBSL
    &GGFDUJWFMZ
70. "WPJE
    NBLJOH
    UIFTF
    DPNNPO
    NJTUBLFT
71. “Benchmarks are useless without reproduction and critique” ! - Jeremy Evans
72. HJUIVCDPNEBWZ CFODINBSLJOH@SVCZ
73. *T
    ZPVS
    FOWJSPONFOU
    DPOTJTUFOU
74. user_generator = Proc.new { |size| Array.new(size) do User.new(random_name) end } ! Benchmark.bigo do |x| x.generator &user_generator ! x.report('#sort') {|array, _| array.sort {|a,b| a.name <=> b.name } } x.report('#sort_by') {|array, _| array.sort_by &:name } ! x.chart! 'sort.html' x.compare! end
77. Benchmark.bigo do |x| x.generator &user_generator ! x.report('#sort') {|array, _| array.sort {|a,b| a.name <=> b.name } } x.report('#sort_by') {|array, _| array.sort_by &:name } ! x.chart! 'sort.html' x.compare! end
80. "SF
    ZPV
    WFSJGZJOH
    CFIBWJPS
81. 8SJUF
    5FTUT
82. def test_a 'a' end ! def test_b 'b' end ! Benchmark.ips do |x| x.report('test_a') { test_a } x.report('test_b') { test_b } end class BenchTest < MiniTest::Test ! def test_a_result assert_equal ‘a', test_a end ! def test_b_result assert_equal ‘b', test_b end ! def test_a_mutability assert_equal test_a, test_a end ! def test_b_mutability assert_equal test_b, test_b end end
83. “The Fibonacci Sequence is the perfect benchmark” ! - Tenderlove
84. class TestFib < MiniTest::Test def test_fib fibs = [0,1,1,2,3,5,8,13,21,34,55,89] ! fibs.each_with_index {|ans, i| assert_equal ans, fib(i) } end ! def test_fib_mutability assert_equal fib(5), fib(5) end end
85. def fib n return n if n < 2 n + fib(n-1) end ! Benchmark.bigo do |x| # 5..24 x.min_size = 5 x.step_size = 1 x.steps = 20 ! x.generate :size ! x.report("fib") {|size, _| fib(size) } ! x.chart! ‘fib.html' x.termplot! end 2 ++-----+------+------+------+------+-----+------+------+------+-----++ + fib+******+ + + + + + + + * 1.8 ++ *+ | *| | * | 1.6 ++ *++ | *** | 1.4 ++ ******* ++ | ******* | 1.2 ++ *** ++ | ******* | | *** | 1 ++ ******* ++ | ******* | 0.8 ++ **** ++ | ******* | | *** | 0.6 ++ ****** ++ + + + + + + + + + + + 0.4 ++-----+------+------+------+------+-----+------+------+------+-----++ 4 6 8 10 12 14 16 18 20 22 24 ! # Running: ! F. ! 1) Failure: TestFib#test_fib [fib/fib_wrong.rb:36]: Expected: 1 Actual: 3 ! 2 runs, 4 assertions, 1 failures
86. def fib n return n if n < 2 fib(n-1) + fib(n-2) end ! Benchmark.bigo do |x| # 5..24 x.min_size = 5 x.step_size = 1 x.steps = 20 ! x.generate :size ! x.report("fib") {|size, _| fib(size) } ! x.chart! ‘fib.html' x.termplot! end 9000 ++-----+------+-----+------+------+------+------+-----+------+-----++ + fib+****** + + + + + + + * 8000 ++ +* | *| 7000 ++ *+ | * | 6000 ++ *++ | * | 5000 ++ * ++ | * | | * | 4000 ++ * ++ | * | 3000 ++ * ++ | * | 2000 ++ *** ++ | *** | 1000 ++ *** ++ + + + + + + + ******* + + + 0 ++-****************************************-----+-----+------+-----++ 4 6 8 10 12 14 16 18 20 22 24 ! ! # Running: ! .. ! Finished in 0.001196s ! 2 runs, 13 assertions, 0 failures
87. def fib n return n if n < 2 fib(n-1) + fib(n-2) end ! Benchmark.bigo do |x| # 5..24 x.min_size = 5 x.step_size = 1 x.steps = 20 ! x.generate :size ! x.report("fib") {|size, _| fib(size) } ! x.chart! ‘fib.html' x.termplot! end 9000 ++-----+------+-----+------+------+------+------+-----+------+-----++ + fib+****** + + + + + + + * 8000 ++ +* | *| 7000 ++ *+ | * | 6000 ++ *++ | * | 5000 ++ * ++ | * | | * | 4000 ++ * ++ | * | 3000 ++ * ++ | * | 2000 ++ *** ++ | *** | 1000 ++ *** ++ + + + + + + + ******* + + + 0 ++-****************************************-----+-----+------+-----++ 4 6 8 10 12 14 16 18 20 22 24 ! ! # Running: ! .. ! Finished in 0.001196s ! 2 runs, 13 assertions, 0 failures
89. "SF
    ZPV
    NPEJGZJOH
    POMZ
    POF
    UIJOH
90. items = 5000.times.collect {|i| {:id => i, :score => SecureRandom.hex } } ! Benchmark.ips do |x| ! x.report("reduce") { items.reduce({}) { |accum, x| accum.merge(x[:id] => x[:score]) } } ! x.report("each with object") { items.each_with_object({}) { |x, accum| accum[x[:id]] = x[:score] } } x.compare! end
91. Calculating ------------------------------------- reduce 1.000 i/100ms each with object 33.000 i/100ms ------------------------------------------------- reduce 0.144 (± 0.0%) i/s - 1.000 in 6.96s each with object 397.831 (±30.2%) i/s - 1.716k ! Comparison: each with object: 397.8 i/s reduce: 0.1 i/s - 2769.63x slower
92. items = 5000.times.collect {|i| {:id => i, :score => SecureRandom.hex } } ! Benchmark.ips do |x| ! x.report("reduce") { items.reduce({}) { |accum, x| accum.merge(x[:id] => x[:score]) } } ! x.report("each with object") { items.each_with_object({}) { |x, accum| accum[x[:id]] = x[:score] } } x.compare! end
93. Benchmark.bigo do |x| ! x.generator {|size| # {0 => “5b2665861e...”, 1 => “e1d9cdef37...” } } ! x.report("merge") { |hash, size| hash.merge(rand(size) => SecureRandom.hex) } ! x.report("assign") { |hash, size| hash[rand(size)] = SecureRandom.hex } ! x.chart! 'merge_vs_assign.html' end
95. Benchmark.bigo do |x| x.generator {|size| size.times.collect {|i| {:id => i, :score => SecureRandom.hex} } } ! x.report("reduce") { |items, _| items.reduce({}) { |hash, x| hash[x[:id]] = x[:score] hash } } ! x.report("reduce-with-merge") { |items, _| items.reduce({}) { |hash, x| hash.merge(x[:id] => x[:score]) } } x.chart! 'reduce_vs_reduce_with_merge.html' end
97. "SF
    ZPV
    BDDJEFOUBMMZ
    NVUBUJOH
    PCKFDUT
98. Benchmark.bigo do |x| ! x.generate :array ! x.report('#delete') {|array, size| (0..(size/2)).each do |i| array.delete i end } ! x.report('#delete_if') {|array, size| array.delete_if {|a| a < size / 2 } } ! x.chart! 'mutating_delete.html' end
99. Benchmark.bigo do |x| ! x.generate :array ! x.report('#delete') {|array, size| (0..(size/2)).each do |i| array.delete i end } ! x.report('#delete_if') {|array, size| array.delete_if {|a| a < size / 2 } } ! x.chart! 'mutating_delete.html' end
100. Benchmark.bigo do |x| ! x.generate :array ! x.report('#delete') {|array, size| (0..(size/2)).each do |i| array.delete i end } ! x.report('#delete_if') {|array, size| array.delete_if {|a| a < size / 2 } } ! x.chart! 'mutating_delete.html' end
102. Benchmark.bigo do |x| ! x.generate :array ! x.report('#delete_if') {|array, size| array.delete_if {|a| a < size / 2 } } ! x.report('#dup+del_if') {|array, size| ary = array.dup ary.delete_if {|a| a < size / 2 } } ! x.chart! 'delete_if.html' end
103. 4FU
     $POUSPMT
104. Benchmark.bigo do |x| ! x.generate :array ! x.report('#dup') {|array, size| array.dup } ! x.report('#delete_if') {|array, size| array.delete_if {|a| a < size / 2 } } ! x.report('#dup+del_if') {|array, size| ary = array.dup ary.delete_if {|a| a < size / 2 } } ! x.chart! 'delete_if_with_control.html' end
107. "SF
     ZPV
     VTJOH
     SBOEPN
     FGGFDUJWFMZ
108. Benchmark.bigo do |x| ! x.generator do |size| (1..size).to_a.shuffle # shuffled end ! x.report("rand#include?(size/2)") do |a, size| a.include? (size / 2) # deterministic size end ! x.chart! 'shuffled_and_deterministic.html' end
110. Benchmark.bigo do |x| ! x.generator do |size| (1..size).to_a.shuffle # shuffled end ! x.report("rand#include?(size/2)") do |a, size| a.include? (size / 2) # deterministic size end ! x.chart! 'shuffled_and_deterministic.html' end
111. Benchmark.bigo do |x| ! x.generator do |size| (1..size).to_a.shuffle # shuffled end ! x.report("rand#include?(rand)") do |a, size| a.include? rand(size) # randomized end ! x.chart! 'shuffled_and_random.html' end
113. Benchmark.bigo do |x| x.generator do |size| (1..size).to_a # not shuffled end ! x.report("ordered#include?(1)") do |a, size| a.include? 1 # deterministic best case end ! x.report("ordered#include?(size)") do |a, size| a.include? size # deterministic worst case end ! x.report("ordered#include?(rand)") do |a, size| a.include? rand(size) # random average case end ! x.chart! 'not_shuffled.html' end
115. #FTU
     DBTF
     PS
     "WFSBHF
     DBTF
     PS
     8PSTU
     DBTF
116. 5FSSBGPSNFS A geometric toolkit for dealing with geometry, geography, formats
117. $POWFY
     )VMM
119. +BSWJT
     .BSDI
120. .POPUPOF
     $IBJO
121. LON = -122.6764, LAT = 45.5165 ! def random_walk size walk = [[LON, LAT]] # start in portland ! size.times do walk << walk.last.map {|i| i + rand(-100..100) / 10000.0 } end ! Terraformer::LineString.new(walk) end ! def circle size size = 3 if size < 3 diam = [100, size].max Terraformer::Circle.new([LON, LAT], diam, size).to_feature end
122. @generator_name = :random_walk ! # dynamically generates either # a random walk or a circle dynamic_generator = Proc.new {|size| self.send(@generator_name, size) } ! def convex_hull obj, impl Terraformer::ConvexHull.impl = impl obj.convex_hull end
123. Benchmark.bigo do |x| x.generator &dynamic_generator ! # sample each point for 20 seconds x.time = 20 ! x.min_size = 200 x.step_size = 200 x.steps = 5 ! ## reports on next slide ! x.chart! 'terraformer.html' end
124. @generator_name = :random_walk ! x.report("#rand-jarvis") {|obj, _| convex_hull obj, :jarvis_march } x.report("#rand-monotone") {|obj, _| convex_hull obj, :monotone } ! @generator_name = :circle ! x.report("#circ-jarvis") {|obj, _| convex_hull obj, :jarvis_march } x.report("#circ-monotone") {|obj, _| convex_hull obj, :monotone }
128. $PODMVTJPO
129. 7FSJGZ
     "TTVNQUJPOT
130. -FBSO
     BCPVU
     ZPVS
     DPEF
131. -FBSO
     BCPVU
     3VCZ
132. 8IFO
     UP
     VTF
     #FODINBSL
     HFN
133. +VTU
     VTF
     #FODINBSL
     *14
134. 8IFO
     UP
     VTF
     #FODINBSL
     *14
135. "MM
     UIF
     UJNF
136. (SFBU
     GPS
     RVJDL
     TQPU
     DIFDLT
137. &YQBOTJWF
     BOBMZTJT
     UPP
138. 8IFO
     UP
     VTF
     #FODINBSL
     #JH0
139. *OQVU
     IBT
     SBOHF
     PG
     TJ[FT
140. 3FTVMUT
     JO
     DIBSU
     GPSN
141. 0L
     JG
     JU
     UBLFT
     BXIJMF
142. xkcd #1445
143. $VSJPVT
     GPS
     NPSF
144. 'BTU
148. "UUSJCVUJPOT From The Noun Project: Island by Athena Manolopoulos Surfboard by Rachel Healey Surf Board v1 by Sean D’Auria Van by Okan Benn Diamond by Ryan Beck Beach Ball by Max Hancock Turtle by im icons Rabbit by Theresa Stoodley
149. 5IBOLT
     2VFTUJPOT !EBWZTUFWFOTPO