Don't Let Your Tests Flake Out

Don't Let Your
Tests Flake Out
Jason Clark
@jasonrclark
Tuesday, June 24, 14

View Slide

TROUBLES

View Slide

TROUBLES
TECHNIQUES

View Slide

TROUBLES
TECHNIQUES
TOOLS

View Slide

3
seattlerb/minitest
https://github.com/seattlerb/minitest

View Slide

TROUBLES
4

View Slide

State
5

View Slide

Globals
6
$evil = true

View Slide

7
include Singleton
Globals

View Slide

Globals
8
def test_isnt_evil
refute $evil
end

View Slide

Globals
9
def test_isnt_evil
refute $evil
end
def test_if_we_were_evil
$evil = true
assert $evil
end

View Slide

Globals
10
def test_isnt_evil
refute $evil
end
$evil = true
assert $evil
end
false

View Slide

Globals
11
def test_isnt_evil
refute $evil
end
$evil = true
assert $evil
end
true

View Slide

Globals
12
$evil = true
assert $evil
end
def test_isnt_evil
refute $evil
end
true

View Slide

Globals
13
$evil = true
assert $evil
end
def test_isnt_evil
refute $evil
end
true!

View Slide

14
https://ﬂic.kr/p/97nB5F

View Slide

15
SPECIAL = 1
"Constants"

View Slide

16
SPECIAL = [1,2,3]
"Constants"

View Slide

17
SPECIAL = [1,2,3]
SPECIAL << "Ha"
"Constants"

View Slide

18
https://ﬂic.kr/p/5vKLwQ

View Slide

19
class ClassyTest < Minitest::Test
@@classy = true
def test_classy
assert @@classy
end
end
Class Variables

View Slide

20
Class Variables
class ClassyTest < Minitest::Test
# ...
def test_not_classy
@@classy = false
refute @@classy
end
end

View Slide

21
https://ﬂic.kr/p/4EZada

View Slide

22
AS = ActiveSupport
AS::Notifications.subscribe("event") do
testing.is_awesome?
end
Events and Callbacks

View Slide

23
https://ﬂic.kr/p/4Mu5NX

View Slide

24
class Object
def boo
puts "Boo"
end
end
Metaprogramming

View Slide

24
class Object
def boo
puts "Boo"
end
end
class Object
remove_method(:boo)
end
Metaprogramming

View Slide

24
class Object
def boo
puts "Boo"
end
end
class Object
remove_method(:boo)
end
class Object
alias :hoo :boo
end
Metaprogramming

View Slide

25
require 'code'
require

View Slide

26
https://ﬂic.kr/p/4YwYM8

View Slide

27
Threading
class ThreadsAreCoolTest < Minitest::Test
def test_coolness
Thread.new do
loop { $cool = true }
end
assert $cool
end
end

View Slide

28
https://ﬂic.kr/p/B72L5

View Slide

29
Databases

View Slide

29
Queues
Databases

View Slide

29
Files
Queues
Databases

View Slide

29
Files
Queues
Databases
Services

View Slide

Comparison
30

View Slide

Floats
31
def test_28
assert_equal 28, (0.28 * 100)
end

View Slide

Floats
32
1) Failure:
test_28(Test_28) [/Users/jclark/test_28.rb]:
<28> expected but was
<28.000000000000004>.

View Slide

Floats
33
def test_28
assert_in_delta(28, 0.28 * 100, 0.00001)
end

View Slide

Time Roundtripping
34
def test_time
time = Time.now.to_f
round_trip = Time.at(time).to_f
assert_equal(time, round_trip)
end

View Slide

35
https://ﬂic.kr/p/62n3eY

View Slide

36
https://twitter.com/joshsusser/status/257725572275376128

View Slide

https://ﬂic.kr/p/bKnFx
https://ﬂic.kr/p/65uiJH

View Slide

38
https://ﬂic.kr/p/btvCeB

View Slide

TECHNIQUE
39

View Slide

Avoid
40

View Slide

Instances are Friends
41
def test_newly_created
subject = Testing.new
assert subject.fresh?
end

View Slide

Reset Consistently
42
def setup_and_teardown_agent(opts = {}, &block)
define_method(:setup) do
setup_agent(opts, &block)
end
define_method(:teardown) do
teardown_agent
end
end

View Slide

Reset Consistently
43
class AgentTest < Minitest::Test
setup_and_teardown_agent
# ...
end

View Slide

https://github.com/travisjeffery/timecop
travisjeffery/timecop
44
describe "some set of tests to mock" do
before do
Timecop.freeze(Time.local(1990))
end
after do
Timecop.return
end
it "should do blah blah blah" {}
end

View Slide

45
rails/rails
https://github.com/rails/rails
Hat tip to http://brandonhilkert.com/blog/rails-4-1-travel-to-test-helper/
travel_to Time.new(1999, 12, 31, 11, 59, 59) do
# Test like it's 1999!
end

View Slide

Time
46
def freeze_time(now=Time.now)
Time.stubs(:now).returns(now)
now
end
def advance_time(seconds)
freeze_time(Time.now + seconds)
end

View Slide

Metaprogramming
47
class TransmogifierTest < Minitest::Test
def test_modifies_class
clazz = Class.new
Transmogifier.go!(clazz)
assert clazz.new.tentacles?
end
end

View Slide

Metaprogramming
48
https://github.com/thoughtbot/appraisal

View Slide

Metaprogramming
48
https://github.com/sinisterchipmunk/testbeds

View Slide

Metaprogramming
48
https://github.com/newrelic/rpm/blob/master/test/multiverse/README.md
https://github.com/sinisterchipmunk/testbeds

View Slide

Locate
49

View Slide

50

View Slide

51
puts "JRC: GOT HERE"
Always Be Logging

View Slide

52
caller.join("\n\t")
Where Am I?

View Slide

53
TESTOPTS="--seed=33023" bundle exec rake
ruby test/awesome/code_test.rb --seed 33023
Seeds of !Change

View Slide

54

View Slide

55
class OrderedTest < Minitest::Test
i_suck_and_my_tests_are_order_dependent!
def test_feel_bad_but_carry_on
# ...
end
end
Method Order

View Slide

56
class SingleFailingTest < Minitest::Test
def test_i_sometimes_fail
flaky_call()
end
end
Loop

View Slide

57
class SingleFailingTest < Minitest::Test
def test_i_sometimes_fail
100.times do
setup
flaky_call()
teardown
end
end
end
Loop

View Slide

58
while true
do
bundle exec rake test || break
done
Loop

View Slide

Differences on the
Build Box
59
OS
Hardware
Performance
Paths, ﬁles, directories
Gem versions

View Slide

Threading
60
class MultiThreadedWorker
def go!
Thread.new do
work()
end
end
end

View Slide

Threading
61
def go!
Thread.new do
# TODO: ONLY FOR DIAGNOSISING!!!!!!!
sleep(0.1)
work()
end
end
end

View Slide

Threading
62
def go!
50.times do
Thread.new do
work()
end
end
end
end

View Slide

Threading
63
attr_reader :thread
def go!
@thread = Thread.new do
# ...
end
end
end

View Slide

Threading
64
class MultiThreadedWorkerTest < Minitest::Test
def test_worker
worker = MultiThreadedWorker.new
worker.go!
worker.thread.join
end
end

View Slide

TOOLS
65

View Slide

66
jasonrclark/flake_test
https://github.com/jasonrclark/ﬂake_test

View Slide

67
gnarg/test_bisect
https://github.com/gnarg/test_bisect

View Slide

gnarg/test_bisect
68
# Gemfile
gem 'test_bisect'
# Rakefile
require 'test_bisect'
TestBisect::BisectTask.new

View Slide

gnarg/test_bisect
69
class Test_01 < Test::Unit::TestCase
def test_truth
refute $globals_are_awesome
end
end

View Slide

70

View Slide

gnarg/test_bisect
71
def test_truth
refute $globals_are_awesome
end
end

View Slide

72

View Slide

73

View Slide

74

View Slide

gnarg/test_bisect
75
def test_filthy_lies
$globals_are_awesome = true
assert $globals_are_awesome
end
end

View Slide

gnarg/test_bisect
76
Test::Unit :(

View Slide

77
jasonrclark/hometown
https://github.com/jasonrclark/hometown

View Slide

78
# Gemfile
gem 'hometown'
# test_helper.rb
Hometown.watch(::Thread)

View Slide

79
class NoticeThreadTest < Minitest::Test
def setup
Thread.new { work! }
end
def test_notice_thread
Thread.list.each do |thread|
trace = Hometown.for(thread)
p trace unless trace.nil?
end
end
end

View Slide

80
#@traced_class=Thread,
@backtrace=[
"...loose_thread_test.rb:5:in `test_loose_...'",
"...minitest-5.3.4/lib/minitest/test.rb:106",
"...minitest-5.3.4/lib/minitest/test.rb:204",
"...minitest-5.3.4/lib/minitest/test.rb:103"
# ... >

View Slide

jasonrclark/minitest-stately
https://github.com/jasonrclark/minitest-stately
81

View Slide

82
# Gemfile
gem 'minitest-stately'
# test_helper.rb
# Watch for freshly started threads
Minitest::Stately.watch("thread count") do
Thread.list.count
end

View Slide

83

View Slide

84
Minitest::Stately.run do
$silly_global_state = nil
end

View Slide

85
Minitest::Stately.fail_if do
$silly_global_state == "seriously"
end

View Slide

Don't Let Your Tests Flake Out

Don't Let Your Tests Flake Out

Jason R Clark

More Decks by Jason R Clark

Other Decks in Technology

Featured

Transcript