Cookbook Refactoring and Extracting Logic into Rubygems

Slides from my ChefConf 2013

Transcript

1. Cookbook
   Refactoring
   A
   

   View Slide

2. Cookbook
   Refactoring
   ... and extracting logic into Rubygems
   A
   

   View Slide

3. sethvargo@opscode.com
   E
   b
   yz
   

   View Slide

4. View Slide

5. We're Hiring!
   

   View Slide

6. We're Hiring!
   Colorado
   

   View Slide

7. New Branding
   We're Hiring!
   

   View Slide

8. U
   DO YOU SOMETIMES
   FEEL LIKE
   THIS
   

   View Slide

9. View Slide

10. View Slide

11. template '/etc/hosts' do
    owner 'root'
    group 'root'
    source 'etc/hosts'
    end
    recipes/default.rb
    

    View Slide

12. # This file is managed by Chef for "<%= node['fqdn'] %>"
    # Do NOT modify this file by hand.
    <%= node['ipaddress'] %> <%= node['fqdn'] %>
    127.0.0.1!
    localhost <%= node['fqdn'] %>
    255.255.255.255!broadcasthost
    ::1 localhost
    fe80::1%lo0! localhost
    templates/default/etc/hosts.erb
    

    View Slide

13. View Slide

14. default['etc']['hosts'] = [] unless node['etc']['hosts']
    attributes/default.rb
    

    View Slide

15. # This file is managed by Chef for "<%= node['fqdn'] %>"
    # Do NOT modify this file by hand.
    <%= node['ipaddress'] %> <%= node['fqdn'] %>
    127.0.0.1!
    localhost <%= node['fqdn'] %>
    255.255.255.255!broadcasthost
    ::1 localhost
    fe80::1%lo0! localhost
    # Custom Entries
    <% node['etc']['hosts'].each do |h| -%>
    <%= h['ip'] %> <%= h['host'] %>
    <% end -%>
    templates/default/etc/hosts.erb
    

    View Slide

16. include_attribute 'hostsfile'
    default['etc']['hosts'] << {
    'ip' => '1.2.3.4',
    'host' => 'www.example.com'
    }
    other_cookbook/attributes/default.rb
    

    View Slide

17. node.default['etc']['hosts'] << {
    'ip' => '1.2.3.4',
    'host' => 'www.example.com'
    }
    other_cookbook/recipes/default.rb
    

    View Slide

18. default_attributes({
    'etc' => {
    'hosts' => [
    {'ip' => '1.2.3.4', 'host' => 'www.example.com'},
    {'ip' => '4.5.6.7', 'host' => 'foo.example.com'}
    ]
    }
    })
    roles/my_role.rb
    

    View Slide

19. {
    "default_attributes": {
    "etc": {
    "hosts": [
    {"ip": "1.2.3.4", "host": "www.example.com"},
    {"ip": "4.5.6.7", "host": "foo.example.com"}
    ]
    }
    }
    }
    environments/production.json
    

    View Slide

20. View Slide

21. node.set['etc']['hosts'] = {
    ip: '7.8.9.0',
    host: 'bar.example.com'
    })
    recipes/default.rb
    

    View Slide

22. View Slide

23. arr = [1,2,3]
    arr << 4 => [1,2,3,4]
    arr = 4 => 4
    

    View Slide

24. arr = [1,2,3]
    arr << 4 => [1,2,3,4]
    arr = 4 => 4
    Not an Array
    

    View Slide

25. TODO: Add infographics
    # This file is managed by Chef for "www.myapp.com"
    # Do NOT modify this file by hand.
    1.2.3.4 www.myapp.com
    127.0.0.1!
    localhost www.myapp.com
    255.255.255.255!broadcasthost
    ::1 localhost
    fe80::1%lo0! localhost
    # Custom Entries
    1.2.3.4 www.example.com
    4.5.6.7 foo.example.com
    7.8.9.0 bar.example.com
    /etc/hosts
    

    View Slide

26. TODO: Add infographics
    # This file is managed by Chef for "www.myapp.com"
    # Do NOT modify this file by hand.
    1.2.3.4 www.myapp.com
    127.0.0.1!
    localhost www.myapp.com
    255.255.255.255!broadcasthost
    ::1 localhost
    fe80::1%lo0! localhost
    # Custom Entries
    7.8.9.0 bar.example.com
    /etc/hosts
    

    View Slide

27. Post Mortem
    

    View Slide

28. View Slide

29. View Slide

30. << =
    

    View Slide

31. << =
    !=
    

    View Slide

32. Post Mortem
    Action Items
    7
    

    View Slide

33. Monkey patch Chef to raise an
    exception when redefining that
    particular node attribute.
    

    View Slide

34. Monkey patch Chef to raise an
    exception when redefining that
    particular node attribute.
    t
    

    View Slide

35. Create a special cookbook that
    uses a threshold value and raises an
    exception if the size of the array
    doesn't "make sense".
    

    View Slide

36. Create a special cookbook that
    uses a threshold value and raises an
    exception if the size of the array
    doesn't "make sense".
    t
    

    View Slide

37. Move all entries to a data bag
    

    View Slide

38. Move all entries to a data bag
    u
    

    View Slide

39. Move all entries to a data bag
    6
    6 Add tests
    

    View Slide

40. Data Bags
    

    View Slide

41. [
    "1.2.3.4 example.com www.example.com",
    "4.5.6.7 foo.example.com",
    "7.8.9.0 bar.example.com"
    ]
    data_bags/etc_hosts.json
    

    View Slide

42. hosts = data_bag('etc_hosts')
    template '/etc/hosts' do
    owner 'root'
    group 'root'
    source 'etc/hosts'
    variables(
    hosts: hosts
    )
    end
    recipes/default.rb
    

    View Slide

43. # This file is managed by Chef for "<%= node['fqdn'] %>"
    # Do NOT modify this file by hand.
    <%= node['ipaddress'] %> <%= node['fqdn'] %>
    127.0.0.1!
    localhost <%= node['fqdn'] %>
    255.255.255.255!broadcasthost
    ::1 localhost
    fe80::1%lo0! localhost
    # Custom Entries
    <%= @hosts.join("\n") %>
    templates/default/etc/hosts.erb
    

    View Slide

44. Move all entries to a data bag
    5
    6 Add tests
    

    View Slide

45. require 'chefspec'
    spec/default_spec.rb
    

    View Slide

46. require 'chefspec'
    describe 'hostsfile::default' do
    end
    spec/default_spec.rb
    

    View Slide

47. require 'chefspec'
    describe 'hostsfile::default' do
    let(:hosts) { ['1.2.3.4 example.com', '4.5.6.7 bar.com'] }
    before do
    Chef::Recipe.any_instance.stub(:data_bag).with('etc_hosts').and_return(hosts)
    end
    end
    spec/default_spec.rb
    

    View Slide

48. require 'chefspec'
    describe 'hostsfile::default' do
    let(:hosts) { ['1.2.3.4 example.com', '4.5.6.7 bar.com'] }
    before do
    Chef::Recipe.any_instance.stub(:data_bag).with('etc_hosts').and_return(hosts)
    end
    let(:runner) { ChefSpec::ChefRunner.new.converge('hostsfile::default') }
    end
    spec/default_spec.rb
    

    View Slide

49. require 'chefspec'
    describe 'hostsfile::default' do
    let(:hosts) { ['1.2.3.4 example.com', '4.5.6.7 bar.com'] }
    before do
    Chef::Recipe.any_instance.stub(:data_bag).with('etc_hosts').and_return(hosts)
    end
    let(:runner) { ChefSpec::ChefRunner.new.converge('hostsfile::default') }
    it 'loads the data bag' do
    Chef::Recipe.any_instance.should_receive(:data_bag).with('etc_hosts')
    end
    end
    spec/default_spec.rb
    

    View Slide

50. require 'chefspec'
    describe 'hostsfile::default' do
    let(:hosts) { ['1.2.3.4 example.com', '4.5.6.7 bar.com'] }
    before do
    Chef::Recipe.any_instance.stub(:data_bag).with('etc_hosts').and_return(hosts)
    end
    let(:runner) { ChefSpec::ChefRunner.new.converge('hostsfile::default') }
    it 'loads the data bag' do
    Chef::Recipe.any_instance.should_receive(:data_bag).with('etc_hosts')
    end
    it 'creates the /etc/hosts template' do
    expect(runner).to create_template('/etc/hosts').with_content(hosts.join("\n"))
    end
    end
    spec/default_spec.rb
    

    View Slide

51. $ rspec cookbooks/hostsfile
    Running all specs
    

    View Slide

52. $ rspec cookbooks/hostsfile
    Running all specs
    **
    Finished in 0.0003 seconds
    2 examples, 0 failures
    

    View Slide

53. $ rspec cookbooks/hostsfile
    Running all specs
    **
    Finished in 0.0003 seconds
    2 examples, 0 failures
    Really Fucking Fast™
    

    View Slide

54. #winning
    

    View Slide

55. 10,000 tests
    

    View Slide

56. 28 seconds
    

    View Slide

57. #winning
    

    View Slide

58. ⏳
    ⏳
    

    View Slide

59. View Slide

60. View Slide

61. hosts = data_bag('etc_hosts')
    hosts << search(:node, 'role:mongo_master').first.tap do |n|
    "#{n['ip_address']} #{n['fqdn']}"
    end
    template '/etc/hosts' do
    owner 'root'
    group 'root'
    source 'etc/hosts'
    variables(
    hosts: hosts
    )
    end
    recipes/default.rb
    

    View Slide

62. hosts = data_bag('etc_hosts')
    hosts << search(:node, 'role:mongo_master').first.tap do |n|
    "#{n['ip_address']} #{n['fqdn']}"
    end
    hosts << search(:node, 'role:mysql_master').first.tap do |n|
    "#{n['ip_address']} #{n['fqdn']}"
    end
    hosts << search(:node, 'role:redis_master').first.tap do |n|
    "#{n['ip_address']} #{n['fqdn']}"
    end
    template '/etc/hosts' do
    owner 'root'
    group 'root'
    recipes/default.rb
    

    View Slide

63. LWRPs
    

    View Slide

64. # List of all actions supported by the provider
    actions :create, :create_if_missing, :update, :remove
    # Make create the default action
    default_action :create
    # Required attributes
    attribute :ip_address,
    kind_of: String,
    name_attribute: true,
    required: true
    attribute :hostname, kind_of: String
    # Optional attributes
    attribute :aliases, kind_of: Array
    attribute :comment, kind_of: String
    resources/entry.rb
    

    View Slide

65. action :create do
    ::Chef::Util::FileEdit.search_file_delete_line(entry)
    ::Chef::Util::FileEdit.insert_line_after_match(/\n/, entry)
    end
    protected
    def entry
    [new_resource.ip_address, new_resource.hostname,
    new_resource.aliases.join(' ')].compact.join(' ').squeeze(' ')
    end
    providers/entry.rb
    

    View Slide

66. hostsfile_entry '1.2.3.4' do
    hostname 'example.com'
    end
    providers/entry.rb
    

    View Slide

67. Chef::Util::FileEdit is slow
    

    View Slide

68. Re-writing the file on each run
    

    View Slide

69. Provider kept growning
    

    View Slide

70. Untested
    

    View Slide

71. Refactor
    A
    

    View Slide

72. Move to pure Ruby classes
    

    View Slide

73. Ditch Chef::Util::FileEdit and
    manage the entire file
    

    View Slide

74. Only implement Ruby classes in
    the Provider (logic-less Provider)
    

    View Slide

75. Test the Ruby code
    

    View Slide

76. Test that the Provider implements
    the proper Ruby classes
    

    View Slide

77. TODO: Add infographics
    class Entry
    attr_accessor :ip_address, :hostname, :aliases, :comment
    def initialize(options = {})
    if options[:ip_address].nil? || options[:hostname].nil?
    raise ':ip_address and :hostname are both required options'
    end
    @ip_address = options[:ip_address]
    @hostname = options[:hostname]
    @aliases = [options[:aliases]].flatten
    @comment = options[:comment]
    end
    # ...
    end
    libraries/entry.rb
    

    View Slide

78. TODO: Add infographics
    class Manipulator
    def initialize
    contents = ::File.readlines(hostsfile_path)
    @entries = contents.collect do |line|
    Entry.parse(line) unless line.strip.nil? || line.strip.empty?
    end.compact
    end
    def add(options = {})
    @entries << Entry.new(
    ip_address: options[:ip_address],
    hostname: options[:hostname],
    aliases: options[:aliases],
    comment: options[:comment]
    )
    end
    end
    libraries/manipulator.rb
    

    View Slide

79. # Creates a new hosts file entry. If an entry already exists, it
    # will be overwritten by this one.
    action :create do
    hostsfile.add(
    ip_address: new_resource.ip_address,
    hostname: new_resource.hostname,
    aliases: new_resource.aliases,
    comment: new_resource.comment
    )
    new_resource.updated_by_last_action(true) if hostsfile.save
    end
    providers/entry.rb
    

    View Slide

80. View Slide

81. View Slide

82. RSpec
    

    View Slide

83. TODO: Add infographics
    describe Entry do
    describe '.initialize' do
    subject { Entry.new(ip_address: '2.3.4.5', hostname:
    'www.example.com', aliases: ['foo', 'bar'], comment: 'This is a
    comment!', priority: 100) }
    it 'raises an exception if :ip_address is missing' do
    expect {
    Entry.new(hostname: 'www.example.com')
    }.to raise_error(ArgumentError)
    end
    it 'sets the ip_address' do
    expect(subject.ip_address).to eq('2.3.4.5')
    end
    end
    spec/entry_spec.rb
    

    View Slide

84. View Slide

85. Chef Spec
    

    View Slide

86. Chef Spec
    

    View Slide

87. TODO: Add infographics
    describe 'hostsfile lwrp' do
    let(:manipulator) { double('manipulator') }
    before do
    Manipulator.stub(:new).and_return(manipulator)
    Manipulator.should_receive(:new).with(kind_of(Chef::Node))
    .and_return(manipulator)
    manipulator.should_receive(:save!)
    end
    let(:chef_run) {
    ChefSpec::ChefRunner.new(
    cookbook_path: $cookbook_paths,
    step_into: ['hostsfile_entry']
    )
    }
    spec/default_spec.rb
    

    View Slide

88. TODO: Add infographics
    context 'actions' do
    describe ':create' do
    it 'adds the entry' do
    manipulator.should_receive(:add).with({
    ip_address: '2.3.4.5',
    hostname: 'www.example.com',
    aliases: nil,
    comment: nil,
    priority: nil
    })
    chef_run.converge('fake::create')
    end
    end
    end
    end
    

    View Slide

89. Open It
    

    View Slide

90. View Slide

91. View Slide

92. View Slide

93. View Slide

94. View Slide

95. Gem It
    

    View Slide

96. $ bundle gem hostsfile
    

    View Slide

97. $ bundle gem hostsfile
    create hostsfile/Gemfile
    create hostsfile/Rakefile
    create hostsfile/LICENSE.txt
    create hostsfile/README.md
    create hostsfile/.gitignore
    create hostsfile/hostsfile.gemspec
    create hostsfile/lib/hostsfile.rb
    create hostsfile/lib/hostsfile/version.rb
    Initializating git repo in ~Development/hostsfile
    

    View Slide

98. entry.rb
    manipulator.rb
    9
    9
    

    View Slide

99. View Slide

100. 9
     ?
     

     View Slide

101. chef_gem 'hostsfile'
     recipes/default.rb
     

     View Slide

102. require 'hostsfile'
     providers/entry.rb
     

     View Slide

103. In another cookbook...
     

     View Slide

104. # ...
     depends 'hostsfile'
     other_cookbook/metadata.rb
     

     View Slide

105. {
     "run_list": [
     "recipe[hostsfile]"
     ]
     }
     www.myapp.com (Chef Node)
     

     View Slide

106. View Slide

107. View Slide

108. View Slide

109. Thank
     You
     z
     

     View Slide