Slide 1
Slide 1 text
The Future of Rails 6
Scalable by Default
Slide 2
Slide 2 text
Eileen M. Uchitelle
eileencodes.com
@eileencodes
Slide 3
Slide 3 text
a
Slide 4
Slide 4 text
Core Team
Slide 5
Slide 5 text
“Rails doesn’t scale!”
Slide 6
Slide 6 text
No content
Slide 7
Slide 7 text
Rails doesn’t scale
Slide 8
Slide 8 text
Rails doesn’t scale
easily
Slide 9
Slide 9 text
Rails doesn’t scale
easily (yet)
Slide 10
Slide 10 text
What does
“scalable” mean?
Slide 11
Slide 11 text
Your test suite shouldn’t
block development
Slide 12
Slide 12 text
Your application can
handle data & traffic
Slide 13
Slide 13 text
The Future of Rails 6
Scalable by Default
Slide 14
Slide 14 text
Parallel Testing
Speeding up your test suite
Slide 15
Slide 15 text
a
Slide 16
Slide 16 text
a
Slide 17
Slide 17 text
Forking
- vs -
Threads
Slide 18
Slide 18 text
Forking Processes
with dRB
Slide 19
Slide 19 text
# test/test_helper.rb
module ActiveSupport::TestCase
parallelize(workers: 2)
end
Slide 20
Slide 20 text
def parallelize(workers: 2, with: :processes)
workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
return if workers <= 1
executor = case with
when :processes
Testing::Parallelization.new(workers)
when :threads
Minitest::Parallel::Executor.new(workers)
else
raise ArgumentError, "#{with} is
not a supported parallelization executor."
end
self.lock_threads = false if defined?(self.lock_threads) &&
with == :threads
Minitest.parallel_executor = executor
parallelize_me!
end
Slide 21
Slide 21 text
def parallelize(workers: 2, with: :processes)
workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
return if workers <= 1
executor = case with
when :processes
Testing::Parallelization.new(workers)
when :threads
Minitest::Parallel::Executor.new(workers)
else
raise ArgumentError, "#{with} is
not a supported parallelization executor."
end
self.lock_threads = false if defined?(self.lock_threads) &&
with == :threads
Minitest.parallel_executor = executor
parallelize_me!
end
Slide 22
Slide 22 text
# activesupport/lib/active_support/testing/parallelization
module ActiveSupport::Testing::Parallelization
[…]
def initialize(queue_size)
@queue_size = queue_size
@queue = Server.new
@pool = []
@url = DRb.start_service("drbunix:", @queue).uri
end
[…]
end
Slide 23
Slide 23 text
# activesupport/lib/active_support/testing/parallelization
module ActiveSupport::Testing::Parallelization
[…]
def initialize(queue_size)
@queue_size = queue_size
@queue = Server.new
@pool = []
@url = DRb.start_service("drbunix:", @queue).uri
end
[…]
end
Slide 24
Slide 24 text
def parallelize(workers: 2, with: :processes)
workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
return if workers <= 1
executor = case with
when :processes
Testing::Parallelization.new(workers)
when :threads
Minitest::Parallel::Executor.new(workers)
else
raise ArgumentError, "#{with} is
not a supported parallelization executor."
end
self.lock_threads = false if defined?(self.lock_threads) &&
with == :threads
Minitest.parallel_executor = executor
parallelize_me!
end
Slide 25
Slide 25 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
DRb.stop_service
after_fork(worker)
queue = DRbObject.new_with_uri(@url)
while job = queue.pop
klass = job[0]
method = job[1]
[…]
Slide 26
Slide 26 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
DRb.stop_service
after_fork(worker)
queue = DRbObject.new_with_uri(@url)
while job = queue.pop
klass = job[0]
method = job[1]
[…]
Slide 27
Slide 27 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
DRb.stop_service
after_fork(worker)
queue = DRbObject.new_with_uri(@url)
while job = queue.pop
klass = job[0]
method = job[1]
[…]
Slide 28
Slide 28 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
DRb.stop_service
after_fork(worker)
queue = DRbObject.new_with_uri(@url)
while job = queue.pop
klass = job[0]
method = job[1]
[…]
Slide 29
Slide 29 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
[…]
while job = queue.pop
klass = job[0]
method = job[1]
reporter = job[2]
result = Minitest.run_one_method(klass, method)
queue.record(reporter, result)
end
run_cleanup(worker)
[…]
Slide 30
Slide 30 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
[…]
while job = queue.pop
klass = job[0]
method = job[1]
reporter = job[2]
result = Minitest.run_one_method(klass, method)
queue.record(reporter, result)
end
run_cleanup(worker)
[…]
Slide 31
Slide 31 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
[…]
while job = queue.pop
klass = job[0]
method = job[1]
reporter = job[2]
result = Minitest.run_one_method(klass, method)
queue.record(reporter, result)
end
run_cleanup(worker)
[…]
Slide 32
Slide 32 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
[…]
while job = queue.pop
klass = job[0]
method = job[1]
reporter = job[2]
result = Minitest.run_one_method(klass, method)
queue.record(reporter, result)
end
run_cleanup(worker)
[…]
Slide 33
Slide 33 text
module ActiveSupport::Testing::Parallelization
[…]
def start
[…]
end
def shutdown
@queue_size.times { @queue << nil }
@pool.each { |pid| Process.waitpid(pid) }
end
[…]
end
Slide 34
Slide 34 text
# test/test_helper.rb
module ActiveSupport::TestCase
parallelize(workers: 2)
end
Slide 35
Slide 35 text
test-database-0 test-database-1
Slide 36
Slide 36 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
DRb.stop_service
after_fork(worker)
queue = DRbObject.new_with_uri(@url)
while job = queue.pop
klass = job[0]
method = job[1]
[…]
Slide 37
Slide 37 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
[…]
while job = queue.pop
klass = job[0]
method = job[1]
reporter = job[2]
result = Minitest.run_one_method(klass, method)
queue.record(reporter, result)
end
run_cleanup(worker)
[…]
Slide 38
Slide 38 text
# test/test_helper.rb
module ActiveSupport::TestCase
parallelize(workers: 2)
parallelize_setup do |worker|
ActiveRecord::Tasks::DatabaseTasks.create(:animals)
ActiveRecord::Base.establish_connection(:animals)
ActiveRecord::Tasks::DatabaseTasks.load_schema
end
parallelize_teardown do |worker|
ActiveRecord::Tasks::DatabaseTasks.drop(:animals)
end
end
Slide 39
Slide 39 text
Threaded testing
with Minitest
Slide 40
Slide 40 text
# test/test_helper.rb
module ActiveSupport::TestCase
parallelize(workers: 2, with: :threads)
end
Slide 41
Slide 41 text
def parallelize(workers: 2, with: :processes)
workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
return if workers <= 1
executor = case with
when :processes
Testing::Parallelization.new(workers)
when :threads
Minitest::Parallel::Executor.new(workers)
else
raise ArgumentError, "#{with} is
not a supported parallelization executor."
end
self.lock_threads = false if defined?(self.lock_threads) &&
with == :threads
Minitest.parallel_executor = executor
parallelize_me!
end
Slide 42
Slide 42 text
No content
Slide 43
Slide 43 text
Connection ID:
#03938752
Connection ID:
#83321340
Slide 44
Slide 44 text
Connection ID:
#83321340
Connection ID:
#83321340
Slide 45
Slide 45 text
This is so weird. We
seem to have an
isolation problem…
Slide 46
Slide 46 text
No content
Slide 47
Slide 47 text
Rails
opens a
connection
Slide 48
Slide 48 text
Puma opens
a second
connection
Rails
opens a
connection
Slide 49
Slide 49 text
Puma opens
a second
connection
Rails
opens a
connection
Slide 50
Slide 50 text
Test thread
connection
number 1
Test thread
connection
number 2
Slide 51
Slide 51 text
def parallelize(workers: 2, with: :processes)
workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
return if workers <= 1
executor = case with
when :processes
Testing::Parallelization.new(workers)
when :threads
Minitest::Parallel::Executor.new(workers)
else
raise ArgumentError, "#{with} is
not a supported parallelization executor."
end
self.lock_threads = false if defined?(self.lock_threads) &&
with == :threads
Minitest.parallel_executor = executor
parallelize_me!
end
Slide 52
Slide 52 text
$ rails test
Slide 53
Slide 53 text
$ PARALLEL_WORKERS=14 rails test
Slide 54
Slide 54 text
def parallelize(workers: 2, with: :processes)
workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
return if workers <= 1
executor = case with
when :processes
Testing::Parallelization.new(workers)
when :threads
Minitest::Parallel::Executor.new(workers)
else
raise ArgumentError, "#{with} is
not a supported parallelization executor."
end
self.lock_threads = false if defined?(self.lock_threads) &&
with == :threads
Minitest.parallel_executor = executor
parallelize_me!
end
Slide 55
Slide 55 text
Parallel Testing PR
github.com/rails/rails/pull/31900
Slide 56
Slide 56 text
Your test suite shouldn’t
block development
Slide 57
Slide 57 text
Your application can
handle data & traffic
Slide 58
Slide 58 text
No content
Slide 59
Slide 59 text
No content
Slide 60
Slide 60 text
DB
1
DB
2
DB
3
Slide 61
Slide 61 text
Multiple Databases
Making multi-db behavior easy
Slide 62
Slide 62 text
Primary
Slide 63
Slide 63 text
DB
1
DB
2
DB
3
Slide 64
Slide 64 text
Table
id: integer
name: string
Table
id: integer
name: string
Table
id: integer
name: string
Table
id: integer
name: string
Primary Animals
Slide 65
Slide 65 text
X Best practices are undocumented
The State of Multi-db in Rails 5
Slide 66
Slide 66 text
# database.yml
production:
<<: *default
database: db_production
production_animals:
<<: *default
database: db_animals_production
Slide 67
Slide 67 text
# database.yml
production:
<<: *default
database: db_production
production_animals:
<<: *default
database: db_animals_production
Slide 68
Slide 68 text
# 3-tier database.yml
production:
primary:
<<: *default
database: db_production
animals:
<<: *default
database: db_animals_production
Slide 69
Slide 69 text
db/animals_migrate
↳ 20180206210429_create_dogs.rb
↳ 20180206210437_create_cats.rb
Slide 70
Slide 70 text
ActiveRecord::Migrator.migrations_paths =
"db/animals_migrate"
ActiveRecord::Base.migrate
Slide 71
Slide 71 text
X Best practices are undocumented
X Migrations don’t work
The State of Multi-db in Rails 5
Slide 72
Slide 72 text
# app/models/animals_base.rb
class AnimalsBase < ActiveRecord::Base
self.abstract_class = true
establish_connection :animals
end
Slide 73
Slide 73 text
# app/models/dog.rb
class Dog < AnimalsBase
[…]
end
# app/models/cat.rb
class Cat < AnimalsBase
[…]
end
Slide 74
Slide 74 text
X Best practices are undocumented
X Migrations don’t work
X Database tasks don’t exist
The State of Multi-db in Rails 5
Slide 75
Slide 75 text
ActiveRecord::AdapterNotSpecified:
'production' database is not configured.
Slide 76
Slide 76 text
# activerecord/lib/active_record/railtie.rb
initializer "active_record.initialize_database" do
ActiveSupport.on_load(:active_record) do
self.configurations =
Rails.application.config.database_configuration
begin
establish_connection
rescue ActiveRecord::NoDatabaseError
warn <<-end_warning
Slide 77
Slide 77 text
# activerecord/lib/active_record/railtie.rb
initializer "active_record.initialize_database" do
ActiveSupport.on_load(:active_record) do
self.configurations =
Rails.application.config.database_configuration
begin
establish_connection
rescue ActiveRecord::NoDatabaseError
warn <<-end_warning
Slide 78
Slide 78 text
X Best practices are undocumented
X Migrations don’t work
X Database tasks don’t exist
X Default connection was broken
The State of Multi-db in Rails 5
Slide 79
Slide 79 text
Migrations
Connections
don’t work
Rake tasks
Slide 80
Slide 80 text
No content
Slide 81
Slide 81 text
Fixing migrations for
multiple databases
Slide 82
Slide 82 text
path = Rails.root.to_s + "/db/animals_migrate"
desc "Migrate the cluster db"
task migrate: :environment do
ActiveRecord::Migrator.migrations_paths = path
ActiveRecord::Base.establish_connection(:animals)
ActiveRecord::Tasks::DatabaseTasks.migrate
db_namespace["db:schema:dump"].invoke
end
Slide 83
Slide 83 text
# 3-tier database.yml
production:
primary:
<<: *default
database: db_production
animals:
<<: *default
database: db_animals_production
migrations_paths: db/animals_migrate
Slide 84
Slide 84 text
>> Dog.connection.migrations_paths
=> "db/animals_migrate"
Slide 85
Slide 85 text
Migrations Paths Refactoring PR
github.com/rails/rails/pull/31727
Slide 86
Slide 86 text
X Best practices are undocumented
✔ Migrations don’t work
X Database tasks don’t exist
X Default connection was broken
The State of Multi-db in Rails 5
Slide 87
Slide 87 text
Migrations
Connections
don’t work
Rake tasks
Slide 88
Slide 88 text
Improving the database
tasks for multi-dbs
Slide 89
Slide 89 text
rails db:drop
rails db:create
rails db:migrate
Slide 90
Slide 90 text
>> ActiveRecord::Base.configurations["production"]
>> {
"adapter"=>"mysql2",
"database"=>"single_database_production"
}
Slide 91
Slide 91 text
>> ActiveRecord::Base.configurations["production"]
>> {
"primary"=>{"adapter"=>"mysql2",
"database"=>"multiple_databases_production"},
"animals"=>{"adapter"=>"mysql2",
“database"=>"multiple_databases_production"}
}
Slide 92
Slide 92 text
module ActiveRecord
module DatabaseConfigurations
class DatabaseConfig
attr_reader :env_name, :spec_name, :config
def initialize(env_name, spec_name, config)
@env_name = env_name
@spec_name = spec_name
@config = config
end
end
end
end
Slide 93
Slide 93 text
module ActiveRecord
module DatabaseConfigurations
class DatabaseConfig
attr_reader :env_name, :spec_name, :config
def initialize(env_name, spec_name, config)
@env_name = env_name
@spec_name = spec_name
@config = config
end
end
end
end
Slide 94
Slide 94 text
module ActiveRecord
module DatabaseConfigurations
class DatabaseConfig
attr_reader :env_name, :spec_name, :config
def initialize(env_name, spec_name, config)
@env_name = env_name
@spec_name = spec_name
@config = config
end
end
end
end
Slide 95
Slide 95 text
module ActiveRecord
module DatabaseConfigurations
class DatabaseConfig
attr_reader :env_name, :spec_name, :config
def initialize(env_name, spec_name, config)
@env_name = env_name
@spec_name = spec_name
@config = config
end
end
end
end
Slide 96
Slide 96 text
>> db_config = DatabaseConfig.new("production",
"animals", {"adapter"=>"mysql2",
"database"=>"multiple_databases_production_animals"
})
>>
#"mysql2",
"database"=>"multiple_databases_production_animals"
}>
Slide 97
Slide 97 text
>> db_config.env_name
"production"
Slide 98
Slide 98 text
>> db_config.env_name
"production"
>> db_config.spec_name
"animals"
Slide 99
Slide 99 text
>> db_config.env_name
"production"
>> db_config.spec_name
"animals"
>> db_config.config
{…"database"=>"multiple_databases_producti
on_animals"}
Slide 100
Slide 100 text
def self.db_configs(configs = configurations)
configs.each_pair.flat_map do |env_name, config|
walk_configs(env_name, "primary", config)
end
end
Slide 101
Slide 101 text
def self.configs_for(env, configs, &blk)
env_with_configs =
db_configs(configs).select do |db_config|
db_config.env_name == env
end
if block_given?
env_with_configs.each do |ewc|
yield ewc.spec_name, ewc.config
end
else
env_with_configs
end
end
Slide 102
Slide 102 text
def self.configs_for(env, configs, &blk)
env_with_configs =
db_configs(configs).select do |db_config|
db_config.env_name == env
end
if block_given?
env_with_configs.each do |ewc|
yield ewc.spec_name, ewc.config
end
else
env_with_configs
end
end
Slide 103
Slide 103 text
def self.configs_for(env, configs, &blk)
env_with_configs =
db_configs(configs).select do |db_config|
db_config.env_name == env
end
if block_given?
env_with_configs.each do |ewc|
yield ewc.spec_name, ewc.config
end
else
env_with_configs
end
end
Slide 104
Slide 104 text
>> DatabaseConfigurations.configs_for("production")
>>[#"mysql2",
"database"=>"multiple_databases_production"}>,
#"mysql2",
"database"=>"multiple_databases_production_animals"
}>]
Slide 105
Slide 105 text
task migrate: :load_config do
ActiveRecord::Tasks::DatabaseTasks.migrate
db_namespace["_dump"].invoke
end
Slide 106
Slide 106 text
task migrate: :load_config do
ActiveRecord::DatabaseConfigurations.configs_for(
Rails.env
) do |_, config|
ActiveRecord::Base.establish_connection(config)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
db_namespace["_dump"].invoke
end
Slide 107
Slide 107 text
task migrate: :load_config do
ActiveRecord::DatabaseConfigurations.configs_for(
Rails.env
) do |_, config|
ActiveRecord::Base.establish_connection(config)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
db_namespace["_dump"].invoke
end
Slide 108
Slide 108 text
task migrate: :load_config do
ActiveRecord::DatabaseConfigurations.configs_for(
Rails.env
) do |_, config|
ActiveRecord::Base.establish_connection(config)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
db_namespace["_dump"].invoke
end
Slide 109
Slide 109 text
task migrate: :load_config do
ActiveRecord::DatabaseConfigurations.configs_for(
Rails.env
) do |_, config|
ActiveRecord::Base.establish_connection(config)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
db_namespace["_dump"].invoke
end
Slide 110
Slide 110 text
rails db:drop
rails db:create
rails db:migrate
Slide 111
Slide 111 text
rails db:drop:primary
rails db:drop:animals
rails db:create:primary
rails db:create:animals
rails db:migrate:primary
rails db:migrate:animals
Slide 112
Slide 112 text
namespace :migrate
ActiveRecord::Tasks::DatabaseTasks.for_each do |spec_name|
desc "Migrate #{spec_name} database"
task spec_name => :load_config do
db_config = ActiveRecord::DatabaseConfigurations
.config_for_env_and_spec(Rails.env, spec_name)
ActiveRecord::Base.establish_connection(
db_config.config
)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
end
end
end
Slide 113
Slide 113 text
namespace :migrate
ActiveRecord::Tasks::DatabaseTasks.for_each do |spec_name|
desc "Migrate #{spec_name} database"
task spec_name => :load_config do
db_config = ActiveRecord::DatabaseConfigurations
.config_for_env_and_spec(Rails.env, spec_name)
ActiveRecord::Base.establish_connection(
db_config.config
)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
end
end
end
Slide 114
Slide 114 text
namespace :migrate
ActiveRecord::Tasks::DatabaseTasks.for_each do |spec_name|
desc "Migrate #{spec_name} database"
task spec_name => :load_config do
db_config = ActiveRecord::DatabaseConfigurations
.config_for_env_and_spec(Rails.env, spec_name)
ActiveRecord::Base.establish_connection(
db_config.config
)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
end
end
end
Slide 115
Slide 115 text
namespace :migrate
ActiveRecord::Tasks::DatabaseTasks.for_each do |spec_name|
desc "Migrate #{spec_name} database"
task spec_name => :load_config do
db_config = ActiveRecord::DatabaseConfigurations
.config_for_env_and_spec(Rails.env, spec_name)
ActiveRecord::Base.establish_connection(
db_config.config
)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
end
end
end
Slide 116
Slide 116 text
namespace :migrate
ActiveRecord::Tasks::DatabaseTasks.for_each do |spec_name|
desc "Migrate #{spec_name} database"
task spec_name => :load_config do
db_config = ActiveRecord::DatabaseConfigurations
.config_for_env_and_spec(Rails.env, spec_name)
ActiveRecord::Base.establish_connection(
db_config.config
)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
end
end
end
Slide 117
Slide 117 text
rails db:drop:primary
rails db:drop:animals
rails db:create:primary
rails db:create:animals
rails db:migrate:primary
rails db:migrate:animals
Slide 118
Slide 118 text
Database task improvements
github.com/rails/rails/pull/32274
github.com/rails/rails/compare/
ab43b5405ea2...5ddcda6d5f56
Slide 119
Slide 119 text
X Best practices are undocumented
✔ Migrations don’t work
✔ Database tasks don’t exist
X Default connection was broken
The State of Multi-db in Rails 5
Slide 120
Slide 120 text
Connections
don’t work
Migrations
Rake tasks
Slide 121
Slide 121 text
No content
Slide 122
Slide 122 text
Thanks
@tenderlove!
Slide 123
Slide 123 text
“Are you concerned
that Rails will become
bloated?”
Slide 124
Slide 124 text
What would Rails look like
if we’d upstreamed these
features 5 years ago?
Slide 125
Slide 125 text
a +
Slide 126
Slide 126 text
' (
)
*
+
,
Slide 127
Slide 127 text
No content
Slide 128
Slide 128 text
Maybe Rails doesn’t
scale…?
Slide 129
Slide 129 text
Your application
isn’t special.
Slide 130
Slide 130 text
Forces you to write
generic code
Slide 131
Slide 131 text
Train your future
workforce
Slide 132
Slide 132 text
Give back to the
community
Slide 133
Slide 133 text
'
(
)
*
+
,
Slide 134
Slide 134 text
a♥
Slide 135
Slide 135 text
Scaling should
make us happy
Slide 136
Slide 136 text
Rails isn’t dying,
it’s maturing
Slide 137
Slide 137 text
No content
Slide 138
Slide 138 text
No content
Slide 139
Slide 139 text
No content
Slide 140
Slide 140 text
“Rails does scale!”
Slide 141
Slide 141 text
The Future of Rails 6
Scalable by Default
Slide 142
Slide 142 text
Eileen M. Uchitelle
eileencodes.com
@eileencodes