Slide 1
Slide 1 text
The Future of Rails 6
Scalable by Default
Slide 2
Slide 2 text
Eileen M. Uchitelle
eileencodes.com
@eileencodes
Slide 3
Slide 3 text
a
Slide 4
Slide 4 text
Core Team
Slide 5
Slide 5 text
“Rails doesn’t scale!”
Slide 6
Slide 6 text
No content
Slide 7
Slide 7 text
Rails doesn’t scale
Slide 8
Slide 8 text
Rails doesn’t scale
easily
Slide 9
Slide 9 text
Rails doesn’t scale
easily (yet)
Slide 10
Slide 10 text
What does
“scalable” mean?
Slide 11
Slide 11 text
Your test suite shouldn’t
block development
Slide 12
Slide 12 text
Your application can
handle data & traffic
Slide 13
Slide 13 text
The Future of Rails 6
Scalable by Default
Slide 14
Slide 14 text
Parallel Testing
Speeding up your test suite
Slide 15
Slide 15 text
a
Slide 16
Slide 16 text
a
Slide 17
Slide 17 text
Forking
- vs -
Threads
Slide 18
Slide 18 text
Forking Processes
with dRB
Slide 19
Slide 19 text
# test/test_helper.rb
module ActiveSupport::TestCase
parallelize(workers: 2)
end
Slide 20
Slide 20 text
def parallelize(workers: 2, with: :processes)
workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
return if workers <= 1
executor = case with
when :processes
Testing::Parallelization.new(workers)
when :threads
Minitest::Parallel::Executor.new(workers)
else
raise ArgumentError, "#{with} is
not a supported parallelization executor."
end
self.lock_threads = false if defined?(self.lock_threads) &&
with == :threads
Minitest.parallel_executor = executor
parallelize_me!
end
Slide 21
Slide 21 text
def parallelize(workers: 2, with: :processes)
workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
return if workers <= 1
executor = case with
when :processes
Testing::Parallelization.new(workers)
when :threads
Minitest::Parallel::Executor.new(workers)
else
raise ArgumentError, "#{with} is
not a supported parallelization executor."
end
self.lock_threads = false if defined?(self.lock_threads) &&
with == :threads
Minitest.parallel_executor = executor
parallelize_me!
end
Slide 22
Slide 22 text
def parallelize(workers: 2, with: :processes)
workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
return if workers <= 1
executor = case with
when :processes
Testing::Parallelization.new(workers)
when :threads
Minitest::Parallel::Executor.new(workers)
else
raise ArgumentError, "#{with} is
not a supported parallelization executor."
end
self.lock_threads = false if defined?(self.lock_threads) &&
with == :threads
Minitest.parallel_executor = executor
parallelize_me!
end
Slide 23
Slide 23 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
DRb.stop_service
after_fork(worker)
queue = DRbObject.new_with_uri(@url)
while job = queue.pop
klass = job[0]
method = job[1]
[…]
Slide 24
Slide 24 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
DRb.stop_service
after_fork(worker)
queue = DRbObject.new_with_uri(@url)
while job = queue.pop
klass = job[0]
method = job[1]
[…]
Slide 25
Slide 25 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
DRb.stop_service
after_fork(worker)
queue = DRbObject.new_with_uri(@url)
while job = queue.pop
klass = job[0]
method = job[1]
[…]
Slide 26
Slide 26 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
DRb.stop_service
after_fork(worker)
queue = DRbObject.new_with_uri(@url)
while job = queue.pop
klass = job[0]
method = job[1]
[…]
Slide 27
Slide 27 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
[…]
while job = queue.pop
klass = job[0]
method = job[1]
reporter = job[2]
result = Minitest.run_one_method(klass, method)
queue.record(reporter, result)
end
run_cleanup(worker)
[…]
Slide 28
Slide 28 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
[…]
while job = queue.pop
klass = job[0]
method = job[1]
reporter = job[2]
result = Minitest.run_one_method(klass, method)
queue.record(reporter, result)
end
run_cleanup(worker)
[…]
Slide 29
Slide 29 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
[…]
while job = queue.pop
klass = job[0]
method = job[1]
reporter = job[2]
result = Minitest.run_one_method(klass, method)
queue.record(reporter, result)
end
run_cleanup(worker)
[…]
Slide 30
Slide 30 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
[…]
while job = queue.pop
klass = job[0]
method = job[1]
reporter = job[2]
result = Minitest.run_one_method(klass, method)
queue.record(reporter, result)
end
run_cleanup(worker)
[…]
Slide 31
Slide 31 text
module ActiveSupport::Testing::Parallelization
[…]
def start
[…]
end
def shutdown
@queue_size.times { @queue << nil }
@pool.each { |pid| Process.waitpid(pid) }
end
[…]
end
Slide 32
Slide 32 text
# test/test_helper.rb
module ActiveSupport::TestCase
parallelize(workers: 2)
end
Slide 33
Slide 33 text
test-database-0 test-database-1
Slide 34
Slide 34 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
DRb.stop_service
after_fork(worker)
queue = DRbObject.new_with_uri(@url)
while job = queue.pop
klass = job[0]
method = job[1]
[…]
Slide 35
Slide 35 text
module ActiveSupport::Testing::Parallelization
[…]
def start
@pool = @queue_size.times.map do |worker|
fork do
[…]
while job = queue.pop
klass = job[0]
method = job[1]
reporter = job[2]
result = Minitest.run_one_method(klass, method)
queue.record(reporter, result)
end
run_cleanup(worker)
[…]
Slide 36
Slide 36 text
Threaded testing
with Minitest
Slide 37
Slide 37 text
# test/test_helper.rb
module ActiveSupport::TestCase
parallelize(workers: 2, with: :threads)
end
Slide 38
Slide 38 text
def parallelize(workers: 2, with: :processes)
workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
return if workers <= 1
executor = case with
when :processes
Testing::Parallelization.new(workers)
when :threads
Minitest::Parallel::Executor.new(workers)
else
raise ArgumentError, "#{with} is
not a supported parallelization executor."
end
self.lock_threads = false if defined?(self.lock_threads) &&
with == :threads
Minitest.parallel_executor = executor
parallelize_me!
end
Slide 39
Slide 39 text
No content
Slide 40
Slide 40 text
Connection ID:
#03938752
Connection ID:
#83321340
Slide 41
Slide 41 text
Connection ID:
#83321340
Connection ID:
#83321340
Slide 42
Slide 42 text
This is so weird. We
seem to have an
isolation problem…
Slide 43
Slide 43 text
No content
Slide 44
Slide 44 text
Rails
opens a
connection
Slide 45
Slide 45 text
Puma opens
a second
connection
Rails
opens a
connection
Slide 46
Slide 46 text
Puma opens
a second
connection
Rails
opens a
connection
Slide 47
Slide 47 text
Test thread
connection
number 1
Test thread
connection
number 2
Slide 48
Slide 48 text
def parallelize(workers: 2, with: :processes)
workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
return if workers <= 1
executor = case with
when :processes
Testing::Parallelization.new(workers)
when :threads
Minitest::Parallel::Executor.new(workers)
else
raise ArgumentError, "#{with} is
not a supported parallelization executor."
end
self.lock_threads = false if defined?(self.lock_threads) &&
with == :threads
Minitest.parallel_executor = executor
parallelize_me!
end
Slide 49
Slide 49 text
$ rails test
Slide 50
Slide 50 text
Parallel Testing PR
github.com/rails/rails/pull/31900
Slide 51
Slide 51 text
Your test suite shouldn’t
block development
Slide 52
Slide 52 text
Your application can
handle data & traffic
Slide 53
Slide 53 text
No content
Slide 54
Slide 54 text
No content
Slide 55
Slide 55 text
DB
1
DB
2
DB
3
Slide 56
Slide 56 text
Multiple Databases
Making multi-db behavior easy
Slide 57
Slide 57 text
Table
id: integer
name: string
Table
id: integer
name: string
Table
id: integer
name: string
Table
id: integer
name: string
Primary Animals
Slide 58
Slide 58 text
X Best practices are undocumented
The State of Multi-db in Rails 5
Slide 59
Slide 59 text
# 3-tier database.yml
production:
primary:
<<: *default
database: db_production
animals:
<<: *default
database: db_animals_production
Slide 60
Slide 60 text
db/animals_migrate
↳ 20180206210429_create_dogs.rb
↳ 20180206210437_create_cats.rb
Slide 61
Slide 61 text
ActiveRecord::Migrator.migrations_paths =
"db/animals_migrate"
ActiveRecord::Base.migrate
Slide 62
Slide 62 text
X Best practices are undocumented
X Migrations don’t work
The State of Multi-db in Rails 5
Slide 63
Slide 63 text
# app/models/animals_base.rb
class AnimalsBase < ActiveRecord::Base
self.abstract_class = true
establish_connection :animals
end
Slide 64
Slide 64 text
# app/models/dog.rb
class Dog < AnimalsBase
[…]
end
# app/models/cat.rb
class Cat < AnimalsBase
[…]
end
Slide 65
Slide 65 text
X Best practices are undocumented
X Migrations don’t work
X Database tasks don’t exist
The State of Multi-db in Rails 5
Slide 66
Slide 66 text
ActiveRecord::AdapterNotSpecified:
'production' database is not configured.
Slide 67
Slide 67 text
# activerecord/lib/active_record/railtie.rb
initializer "active_record.initialize_database" do
ActiveSupport.on_load(:active_record) do
self.configurations =
Rails.application.config.database_configuration
begin
establish_connection
rescue ActiveRecord::NoDatabaseError
warn <<-end_warning
Slide 68
Slide 68 text
# activerecord/lib/active_record/railtie.rb
initializer "active_record.initialize_database" do
ActiveSupport.on_load(:active_record) do
self.configurations =
Rails.application.config.database_configuration
begin
establish_connection
rescue ActiveRecord::NoDatabaseError
warn <<-end_warning
Slide 69
Slide 69 text
X Best practices are undocumented
X Migrations don’t work
X Database tasks don’t exist
X Default connection was broken
The State of Multi-db in Rails 5
Slide 70
Slide 70 text
Migrations
Connections
don’t work
Rake tasks
Slide 71
Slide 71 text
No content
Slide 72
Slide 72 text
Fixing migrations for
multiple databases
Slide 73
Slide 73 text
path = Rails.root.to_s + "/db/animals_migrate"
desc "Migrate the cluster db"
task migrate: :environment do
ActiveRecord::Migrator.migrations_paths = path
ActiveRecord::Base.establish_connection(:animals)
ActiveRecord::Tasks::DatabaseTasks.migrate
db_namespace["db:schema:dump"].invoke
end
Slide 74
Slide 74 text
# 3-tier database.yml
production:
primary:
<<: *default
database: db_production
animals:
<<: *default
database: db_animals_production
migrations_paths: db/animals_migrate
Slide 75
Slide 75 text
>> Dog.connection.migrations_paths
=> "db/animals_migrate"
Slide 76
Slide 76 text
Migrations Paths Refactoring PR
github.com/rails/rails/pull/31727
Slide 77
Slide 77 text
X Best practices are undocumented
✔ Migrations don’t work
X Database tasks don’t exist
X Default connection was broken
The State of Multi-db in Rails 5
Slide 78
Slide 78 text
Migrations
Connections
don’t work
Rake tasks
Slide 79
Slide 79 text
Improving the database
tasks for multi-dbs
Slide 80
Slide 80 text
rails db:drop
rails db:create
rails db:migrate
Slide 81
Slide 81 text
>> ActiveRecord::Base.configurations["production"]
>> {
"adapter"=>"mysql2",
"database"=>"single_database_production"
}
Slide 82
Slide 82 text
>> ActiveRecord::Base.configurations["production"]
>> {
"primary"=>{"adapter"=>"mysql2",
"database"=>"multiple_databases_production"},
"animals"=>{"adapter"=>"mysql2",
“database"=>"multiple_databases_production"}
}
Slide 83
Slide 83 text
module ActiveRecord
module DatabaseConfigurations
class DatabaseConfig
attr_reader :env_name, :spec_name, :config
def initialize(env_name, spec_name, config)
@env_name = env_name
@spec_name = spec_name
@config = config
end
end
end
end
Slide 84
Slide 84 text
>> db_config = DatabaseConfig.new("production",
"animals", {"adapter"=>"mysql2",
"database"=>"multiple_databases_production_animals"
})
>>
#"mysql2",
"database"=>"multiple_databases_production_animals"
}>
Slide 85
Slide 85 text
>> db_config.env_name
"production"
Slide 86
Slide 86 text
>> db_config.env_name
"production"
>> db_config.spec_name
"animals"
Slide 87
Slide 87 text
>> db_config.env_name
"production"
>> db_config.spec_name
"animals"
>> db_config.config
{…"database"=>"multiple_databases_producti
on_animals"}
Slide 88
Slide 88 text
def self.db_configs(configs = configurations)
configs.each_pair.flat_map do |env_name, config|
walk_configs(env_name, "primary", config)
end
end
Slide 89
Slide 89 text
def self.configs_for(env, configs, &blk)
env_with_configs =
db_configs(configs).select do |db_config|
db_config.env_name == env
end
if block_given?
env_with_configs.each do |ewc|
yield ewc.spec_name, ewc.config
end
else
env_with_configs
end
end
Slide 90
Slide 90 text
>> DatabaseConfigurations.configs_for("production")
>>[#"mysql2",
"database"=>"multiple_databases_production"}>,
#"mysql2",
"database"=>"multiple_databases_production_animals"
}>]
Slide 91
Slide 91 text
task migrate: :load_config do
ActiveRecord::Tasks::DatabaseTasks.migrate
db_namespace["_dump"].invoke
end
Slide 92
Slide 92 text
task migrate: :load_config do
ActiveRecord::DatabaseConfigurations.configs_for(
Rails.env
) do |_, config|
ActiveRecord::Base.establish_connection(config)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
db_namespace["_dump"].invoke
end
Slide 93
Slide 93 text
task migrate: :load_config do
ActiveRecord::DatabaseConfigurations.configs_for(
Rails.env
) do |_, config|
ActiveRecord::Base.establish_connection(config)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
db_namespace["_dump"].invoke
end
Slide 94
Slide 94 text
task migrate: :load_config do
ActiveRecord::DatabaseConfigurations.configs_for(
Rails.env
) do |_, config|
ActiveRecord::Base.establish_connection(config)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
db_namespace["_dump"].invoke
end
Slide 95
Slide 95 text
task migrate: :load_config do
ActiveRecord::DatabaseConfigurations.configs_for(
Rails.env
) do |_, config|
ActiveRecord::Base.establish_connection(config)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
db_namespace["_dump"].invoke
end
Slide 96
Slide 96 text
rails db:drop
rails db:create
rails db:migrate
Slide 97
Slide 97 text
rails db:drop:primary
rails db:drop:animals
rails db:create:primary
rails db:create:animals
rails db:migrate:primary
rails db:migrate:animals
Slide 98
Slide 98 text
namespace :migrate
ActiveRecord::Tasks::DatabaseTasks.for_each do |spec_name|
desc "Migrate #{spec_name} database"
task spec_name => :load_config do
db_config = ActiveRecord::DatabaseConfigurations
.config_for_env_and_spec(Rails.env, spec_name)
ActiveRecord::Base.establish_connection(
db_config.config
)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
end
end
end
Slide 99
Slide 99 text
namespace :migrate
ActiveRecord::Tasks::DatabaseTasks.for_each do |spec_name|
desc "Migrate #{spec_name} database"
task spec_name => :load_config do
db_config = ActiveRecord::DatabaseConfigurations
.config_for_env_and_spec(Rails.env, spec_name)
ActiveRecord::Base.establish_connection(
db_config.config
)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
end
end
end
Slide 100
Slide 100 text
namespace :migrate
ActiveRecord::Tasks::DatabaseTasks.for_each do |spec_name|
desc "Migrate #{spec_name} database"
task spec_name => :load_config do
db_config = ActiveRecord::DatabaseConfigurations
.config_for_env_and_spec(Rails.env, spec_name)
ActiveRecord::Base.establish_connection(
db_config.config
)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
end
end
end
Slide 101
Slide 101 text
namespace :migrate
ActiveRecord::Tasks::DatabaseTasks.for_each do |spec_name|
desc "Migrate #{spec_name} database"
task spec_name => :load_config do
db_config = ActiveRecord::DatabaseConfigurations
.config_for_env_and_spec(Rails.env, spec_name)
ActiveRecord::Base.establish_connection(
db_config.config
)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
end
end
end
Slide 102
Slide 102 text
namespace :migrate
ActiveRecord::Tasks::DatabaseTasks.for_each do |spec_name|
desc "Migrate #{spec_name} database"
task spec_name => :load_config do
db_config = ActiveRecord::DatabaseConfigurations
.config_for_env_and_spec(Rails.env, spec_name)
ActiveRecord::Base.establish_connection(
db_config.config
)
ActiveRecord::Tasks::DatabaseTasks.migrate
end
end
end
end
Slide 103
Slide 103 text
rails db:drop:primary
rails db:drop:animals
rails db:create:primary
rails db:create:animals
rails db:migrate:primary
rails db:migrate:animals
Slide 104
Slide 104 text
Database task improvements
github.com/rails/rails/pull/32274
github.com/rails/rails/compare/
ab43b5405ea2...5ddcda6d5f56
Slide 105
Slide 105 text
Connections
don’t work
Migrations
Rake tasks
Slide 106
Slide 106 text
No content
Slide 107
Slide 107 text
“Are you concerned
that Rails will become
bloated?”
Slide 108
Slide 108 text
What would Rails look like
if we’d upstreamed these
features 5 years ago?
Slide 109
Slide 109 text
a +
Slide 110
Slide 110 text
' (
)
*
+
,
Slide 111
Slide 111 text
No content
Slide 112
Slide 112 text
Maybe Rails doesn’t
scale…?
Slide 113
Slide 113 text
Your application
isn’t special.
Slide 114
Slide 114 text
Forces you to write
generic code
Slide 115
Slide 115 text
Train your future
workforce
Slide 116
Slide 116 text
Give back to the
community
Slide 117
Slide 117 text
'
(
)
*
+
,
Slide 118
Slide 118 text
a♥
Slide 119
Slide 119 text
Scaling should
make us happy
Slide 120
Slide 120 text
Rails isn’t dying,
it’s maturing
Slide 121
Slide 121 text
No content
Slide 122
Slide 122 text
No content
Slide 123
Slide 123 text
No content
Slide 124
Slide 124 text
“Rails does scale!”
Slide 125
Slide 125 text
The Future of Rails 6
Scalable by Default
Slide 126
Slide 126 text
Eileen M. Uchitelle
eileencodes.com
@eileencodes