Procesando Millones de Imágenes: Ruby vs. Elixir

David Padilla (@dabit) nos habla de la experiencia que tuvo al intentar procesar más de 2 millones de imágenes usando Ruby, y cómo Elixir le ahorró muchísimo tiempo.

Transcript

1. Procesando millones de imágenes David Padilla @dabit
2. Carrierwave
6. image.recreate_versions!
7. Image.each do |i| i.file.recreate_versions! end
8. 2,700,000
9. 1 segundo
10. 2,700,000 ÷ 60 = 45,000 minutos
11. 45,000 ÷ 60 = 750 horas
12. 750 ÷ 24 = 31.25 días
13. MAX_WORKERS = 20 Image.find_each(batch_size: 1200) do |batch| queue = Queue.new batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do |w| Thread.new do while image = queue.pop(true) image.file.recreate_versions! end end end workers.map(&:join) end
14. MAX_WORKERS = 20 Image.find_each(batch_size: 1200) do |batch| queue = Queue.new batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do |w| Thread.new do while image = queue.pop(true) image.file.recreate_versions! end end end workers.map(&:join) end
15. MAX_WORKERS = 20 Image.find_each(batch_size: 1200) do |batch| queue = Queue.new batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do |w| Thread.new do while image = queue.pop(true) image.file.recreate_versions! end end end workers.map(&:join) end
16. MAX_WORKERS = 20 Image.find_each(batch_size: 1200) do |batch| queue = Queue.new batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do |w| Thread.new do while image = queue.pop(true) image.file.recreate_versions! end end end workers.map(&:join) end
17. MAX_WORKERS = 20 Image.find_each(batch_size: 1200) do |batch| queue = Queue.new batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do |w| Thread.new do while image = queue.pop(true) image.file.recreate_versions! end end end workers.map(&:join) end
19. array = [] 5.times.map do Thread.new do 1000.times do array << nil end end end.each(&:join) puts array.size
20. $ ruby pushing_nil.rb 5000 $ jruby pushing_nil.rb 4446 $ rbx pushing_nil.rb 3088
21. array = [] 5.times.map do Thread.new do 1000.times do array << nil end end end.each(&:join) puts array.size
22. 1 segundo
23. 0.6 segundos
24. 18 días
25. Elixir
26. class Code def method(*args) code = code + code while(true) do something_awesome end end def something_awesome add = 1 + 2 o = Object.new o.save_to_database end end
27. Exception
33. Traer registros de la Base de Datos Bajar de S3 imagen original Crear dos tamaños Subir a S3 Crear aplicación
34. Crear aplicación
35. $ mix new images
37. Traer registros de la Base de Datos
38. use Mix.Config config :images, Images.Repo, adapter: Ecto.Adapters.MySQL, database: "images_database", username: "root", password: "", hostname: "localhost"
39. defmodule Images.PropertyImage do use Ecto.Model schema "images" do field :file, :string end end
40. defmodule Images.Image do use Ecto.Model def main_query do from i in Images.Image, select: i end def find(image_id) do from i in main_query where: id = ^image_id end def paged(offset, limit) do from i in main_query, limit: ^limit, offset: ^offset end def all do Images.Repo.all main_query end end
41. Bajar de S3 la imagen original
42. def download_original(filename, id) do file = temp_filename(filename, id) ibrowse = [save_response_to_file: String.to_char_list(file)] s3_url(filename, id) |> HTTPotion.get([ibrowse: ibrowse]) file end def s3_url(file, id) do "#{s3_path}/#{id}/#{file}" end
43. def download_original(filename, id) do file = temp_filename(filename, id) ibrowse = [save_response_to_file: String.to_char_list(file)] s3_url(filename, id) |> HTTPotion.get([ibrowse: ibrowse]) file end def s3_url(file, id) do "#{s3_path}/#{id}/#{file}" end
44. def download_original(filename, id) do file = temp_filename(filename, id) ibrowse = [save_response_to_file: String.to_char_list(file)] HTTPotion.get(s3_url(filename, id), [ibrowse: ibrowse]) file end def s3_url(file, id) do "#{s3_path}/#{id}/#{file}" end
45. def download_original(filename, id) do file = temp_filename(filename, id) ibrowse = [save_response_to_file: String.to_char_list(file)] s3_url(filename, id) |> HTTPotion.get([ibrowse: ibrowse]) file end def s3_url(file, id) do "#{s3_path}/#{id}/#{file}" end
46. Crear dos tamaños
48. def generate_medium(file, filename, id) do result = Path.join(System.tmp_dir, size_name(filename, :medium)) Mogrify.open(file) |> Mogrify.copy |> Mogrify.resize_to_fill("450x300") |> Mogrify.save(result) end
50. Subir a S3
52. {status, _} = System.cmd("s3cmd", ["-P", "put", filename, s3_name])
56. {_, file} = File.read(result) Application.get_env(:images, :s3_bucket) |> String.to_char_list |> :erlcloud_s3.put_object(s3_name, file, [], [{#…}])
57. y la concurrencia?
58. Traer registros de la Base de Datos Bajar de S3 imagen original Crear dos tamaños Subir a S3
59. :poolboy
61. defmodule Images.PropertyImageWorker do use GenServer def start_link([]) do :gen_server.start_link(__MODULE__, [], []) end def init(state) do secret = to_char_list(Application.get_env(:images, :aws_secret_key)) Application.get_env(:images, :aws_access_key) |> to_char_list |> :erlcloud_s3.configure(secret) {:ok, state} end def handle_call(image, from, state) do result = Images.PropertyImage.process(image) {:reply, [result], state} end end
62. defmodule Images.PropertyImageWorker do use GenServer def start_link([]) do :gen_server.start_link(__MODULE__, [], []) end def init(state) do secret = to_char_list(Application.get_env(:images, :aws_secret_key)) Application.get_env(:images, :aws_access_key) |> to_char_list |> :erlcloud_s3.configure(secret) {:ok, state} end def handle_call(image, from, state) do result = Images.PropertyImage.process(image) {:reply, [result], state} end end
63. defmodule Images.ImagesSupervisor do use Supervisor def start_link do :supervisor.start_link(__MODULE__, []) end def init([]) do poolboy_config = [ {:name, {:local, pool_name()}}, {:worker_module, Images.PropertyImageWorker}, {:size, 20}, {:max_overflow, 0} ] children = [ :poolboy.child_spec(pool_name(), poolboy_config, []), worker(Images.Repo, []) ] supervise(children, strategy: :one_for_one) end def pool_name do :property_images end end
64. defmodule Images.ImagesSupervisor do use Supervisor def start_link do :supervisor.start_link(__MODULE__, []) end def init([]) do poolboy_config = [ {:name, {:local, pool_name()}}, {:worker_module, Images.PropertyImageWorker}, {:size, 20}, {:max_overflow, 0} ] children = [ :poolboy.child_spec(pool_name(), poolboy_config, []), worker(Images.Repo, []) ] supervise(children, strategy: :one_for_one) end def pool_name do :property_images end end
65. defmodule Images do def start(_type, _args) do supervisor = Images.ImagesSupervisor.start_link enqueue supervisor end def enqueue do Images.PropertyImage.all |> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end end def pool_image(image) do :poolboy.transaction( Images.ImagesSupervisor.pool_name, fn(pid) -> :gen_server.call(pid, image) end, :infinity ) end end
66. defmodule Images do def start(_type, _args) do supervisor = Images.ImagesSupervisor.start_link enqueue supervisor end def enqueue do Images.PropertyImage.all |> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end end def pool_image(image) do :poolboy.transaction( Images.ImagesSupervisor.pool_name, fn(pid) -> :gen_server.call(pid, image) end, :infinity ) end end
67. defmodule Images do def start(_type, _args) do supervisor = Images.ImagesSupervisor.start_link enqueue supervisor end def enqueue do Images.PropertyImage.all |> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end end def pool_image(image) do :poolboy.transaction( Images.ImagesSupervisor.pool_name, fn(pid) -> :gen_server.call(pid, image) end, :infinity ) end end
68. defmodule Images do def start(_type, _args) do supervisor = Images.ImagesSupervisor.start_link enqueue supervisor end def enqueue do Images.PropertyImage.all |> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end end def pool_image(image) do :poolboy.transaction( Images.ImagesSupervisor.pool_name, fn(pid) -> :gen_server.call(pid, image) end, :infinity ) end end
70. BEAM
71. Conclusión
72. 4 días
73. 4 x 24 = 96 96 x 60 = 5,760 5,760 x 60 = 345,600 345,600 / 2,700,000 = 0.128s
74. Conclusión II
75. Fin