Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Procesando millones de imágenes con Elixir

Procesando millones de imágenes con Elixir

David Padilla

November 28, 2015
Tweet

More Decks by David Padilla

Other Decks in Technology

Transcript

  1. MAX_WORKERS = 20 Image.find_each(batch_size: 1200) do |batch| queue = Queue.new

    batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do |w| Thread.new do while image = queue.pop(true) image.file.recreate_versions! end end end workers.map(&:join) end
  2. MAX_WORKERS = 20 Image.find_each(batch_size: 1200) do |batch| queue = Queue.new

    batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do |w| Thread.new do while image = queue.pop(true) image.file.recreate_versions! end end end workers.map(&:join) end
  3. MAX_WORKERS = 20 Image.find_each(batch_size: 1200) do |batch| queue = Queue.new

    batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do |w| Thread.new do while image = queue.pop(true) image.file.recreate_versions! end end end workers.map(&:join) end
  4. MAX_WORKERS = 20 Image.find_each(batch_size: 1200) do |batch| queue = Queue.new

    batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do |w| Thread.new do while image = queue.pop(true) image.file.recreate_versions! end end end workers.map(&:join) end
  5. MAX_WORKERS = 20 Image.find_each(batch_size: 1200) do |batch| queue = Queue.new

    batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do |w| Thread.new do while image = queue.pop(true) image.file.recreate_versions! end end end workers.map(&:join) end
  6. array = [] 5.times.map do Thread.new do 1000.times do array

    << nil end end end.each(&:join) puts array.size
  7. array = [] 5.times.map do Thread.new do 1000.times do array

    << nil end end end.each(&:join) puts array.size
  8. class Code def method(*args) code = code + code while(true)

    do something_awesome end end def something_awesome add = 1 + 2 o = Object.new o.save_to_database end end
  9. Traer registros de la Base de Datos Bajar de S3

    imagen original Crear dos tamaños Subir a S3 Crear aplicación
  10. defmodule Images.Image do use Ecto.Model def main_query do from i

    in Images.Image, select: i end def find(image_id) do from i in main_query where: id = ^image_id end def paged(offset, limit) do from i in main_query, limit: ^limit, offset: ^offset end def all do Images.Repo.all main_query end end
  11. def download_original(filename, id) do file = temp_filename(filename, id) ibrowse =

    [save_response_to_file: String.to_char_list(file)] s3_url(filename, id) |> HTTPotion.get([ibrowse: ibrowse]) file end def s3_url(file, id) do "#{s3_path}/#{id}/#{file}" end
  12. def download_original(filename, id) do file = temp_filename(filename, id) ibrowse =

    [save_response_to_file: String.to_char_list(file)] s3_url(filename, id) |> HTTPotion.get([ibrowse: ibrowse]) file end def s3_url(file, id) do "#{s3_path}/#{id}/#{file}" end
  13. def download_original(filename, id) do file = temp_filename(filename, id) ibrowse =

    [save_response_to_file: String.to_char_list(file)] HTTPotion.get(s3_url(filename, id), [ibrowse: ibrowse]) file end def s3_url(file, id) do "#{s3_path}/#{id}/#{file}" end
  14. def download_original(filename, id) do file = temp_filename(filename, id) ibrowse =

    [save_response_to_file: String.to_char_list(file)] s3_url(filename, id) |> HTTPotion.get([ibrowse: ibrowse]) file end def s3_url(file, id) do "#{s3_path}/#{id}/#{file}" end
  15. def generate_medium(file, filename, id) do result = Path.join(System.tmp_dir, size_name(filename, :medium))

    Mogrify.open(file) |> Mogrify.copy |> Mogrify.resize_to_fill("450x300") |> Mogrify.save(result) end
  16. Traer registros de la Base de Datos Bajar de S3

    imagen original Crear dos tamaños Subir a S3
  17. defmodule Images.PropertyImageWorker do use GenServer def start_link([]) do :gen_server.start_link(__MODULE__, [],

    []) end def init(state) do secret = to_char_list(Application.get_env(:images, :aws_secret_key)) Application.get_env(:images, :aws_access_key) |> to_char_list |> :erlcloud_s3.configure(secret) {:ok, state} end def handle_call(image, from, state) do result = Images.PropertyImage.process(image) {:reply, [result], state} end end
  18. defmodule Images.PropertyImageWorker do use GenServer def start_link([]) do :gen_server.start_link(__MODULE__, [],

    []) end def init(state) do secret = to_char_list(Application.get_env(:images, :aws_secret_key)) Application.get_env(:images, :aws_access_key) |> to_char_list |> :erlcloud_s3.configure(secret) {:ok, state} end def handle_call(image, from, state) do result = Images.PropertyImage.process(image) {:reply, [result], state} end end
  19. defmodule Images.ImagesSupervisor do use Supervisor def start_link do :supervisor.start_link(__MODULE__, [])

    end def init([]) do poolboy_config = [ {:name, {:local, pool_name()}}, {:worker_module, Images.PropertyImageWorker}, {:size, 20}, {:max_overflow, 0} ] children = [ :poolboy.child_spec(pool_name(), poolboy_config, []), worker(Images.Repo, []) ] supervise(children, strategy: :one_for_one) end def pool_name do :property_images end end
  20. defmodule Images.ImagesSupervisor do use Supervisor def start_link do :supervisor.start_link(__MODULE__, [])

    end def init([]) do poolboy_config = [ {:name, {:local, pool_name()}}, {:worker_module, Images.PropertyImageWorker}, {:size, 20}, {:max_overflow, 0} ] children = [ :poolboy.child_spec(pool_name(), poolboy_config, []), worker(Images.Repo, []) ] supervise(children, strategy: :one_for_one) end def pool_name do :property_images end end
  21. defmodule Images do def start(_type, _args) do supervisor = Images.ImagesSupervisor.start_link

    enqueue supervisor end def enqueue do Images.PropertyImage.all |> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end end def pool_image(image) do :poolboy.transaction( Images.ImagesSupervisor.pool_name, fn(pid) -> :gen_server.call(pid, image) end, :infinity ) end end
  22. defmodule Images do def start(_type, _args) do supervisor = Images.ImagesSupervisor.start_link

    enqueue supervisor end def enqueue do Images.PropertyImage.all |> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end end def pool_image(image) do :poolboy.transaction( Images.ImagesSupervisor.pool_name, fn(pid) -> :gen_server.call(pid, image) end, :infinity ) end end
  23. defmodule Images do def start(_type, _args) do supervisor = Images.ImagesSupervisor.start_link

    enqueue supervisor end def enqueue do Images.PropertyImage.all |> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end end def pool_image(image) do :poolboy.transaction( Images.ImagesSupervisor.pool_name, fn(pid) -> :gen_server.call(pid, image) end, :infinity ) end end
  24. defmodule Images do def start(_type, _args) do supervisor = Images.ImagesSupervisor.start_link

    enqueue supervisor end def enqueue do Images.PropertyImage.all |> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end end def pool_image(image) do :poolboy.transaction( Images.ImagesSupervisor.pool_name, fn(pid) -> :gen_server.call(pid, image) end, :infinity ) end end
  25. 4 x 24 = 96 96 x 60 = 5,760

    5,760 x 60 = 345,600 345,600 / 2,700,000 = 0.128s
  26. Fin