Slide 1
Slide 1 text
GenStage
& Flow
Poznań Elixir Meetup
25.05.2017
Slide 2
Slide 2 text
No content
Slide 3
Slide 3 text
Hi!
Tymon Tobolski
• GitHub: teamon
• Twitter: @iteamon
• Blog: teamon.eu
• 8+ years with Ruby, 2+ years with Elixir
• Currently Elixir+Dev+Ops @ Recruitee.com
• Hex: tesla, mix_docker
Slide 4
Slide 4 text
No content
Slide 5
Slide 5 text
The Job
Slide 6
Slide 6 text
Library
defmodule Poz do
def select do
items = Enum.to_list (1..100)
:timer.sleep(:rand.uniform(100))
Progress.incr(:select, length(items))
items
end
def download(record) do
:timer.sleep(:rand.uniform(100))
Progress.incr(:download)
{:file, record}
end
Slide 7
Slide 7 text
Library cont.
def extract(file) do
:timer.sleep(:rand.uniform(10))
Progress.incr(:extract)
{:text, file}
end
def index(texts) do
:timer.sleep(:rand.uniform(1000))
Progress.incr(:index, length(texts))
:ok
end
end
Slide 8
Slide 8 text
Example blueprint
defmodule Example do
import Poz
def run do
Progress.start_link([:select, :download, :extract, :index])
# work work work
Progress.stop()
end
end
Slide 9
Slide 9 text
Example 1 - Enum
select()
|> Enum.map(&download/1)
|> Enum.map(&extract/1)
|> index()
Slide 10
Slide 10 text
Example 1 - Enum
Slide 11
Slide 11 text
Example 2 - Stream
select()
|> Stream.concat([])
|> Stream.map(&download/1)
|> Stream.map(&extract/1)
|> Enum.to_list()
|> index()
Slide 12
Slide 12 text
Example 2 - Stream
Slide 13
Slide 13 text
Example 3 - Task.async
select()
|> Enum.map(fn e -> Task.async(Poz, :download, [e]) end)
|> Enum.map(&Task.await/1)
|> Enum.map(fn e -> Task.async(Poz, :extract, [e]) end)
|> Enum.map(&Task.await/1)
|> index()
Slide 14
Slide 14 text
Example 3 - Task.async
Slide 15
Slide 15 text
Stages
Slide 16
Slide 16 text
GenStage explained
Slide 17
Slide 17 text
Example 4 - GenStage - SELECT
defmodule Select do
use GenStage
def init(_), do: {:producer, :ok}
def handle_demand(demand, :ok) do
items = Poz.select()
{:noreply, items, :empty}
end
def handle_demand(_demand, :empty) do
{:noreply, [], :empty}
end
end
Slide 18
Slide 18 text
Example 4 - GenStage - DOWNLOAD
defmodule Download do
use GenStage
def init(_), do: {:producer_consumer, :ok}
def handle_events(items, _from, state) do
files = Enum.map(items, &Poz.download/1)
{:noreply, files, state}
end
end
Slide 19
Slide 19 text
Example 4 - GenStage - EXTRACT
defmodule Extract do
use GenStage
def init(_), do: {:producer_consumer, :ok}
def handle_events(files, _from, state) do
texts = Enum.map(files, &Poz.extract/1)
{:noreply, files, texts}
end
end
Slide 20
Slide 20 text
Example 4 - GenStage - INDEX
defmodule Index do
use GenStage
def init(_), do: {:consumer, :ok}
def handle_events(texts, _from, state) do
Poz.index(texts)
{:noreply, [], state}
end
end
Slide 21
Slide 21 text
Example 4 - GenStage
{:ok, select} = GenStage.start_link(Select, :ok)
{:ok, download} = GenStage.start_link(Download, :ok)
{:ok, extract} = GenStage.start_link(Extract, :ok)
{:ok, index} = GenStage.start_link(Index, :ok)
GenStage.sync_subscribe(download, to: select)
GenStage.sync_subscribe(extract, to: download)
GenStage.sync_subscribe(index, to: extract)
:timer.sleep(:infinity)
Slide 22
Slide 22 text
Example 4 - GenStage (defaults)
Slide 23
Slide 23 text
Example 5 - GenStage (custom demand)
{:ok, select} = GenStage.start_link(Select, :ok)
{:ok, download} = GenStage.start_link(Download, :ok)
{:ok, extract} = GenStage.start_link(Extract, :ok)
{:ok, index} = GenStage.start_link(Index, :ok)
GenStage.sync_subscribe(download, to: select, max_demand: 20)
GenStage.sync_subscribe(extract, to: download, max_demand: 20)
GenStage.sync_subscribe(index, to: extract, max_demand: 100)
:timer.sleep(:infinity)
Slide 24
Slide 24 text
Example 5 - GenStage (tuned demand)
Slide 25
Slide 25 text
Example 6 - GenStage (multiprocess)
{:ok, select} = GenStage.start_link(Select, :ok)
{:ok, extract} = GenStage.start_link(Extract, :ok)
{:ok, index} = GenStage.start_link(Index, :ok)
for i <- (1..10) do
{:ok, download} = GenStage.start_link(Download, i)
GenStage.sync_subscribe(download, to: select, max_demand: 20)
GenStage.sync_subscribe(extract, to: download, max_demand: 20)
end
GenStage.sync_subscribe(index, to: extract, max_demand: 100)
Slide 26
Slide 26 text
No content
Slide 27
Slide 27 text
Example 7 - Flow
select()
|> Flow.from_enumerable(max_demand: 20)
|> Flow.partition(max_demand: 20, stages: 5)
|> Flow.map(&download/1)
|> Flow.partition(max_demand: 20, stages: 2)
|> Flow.map(&extract/1)
|> Flow.partition(window: Flow.Window.count(50), stages: 1)
|> Flow.reduce(fn -> [] end, fn item, list -> [item | list] end)
|> Flow.emit(:state)
|> Flow.partition(stages: 2)
|> Flow.map(&index/1)
|> Flow.run()
Slide 28
Slide 28 text
Example 7 - Flow
Slide 29
Slide 29 text
Real World
Slide 30
Slide 30 text
Real World - optimised
Slide 31
Slide 31 text
References
• https:/
/github.com/teamon/poz
• https:/
/elixir-lang.org/blog/2016/07/14/announcing-genstage/
• https:/
/hexdocs.pm/gen_stage/GenStage.html
• https:/
/hexdocs.pm/flow/Flow.html
• http:/
/teamon.eu/2016/tuning-elixir-genstage-flow-pipeline-
processing/
• http:/
/teamon.eu/2016/measuring-visualizing-genstage-flow-
with-gnuplot/
Slide 32
Slide 32 text
Thanks!
Questions?