Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Building a transactional pipeline with TDD and Elixir

Building a transactional pipeline with TDD and Elixir

We are used to work with database level transactions. But, what happens when the transactional limits go beyond the database? How do we handle a complex transaction that affects different systems? How would you solve this problem using a functional language?

In this talk we will describe a real problem that we faced at DNSimple. We will discuss the challenges that we faced and how we used TDD and Elixir to solve it.

Javier Acero

June 03, 2017
Tweet

More Decks by Javier Acero

Other Decks in Technology

Transcript

  1. Transactional
    Pipeline
    function()
    Building a
    with TDD
    and Elixir
    #scpna17
    #teamtinyp

    View Slide

  2. jacegu jacegu
    javieracero.com

    View Slide

  3. https://dnsimple.com

    View Slide

  4. d s
    n
    http://howdns.works

    View Slide

  5. pamplonaswcraft.com
    192.30.252.153
    192.30.252.154

    View Slide

  6. pamplonaswcraft.com

    View Slide

  7. https://help.github.com/articles/using-a-custom-domain-with-github-pages

    View Slide

  8. https://help.github.com/articles/using-a-custom-domain-with-github-pages

    View Slide

  9. View Slide

  10. View Slide

  11. domains
    services
    Connect
    to

    View Slide

  12. api

    View Slide

  13. api

    View Slide

  14. connectors
    http://platform.dnsimple.com

    View Slide

  15. View Slide

  16. transactional

    View Slide

  17. View Slide

  18. if elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif else end

    View Slide

  19. connection
    |> add_alias_record(…)
    |> add_cname_record(…)
    |> configure_cname_file(…)
    |> save_connection(…)

    View Slide

  20. WHERE ARE MY OBJECTS
    WHERE ARE MY OBJECTS

    View Slide

  21. View Slide

  22. Runs anonymous functions without params
    1

    View Slide

  23. Runs anonymous functions without params
    defmodule TpTest do
    use ExUnit.Case
    test "runs anonymous functions without params" do
    assert Tp.run(fn -> 2 + 2 end) == 4
    end
    end
    1
    test/tp_test.ex Test [1/1]

    View Slide

  24. test "runs anonymous functions without params" do
    assert Tp.run(fn -> 2 + 2 end) == 4
    end
    Runs anonymous functions without params
    1
    test/tp_test.ex Test [1/1]

    View Slide

  25. test "runs anonymous functions without params" do
    assert Tp.run(fn -> 2 + 2 end) == 4
    end
    Runs anonymous functions without params
    1
    test/tp_test.ex Test [1/1]

    View Slide

  26. Runs anonymous functions without params
    defmodule Tp do
    def run(_) do
    nil
    end
    end
    1
    lib/tp.ex

    View Slide

  27. Runs anonymous functions without params
    def run(_) do
    nil
    end
    1
    lib/tp.ex

    View Slide

  28. Runs anonymous functions without params
    def run(_) do
    4
    end
    1
    lib/tp.ex

    View Slide

  29. Runs anonymous functions without params
    test "runs an anonymous function without params" do
    assert Tp.run(fn -> 2 + 2 end) == 4
    end
    1
    test/tp_test.ex Test [1/1]

    View Slide

  30. Runs anonymous functions without params
    test "runs an anonymous function without params" do
    assert Tp.run(fn -> 2 + 2 end) == 4
    assert Tp.run(fn -> String.upcase("x") end) == "X"
    end
    1
    test/tp_test.ex Test [1/1]

    View Slide

  31. Runs anonymous functions without params
    def run(_) do
    4
    end
    1
    lib/tp.ex

    View Slide

  32. Runs anonymous functions without params
    def run(fun) do
    fun.()
    end
    1
    lib/tp.ex

    View Slide

  33. Runs anonymous functions with 1 parameter
    2

    View Slide

  34. test "runs anonymous functions with 1 param" do
    assert Tp.run(fn(x) -> 2 + x end, 2) == 4
    end
    Runs anonymous functions with 1 parameter
    2
    test/tp_test.ex Test [2/2]

    View Slide

  35. Runs anonymous functions with 1 parameter
    def run(fun) do
    fun.()
    end
    2
    lib/tp.ex

    View Slide

  36. Runs anonymous functions with 1 parameter
    def run(fun) do
    fun.()
    end
    def run(fun, param), do: fun.(param)
    2
    lib/tp.ex

    View Slide

  37. Runs anonymous functions with 1 parameter
    def run(fun), do: fun.()
    def run(fun, param), do: fun.(param)
    2
    lib/tp.ex

    View Slide

  38. Runs anonymous functions with 2 parameters
    3

    View Slide

  39. Runs anonymous functions with 2 parameters
    test "runs an anonymous function with 2 params" do
    assert Tp.run(fn(x, y) -> x+y end, [1, 2]) == 3
    end
    3
    test/tp_test.ex Test [3/3]

    View Slide

  40. Runs anonymous functions with 2 parameters
    def run(fun), do: fun.()
    def run(fun, param), do: fun.(param)
    3
    lib/tp.ex

    View Slide

  41. Runs anonymous functions with 2 parameters
    def run(fun), do: fun.()
    def run(fun, params) when is_list(params),
    do: apply(fun, params)
    def run(fun, param), do: fun.(param)
    3
    lib/tp.ex

    View Slide

  42. Runs anonymous functions with 2 parameters
    test "runs an anonymous function with 2 params" do
    assert Tp.run(fn(x, y) -> x+y end, [1, 2]) == 3
    end
    3
    test/tp_test.ex Test [3/3]

    View Slide

  43. test "runs an anonymous function with 2 params" do
    assert Tp.run(fn -> 2 + 2 end, []) == 4
    assert Tp.run(&String.upcase/1, ["x"]) == "X"
    assert Tp.run(fn(x, y) -> x+y end, [1, 2]) == 3
    end
    3
    test/tp_test.ex Test [3/3]
    Runs anonymous functions with 2 parameters

    View Slide

  44. test "runs a function" do
    assert Tp.run(fn -> 2 + 2 end, []) == 4
    assert Tp.run(&String.upcase/1, ["x"]) == "X"
    assert Tp.run(fn(x, y) -> x+y end, [1, 2]) == 3
    end
    3
    test/tp_test.ex Test [3/3]
    Runs a function
    Test [1/1]

    View Slide

  45. Runs a function
    def run(fun), do: fun.()
    def run(fun, params) when is_list(params),
    do: apply(fun, params)
    def run(fun, param), do: fun.(param)
    3
    lib/tp.ex

    View Slide

  46. Runs a function
    def run(fun, params), do: apply(fun, params)
    3
    lib/tp.ex

    View Slide

  47. Runs multiple functions
    4

    View Slide

  48. Runs multiple functions
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> [x*x, y*y, z*z] end
    f2 = fn(x, y, z) -> [z, y, x] end
    assert Tp.run([f1, f2], [2, 3, 5]) == [25, 9, 4]
    end
    4
    test/tp_test.ex Test [2/2]

    View Slide

  49. Runs multiple functions
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex

    View Slide

  50. Runs multiple functions
    def run([fun1, fun2], params) do
    apply(fun2, apply(fun1, params))
    end
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex

    View Slide

  51. Runs multiple functions
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> [x*x, y*y, z*z] end
    f2 = fn(x, y, z) -> [z, y, x] end
    assert Tp.run([f1, f2], [2, 3, 5]) == [25, 9, 4]
    end
    4
    test/tp_test.ex Test [2/2]

    View Slide

  52. Runs multiple functions
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> [x*x, y*y, z*z] end
    f2 = fn(x, y, z) -> [z, y, x] end
    assert Tp.run([], [1, 2, 3]) == [1, 2, 3]
    assert Tp.run([f2], [7, 8, 9]) == [9, 8, 7]
    assert Tp.run([f1, f2], [2, 3, 5]) == [25, 9, 4]
    end
    4
    test/tp_test.ex Test [2/2]

    View Slide

  53. Runs multiple functions
    def run([fun1, fun2], params) do
    apply(fun2, apply(fun1, params))
    end
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex

    View Slide

  54. Runs multiple functions
    def run([first|rest], params) do
    apply(fun2, apply(fun1, params))
    end
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex

    View Slide

  55. Runs multiple functions
    def run([first|rest], params) do
    run(rest, apply(first, params))
    end
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex

    View Slide

  56. Runs multiple functions
    def run([], params), do: params
    def run([first|rest], params) do
    run(rest, apply(first, params))
    end
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex

    View Slide

  57. Runs multiple functions
    4
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params), do: run(t, apply(h, params))
    def run(fun, params), do: apply(fun, params)

    View Slide

  58. Runs multiple functions
    4
    test/tp_test.ex Test [1/1]
    test "runs a function" do
    assert Tp.run(fn -> 2 + 2 end, []) == 4
    assert Tp.run(&String.upcase/1, ["x"]) == "X"
    assert Tp.run(fn(x, y) -> x+y end, [1, 2]) == 3
    end

    View Slide

  59. Runs multiple functions
    test "runs a function" do
    assert Tp.run([fn -> 2 + 2 end], []) == 4
    assert Tp.run([&String.upcase/1], ["x"]) == "X"
    assert Tp.run([fn(x, y) -> x+y end], [1, 2]) == 3
    end
    4
    test/tp_test.ex Test [1/1]

    View Slide

  60. Runs multiple functions
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> [x*x, y*y, z*z] end
    f2 = fn(x, y, z) -> [z, y, x] end
    assert Tp.run([], [1, 2, 3]) == [1, 2, 3]
    assert Tp.run([f2], [7, 8, 9]) == [9, 8, 7]
    assert Tp.run([f1, f2], [2, 3, 5]) == [25, 9, 4]
    end
    4
    test/tp_test.ex Test [2/2]
    Test [1/1]

    View Slide

  61. Runs multiple functions
    4
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params), do: run(t, apply(h, params))
    def run(fun, params), do: apply(fun, params)

    View Slide

  62. Runs multiple functions
    4
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params), do: run(t, apply(h, params))

    View Slide

  63. View Slide

  64. View Slide

  65. View Slide

  66. Runs functions with side effects
    Runs “real world” functions
    5

    View Slide

  67. Runs “real world” functions
    defmodule State do
    def start(initial \\ nil), do: #...
    def stop, do: #...
    def get, do: #...
    def put(new), do: #...
    end
    5
    test/test_helper.ex

    View Slide

  68. Runs “real world” functions
    5
    test/tp_test.ex
    test "runs functions with side effects" do
    State.start
    f1 = fn(x) -> State.put(x); [x] end
    f2 = fn(x) -> [x*x*x] end
    assert Tp.run([f1, f2], [3]) == [27]
    assert State.get == 3
    State.stop
    end
    Test [2/2]

    View Slide

  69. Returns whether it failed
    6

    View Slide

  70. ERR R
    HANDLING

    View Slide

  71. http://elixir-lang.org/getting-started/try-catch-and-rescue.html

    View Slide

  72. http://elixir-lang.org/getting-started/try-catch-and-rescue.html

    View Slide

  73. try do
    result = this_can_fail!(params)
    continue(result)
    rescue
    error -> handle_error(error)
    end
    case this_can_fail(params) do
    {:ok, result} -> continue(result)
    {:error, error} -> handle_error(params)
    end
    vs

    View Slide

  74. Returns whether it failed
    6

    View Slide

  75. Returns whether it failed
    6
    test "returns whether it failed" do
    State.start
    f1 = fn(x) -> State.put(x); [x] end
    f2 = fn(x) -> [x*x*x] end
    f3 = fn(_) -> {:error, “KO"} end
    assert Tp.run([f1, f2], [3]) == [27]
    assert Tp.run([f3, f2], [3]) == {:error, “KO"}
    State.stop
    end
    test/tp_test.ex Test [3/3]

    View Slide

  76. Returns whether it failed
    6
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params), do: run(t, apply(h, params))

    View Slide

  77. Returns whether it failed
    6
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params) do
    run(t, apply(h, params))
    end

    View Slide

  78. Returns whether it failed
    6
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params) do
    case apply(h, params) do
    run(t,)
    end
    end

    View Slide

  79. Returns whether it failed
    6
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params) do
    case apply(h, params) do
    {:error, details} -> {:error, details}
    run(t,)
    end
    end

    View Slide

  80. Returns whether it failed
    6
    lib/tp.ex
    def run([], params), do: params
    def run([h | t], params) do
    case apply(h, params) do
    {:error, details} -> {:error, details}
    new_params -> run(t, new_params)
    end
    end

    View Slide

  81. Returns whether it succeeded or failed
    7

    View Slide

  82. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [3/3]
    test "returns whether it failed" do
    State.start
    f1 = fn(x) -> State.put(x); [x] end
    f2 = fn(x) -> [x*x*x] end
    f3 = fn(_) -> {:error, "KO"} end
    assert Tp.run([f1, f2], [3]) == [27]
    assert Tp.run([f3, f2], [3]) == {:error, "KO"}
    State.stop
    end

    View Slide

  83. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [3/3]
    test "returns whether it succeeded or failed" do
    State.start
    f1 = fn(x) -> State.put(x); [x] end
    f2 = fn(x) -> [x*x*x] end
    f3 = fn(_) -> {:error, "KO"} end
    assert Tp.run([f1, f2], [3]) == {:ok, [27]}
    assert Tp.run([f3, f2], [3]) == {:error, "KO"}
    State.stop
    end

    View Slide

  84. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [3/3]
    test "returns whether it succeeded or failed" do
    State.start
    f1 = fn(x) -> State.put(x); {:ok, [x]} end
    f2 = fn(x) -> {:ok, [x*x*x]} end
    f3 = fn(_) -> {:error, "KO"} end
    assert Tp.run([f1, f2], [3]) == {:ok, [27]}
    assert Tp.run([f3, f2], [3]) == {:error, "KO"}
    State.stop
    end

    View Slide

  85. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [2/3]
    test "runs functions with side effects" do
    State.start
    f1 = fn(x) -> State.put(x); [x] end
    f2 = fn(x) -> [x*x*x] end
    assert Tp.run([f1, f2], [3]) == [27]
    assert State.get == 3
    State.stop
    end

    View Slide

  86. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [2/3]
    test "runs functions with side effects" do
    State.start
    f1 = fn(x) -> State.put(x); {:ok, [x]} end
    f2 = fn(x) -> {:ok, [x*x*x]} end
    assert Tp.run([f1, f2], [3]) == {:ok, [27]}
    assert State.get == 3
    State.stop
    end

    View Slide

  87. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [1/3]
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> [x*x, y*y, z*z] end
    f2 = fn(x, y, z) -> [z, y, x] end
    assert Tp.run([], [1, 2, 3]) == [1, 2, 3]
    assert Tp.run([f2], [7, 8, 9]) == [9, 8, 7]
    assert Tp.run([f1, f2], [2, 3, 5]) == [25, 9, 4]
    end

    View Slide

  88. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [1/3]
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> {:ok, [x*x, y*y, z*z]} end
    f2 = fn(x, y, z) -> {:ok, [z, y, x]} end
    assert Tp.run([], [1, 2, 3]) == {:ok, [1, 2, 3]}
    assert Tp.run([f2], [7, 8, 9]) ==
    {:ok, [9, 8, 7]}
    assert Tp.run([f1, f2], [2, 3, 5]) ==
    {:ok, [25, 9, 4]}
    end

    View Slide

  89. Returns whether it succeeded or failed
    7
    lib/tp.ex
    def run([], params), do: params
    def run([h | t], params) do
    case apply(h, params) do
    {:error, details} -> {:error, details}
    new_params -> run(t, new_params)
    end
    end

    View Slide

  90. Returns whether it succeeded or failed
    7
    lib/tp.ex
    def run([], params), do: params
    def run([h | t], params) do
    case apply(h, params) do
    {:error, details} -> {:error, details}
    {:ok, new_params} -> run(t, new_params)
    end
    end

    View Slide

  91. Returns whether it succeeded or failed
    7
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:error, details} -> {:error, details}
    {:ok, new_params} -> run(t, new_params)
    end
    end

    View Slide

  92. Returns whether it succeeded or failed
    7
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:ok, new_params} -> run(t, new_params)
    {:error, details} -> {:error, details}
    end
    end

    View Slide

  93. Rolls back when it fails
    8

    View Slide

  94. Rolls back when it fails
    8
    test "rolls back when it fails" do
    State.start("initial")
    f1 = fn(x) -> State.put(x); {:ok, [x]} end
    f2 = fn(_) -> {:error, "BOOM!"} end
    Tp.run([f1, f2], [0])
    assert State.get == "initial"
    State.stop
    end
    test/tp_test.ex Test [4/4]

    View Slide

  95. Rolls back when it fails
    8
    test "rolls back when it fails" do
    State.start("initial")
    f1 = fn(x) ->
    state = State.get
    State.put(x)
    {:ok, [x], fn() -> State.put(state) end}
    end
    f2 = fn(_) -> {:error, "BOOM!"} end
    Tp.run([f1, f2], [0])
    assert State.get == "initial"
    State.stop
    end
    test/tp_test.ex Test [4/4]

    View Slide

  96. Rolls back when it fails
    8
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:ok, new_params} -> run(t, new_params)
    {:error, details} -> {:error, details}
    end
    end

    View Slide

  97. Rolls back when it fails
    8
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:ok, new_params} ->
    run(t, new_params)
    {:error, details} ->
    {:error, details}
    end
    end

    View Slide

  98. Rolls back when it fails
    8
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:ok, new_params} ->
    run(t, new_params)
    {:error, details} ->
    rollback.()
    {:error, details}
    end
    end

    View Slide

  99. Rolls back when it fails
    8
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:ok, new_params} ->
    run(t, new_params)
    {:ok, new_params, new_rollback} ->
    run(t, new_params)
    {:error, details} ->
    rollback.()
    {:error, details}
    end
    end

    View Slide

  100. Rolls back when it fails
    8
    lib/tp.ex
    def run([], params, _), do: {:ok, params}
    def run([h | t], params, rollback) do
    case apply(h, params) do
    {:ok, new_params} ->
    run(t, new_params, rollback)
    {:ok, new_params, new_rollback} ->
    run(t, new_params, new_rollback)
    {:error, details} ->
    rollback.()
    {:error, details}
    end
    end

    View Slide

  101. Rolls back when it fails
    8
    lib/tp.ex
    defp do_run([], params, _), do: {:ok, params}
    defp do_run([h | t], params, rollback) do
    case apply(h, params) do
    {:ok, new_params} ->
    run(t, new_params, rollback)
    {:ok, new_params, new_rollback} ->
    run(t, new_params, new_rollback)
    {:error, details} ->
    rollback.()
    {:error, details}
    end
    end

    View Slide

  102. Rolls back when it fails
    8
    lib/tp.ex
    def run(functions, params) do
    default_rollback = fn -> nil end
    do_run(functions, params, default_rollback)
    end

    View Slide

  103. Rolls back when it fails
    9

    View Slide

  104. Rolls back when it fails
    9
    test/tp_test.ex Test [4/4]
    test "rolls back when it fails" do
    State.start("initial")
    f1 = fn(x) ->
    state = State.get
    {State.put(x), [x], fn() -> State.put(state) end}
    end
    f2 = fn(_) -> {:error, "BOOM!"} end
    Tp.run([f1, f2], [0])
    assert State.get == "initial"
    State.stop
    end

    View Slide

  105. Rolls back when it fails
    9
    test/tp_test.ex Test [4/4]
    test "rolls back when it fails" do
    State.start("initial")
    f1 = fn(x) ->
    state = State.get
    {State.put(x), [x], fn() -> State.put(state) end}
    end
    f2 = fn(_) -> {:error, "BOOM!"} end
    Tp.run([f1, f1, f2], [0])
    assert State.get == "initial"
    State.stop
    end

    View Slide

  106. Rolls back when it fails
    8
    lib/tp.ex
    def run(functions, params) do
    default_rollback = fn -> nil end
    do_run(functions, params, default_rollback)
    end

    View Slide

  107. Rolls back when it fails
    9
    lib/tp.ex
    def run(functions, params) do
    do_run(functions, params, _rollback = [])
    end

    View Slide

  108. Rolls back when it fails
    9
    lib/tp.ex
    defp do_run([], params, _), do: {:ok, params}
    defp do_run([h | t], params, rollback) do
    case apply(h, params) do
    {:ok, new_params} ->
    run(t, new_params, rollback)
    {:ok, new_params, new_rollback} ->
    run(t, new_params, new_rollback)
    {:error, details} ->
    rollback.()
    {:error, details}
    end
    end

    View Slide

  109. Rolls back when it fails
    9
    lib/tp.ex
    defp do_run([], params, _), do: {:ok, params}
    defp do_run([h | t], params, rollback) do
    case apply(h, params) do
    {:ok, new_params} ->
    run(t, new_params, rollback)
    {:ok, new_params, new_rollback} ->
    run(t, new_params, [new_rollback | rollback])
    {:error, details} ->
    rollback.()
    {:error, details}
    end
    end

    View Slide

  110. Rolls back when it fails
    9
    lib/tp.ex
    defp do_run([], params, _), do: {:ok, params}
    defp do_run([h | t], params, rollback) do
    case apply(h, params) do
    {:ok, new_params} ->
    run(t, new_params, rollback)
    {:ok, new_params, new_rollback} ->
    run(t, new_params, [new_rollback | rollback])
    {:error, details} ->
    Enum.each(rollback, fn(f) -> f.() end)
    {:error, details}
    end
    end

    View Slide

  111. it works!!

    View Slide

  112. Tp.run([
    &add_alias_record/2,
    &add_cname_record/2,
    &configure_cname_file/2,
    &save_connection/2,
    ], [connection, …])

    View Slide

  113. what’s
    point
    your
    ?

    View Slide

  114. practice

    View Slide

  115. be smart

    View Slide

  116. thank you

    View Slide

  117. questions

    View Slide