Building a transactional pipeline with TDD and Elixir

We are used to work with database level transactions. But, what happens when the transactional limits go beyond the database? How do we handle a complex transaction that affects different systems? How would you solve this problem using a functional language?

In this talk we will describe a real problem that we faced at DNSimple. We will discuss the challenges that we faced and how we used TDD and Elixir to solve it.

Transcript

1. Transactional
   Pipeline
   function()
   Building a
   with TDD
   and Elixir
   #scpna17
   #teamtinyp
   

   View Slide

2. jacegu jacegu
   javieracero.com
   

   View Slide

3. https://dnsimple.com
   

   View Slide

4. d s
   n
   http://howdns.works
   

   View Slide

5. pamplonaswcraft.com
   192.30.252.153
   192.30.252.154
   

   View Slide

6. pamplonaswcraft.com
   

   View Slide

7. https://help.github.com/articles/using-a-custom-domain-with-github-pages
   

   View Slide

8. https://help.github.com/articles/using-a-custom-domain-with-github-pages
   

   View Slide

9. View Slide

10. View Slide

11. domains
    services
    Connect
    to
    

    View Slide

12. api
    

    View Slide

13. api
    

    View Slide

14. connectors
    http://platform.dnsimple.com
    

    View Slide

15. View Slide

16. transactional
    

    View Slide

17. View Slide

18. if elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif elsif elsif
    elsif elsif elsif else end
    

    View Slide

19. connection
    |> add_alias_record(…)
    |> add_cname_record(…)
    |> configure_cname_file(…)
    |> save_connection(…)
    

    View Slide

20. WHERE ARE MY OBJECTS
    WHERE ARE MY OBJECTS
    

    View Slide

21. View Slide

22. Runs anonymous functions without params
    1
    

    View Slide

23. Runs anonymous functions without params
    defmodule TpTest do
    use ExUnit.Case
    test "runs anonymous functions without params" do
    assert Tp.run(fn -> 2 + 2 end) == 4
    end
    end
    1
    test/tp_test.ex Test [1/1]
    

    View Slide

24. test "runs anonymous functions without params" do
    assert Tp.run(fn -> 2 + 2 end) == 4
    end
    Runs anonymous functions without params
    1
    test/tp_test.ex Test [1/1]
    

    View Slide

25. test "runs anonymous functions without params" do
    assert Tp.run(fn -> 2 + 2 end) == 4
    end
    Runs anonymous functions without params
    1
    test/tp_test.ex Test [1/1]
    

    View Slide

26. Runs anonymous functions without params
    defmodule Tp do
    def run(_) do
    nil
    end
    end
    1
    lib/tp.ex
    

    View Slide

27. Runs anonymous functions without params
    def run(_) do
    nil
    end
    1
    lib/tp.ex
    

    View Slide

28. Runs anonymous functions without params
    def run(_) do
    4
    end
    1
    lib/tp.ex
    

    View Slide

29. Runs anonymous functions without params
    test "runs an anonymous function without params" do
    assert Tp.run(fn -> 2 + 2 end) == 4
    end
    1
    test/tp_test.ex Test [1/1]
    

    View Slide

30. Runs anonymous functions without params
    test "runs an anonymous function without params" do
    assert Tp.run(fn -> 2 + 2 end) == 4
    assert Tp.run(fn -> String.upcase("x") end) == "X"
    end
    1
    test/tp_test.ex Test [1/1]
    

    View Slide

31. Runs anonymous functions without params
    def run(_) do
    4
    end
    1
    lib/tp.ex
    

    View Slide

32. Runs anonymous functions without params
    def run(fun) do
    fun.()
    end
    1
    lib/tp.ex
    

    View Slide

33. Runs anonymous functions with 1 parameter
    2
    

    View Slide

34. test "runs anonymous functions with 1 param" do
    assert Tp.run(fn(x) -> 2 + x end, 2) == 4
    end
    Runs anonymous functions with 1 parameter
    2
    test/tp_test.ex Test [2/2]
    

    View Slide

35. Runs anonymous functions with 1 parameter
    def run(fun) do
    fun.()
    end
    2
    lib/tp.ex
    

    View Slide

36. Runs anonymous functions with 1 parameter
    def run(fun) do
    fun.()
    end
    def run(fun, param), do: fun.(param)
    2
    lib/tp.ex
    

    View Slide

37. Runs anonymous functions with 1 parameter
    def run(fun), do: fun.()
    def run(fun, param), do: fun.(param)
    2
    lib/tp.ex
    

    View Slide

38. Runs anonymous functions with 2 parameters
    3
    

    View Slide

39. Runs anonymous functions with 2 parameters
    test "runs an anonymous function with 2 params" do
    assert Tp.run(fn(x, y) -> x+y end, [1, 2]) == 3
    end
    3
    test/tp_test.ex Test [3/3]
    

    View Slide

40. Runs anonymous functions with 2 parameters
    def run(fun), do: fun.()
    def run(fun, param), do: fun.(param)
    3
    lib/tp.ex
    

    View Slide

41. Runs anonymous functions with 2 parameters
    def run(fun), do: fun.()
    def run(fun, params) when is_list(params),
    do: apply(fun, params)
    def run(fun, param), do: fun.(param)
    3
    lib/tp.ex
    

    View Slide

42. Runs anonymous functions with 2 parameters
    test "runs an anonymous function with 2 params" do
    assert Tp.run(fn(x, y) -> x+y end, [1, 2]) == 3
    end
    3
    test/tp_test.ex Test [3/3]
    

    View Slide

43. test "runs an anonymous function with 2 params" do
    assert Tp.run(fn -> 2 + 2 end, []) == 4
    assert Tp.run(&String.upcase/1, ["x"]) == "X"
    assert Tp.run(fn(x, y) -> x+y end, [1, 2]) == 3
    end
    3
    test/tp_test.ex Test [3/3]
    Runs anonymous functions with 2 parameters
    

    View Slide

44. test "runs a function" do
    assert Tp.run(fn -> 2 + 2 end, []) == 4
    assert Tp.run(&String.upcase/1, ["x"]) == "X"
    assert Tp.run(fn(x, y) -> x+y end, [1, 2]) == 3
    end
    3
    test/tp_test.ex Test [3/3]
    Runs a function
    Test [1/1]
    

    View Slide

45. Runs a function
    def run(fun), do: fun.()
    def run(fun, params) when is_list(params),
    do: apply(fun, params)
    def run(fun, param), do: fun.(param)
    3
    lib/tp.ex
    

    View Slide

46. Runs a function
    def run(fun, params), do: apply(fun, params)
    3
    lib/tp.ex
    

    View Slide

47. Runs multiple functions
    4
    

    View Slide

48. Runs multiple functions
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> [x*x, y*y, z*z] end
    f2 = fn(x, y, z) -> [z, y, x] end
    assert Tp.run([f1, f2], [2, 3, 5]) == [25, 9, 4]
    end
    4
    test/tp_test.ex Test [2/2]
    

    View Slide

49. Runs multiple functions
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex
    

    View Slide

50. Runs multiple functions
    def run([fun1, fun2], params) do
    apply(fun2, apply(fun1, params))
    end
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex
    

    View Slide

51. Runs multiple functions
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> [x*x, y*y, z*z] end
    f2 = fn(x, y, z) -> [z, y, x] end
    assert Tp.run([f1, f2], [2, 3, 5]) == [25, 9, 4]
    end
    4
    test/tp_test.ex Test [2/2]
    

    View Slide

52. Runs multiple functions
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> [x*x, y*y, z*z] end
    f2 = fn(x, y, z) -> [z, y, x] end
    assert Tp.run([], [1, 2, 3]) == [1, 2, 3]
    assert Tp.run([f2], [7, 8, 9]) == [9, 8, 7]
    assert Tp.run([f1, f2], [2, 3, 5]) == [25, 9, 4]
    end
    4
    test/tp_test.ex Test [2/2]
    

    View Slide

53. Runs multiple functions
    def run([fun1, fun2], params) do
    apply(fun2, apply(fun1, params))
    end
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex
    

    View Slide

54. Runs multiple functions
    def run([first|rest], params) do
    apply(fun2, apply(fun1, params))
    end
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex
    

    View Slide

55. Runs multiple functions
    def run([first|rest], params) do
    run(rest, apply(first, params))
    end
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex
    

    View Slide

56. Runs multiple functions
    def run([], params), do: params
    def run([first|rest], params) do
    run(rest, apply(first, params))
    end
    def run(fun, params), do: apply(fun, params)
    4
    lib/tp.ex
    

    View Slide

57. Runs multiple functions
    4
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params), do: run(t, apply(h, params))
    def run(fun, params), do: apply(fun, params)
    

    View Slide

58. Runs multiple functions
    4
    test/tp_test.ex Test [1/1]
    test "runs a function" do
    assert Tp.run(fn -> 2 + 2 end, []) == 4
    assert Tp.run(&String.upcase/1, ["x"]) == "X"
    assert Tp.run(fn(x, y) -> x+y end, [1, 2]) == 3
    end
    

    View Slide

59. Runs multiple functions
    test "runs a function" do
    assert Tp.run([fn -> 2 + 2 end], []) == 4
    assert Tp.run([&String.upcase/1], ["x"]) == "X"
    assert Tp.run([fn(x, y) -> x+y end], [1, 2]) == 3
    end
    4
    test/tp_test.ex Test [1/1]
    

    View Slide

60. Runs multiple functions
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> [x*x, y*y, z*z] end
    f2 = fn(x, y, z) -> [z, y, x] end
    assert Tp.run([], [1, 2, 3]) == [1, 2, 3]
    assert Tp.run([f2], [7, 8, 9]) == [9, 8, 7]
    assert Tp.run([f1, f2], [2, 3, 5]) == [25, 9, 4]
    end
    4
    test/tp_test.ex Test [2/2]
    Test [1/1]
    

    View Slide

61. Runs multiple functions
    4
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params), do: run(t, apply(h, params))
    def run(fun, params), do: apply(fun, params)
    

    View Slide

62. Runs multiple functions
    4
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params), do: run(t, apply(h, params))
    

    View Slide

63. View Slide

64. View Slide

65. View Slide

66. Runs functions with side effects
    Runs “real world” functions
    5
    

    View Slide

67. Runs “real world” functions
    defmodule State do
    def start(initial \\ nil), do: #...
    def stop, do: #...
    def get, do: #...
    def put(new), do: #...
    end
    5
    test/test_helper.ex
    

    View Slide

68. Runs “real world” functions
    5
    test/tp_test.ex
    test "runs functions with side effects" do
    State.start
    f1 = fn(x) -> State.put(x); [x] end
    f2 = fn(x) -> [x*x*x] end
    assert Tp.run([f1, f2], [3]) == [27]
    assert State.get == 3
    State.stop
    end
    Test [2/2]
    

    View Slide

69. Returns whether it failed
    6
    

    View Slide

70. ERR R
    HANDLING
    

    View Slide

71. http://elixir-lang.org/getting-started/try-catch-and-rescue.html
    

    View Slide

72. http://elixir-lang.org/getting-started/try-catch-and-rescue.html
    

    View Slide

73. try do
    result = this_can_fail!(params)
    continue(result)
    rescue
    error -> handle_error(error)
    end
    case this_can_fail(params) do
    {:ok, result} -> continue(result)
    {:error, error} -> handle_error(params)
    end
    vs
    

    View Slide

74. Returns whether it failed
    6
    

    View Slide

75. Returns whether it failed
    6
    test "returns whether it failed" do
    State.start
    f1 = fn(x) -> State.put(x); [x] end
    f2 = fn(x) -> [x*x*x] end
    f3 = fn(_) -> {:error, “KO"} end
    assert Tp.run([f1, f2], [3]) == [27]
    assert Tp.run([f3, f2], [3]) == {:error, “KO"}
    State.stop
    end
    test/tp_test.ex Test [3/3]
    

    View Slide

76. Returns whether it failed
    6
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params), do: run(t, apply(h, params))
    

    View Slide

77. Returns whether it failed
    6
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params) do
    run(t, apply(h, params))
    end
    

    View Slide

78. Returns whether it failed
    6
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params) do
    case apply(h, params) do
    run(t,)
    end
    end
    

    View Slide

79. Returns whether it failed
    6
    lib/tp.ex
    def run([], params), do: params
    def run([h|t], params) do
    case apply(h, params) do
    {:error, details} -> {:error, details}
    run(t,)
    end
    end
    

    View Slide

80. Returns whether it failed
    6
    lib/tp.ex
    def run([], params), do: params
    def run([h | t], params) do
    case apply(h, params) do
    {:error, details} -> {:error, details}
    new_params -> run(t, new_params)
    end
    end
    

    View Slide

81. Returns whether it succeeded or failed
    7
    

    View Slide

82. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [3/3]
    test "returns whether it failed" do
    State.start
    f1 = fn(x) -> State.put(x); [x] end
    f2 = fn(x) -> [x*x*x] end
    f3 = fn(_) -> {:error, "KO"} end
    assert Tp.run([f1, f2], [3]) == [27]
    assert Tp.run([f3, f2], [3]) == {:error, "KO"}
    State.stop
    end
    

    View Slide

83. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [3/3]
    test "returns whether it succeeded or failed" do
    State.start
    f1 = fn(x) -> State.put(x); [x] end
    f2 = fn(x) -> [x*x*x] end
    f3 = fn(_) -> {:error, "KO"} end
    assert Tp.run([f1, f2], [3]) == {:ok, [27]}
    assert Tp.run([f3, f2], [3]) == {:error, "KO"}
    State.stop
    end
    

    View Slide

84. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [3/3]
    test "returns whether it succeeded or failed" do
    State.start
    f1 = fn(x) -> State.put(x); {:ok, [x]} end
    f2 = fn(x) -> {:ok, [x*x*x]} end
    f3 = fn(_) -> {:error, "KO"} end
    assert Tp.run([f1, f2], [3]) == {:ok, [27]}
    assert Tp.run([f3, f2], [3]) == {:error, "KO"}
    State.stop
    end
    

    View Slide

85. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [2/3]
    test "runs functions with side effects" do
    State.start
    f1 = fn(x) -> State.put(x); [x] end
    f2 = fn(x) -> [x*x*x] end
    assert Tp.run([f1, f2], [3]) == [27]
    assert State.get == 3
    State.stop
    end
    

    View Slide

86. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [2/3]
    test "runs functions with side effects" do
    State.start
    f1 = fn(x) -> State.put(x); {:ok, [x]} end
    f2 = fn(x) -> {:ok, [x*x*x]} end
    assert Tp.run([f1, f2], [3]) == {:ok, [27]}
    assert State.get == 3
    State.stop
    end
    

    View Slide

87. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [1/3]
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> [x*x, y*y, z*z] end
    f2 = fn(x, y, z) -> [z, y, x] end
    assert Tp.run([], [1, 2, 3]) == [1, 2, 3]
    assert Tp.run([f2], [7, 8, 9]) == [9, 8, 7]
    assert Tp.run([f1, f2], [2, 3, 5]) == [25, 9, 4]
    end
    

    View Slide

88. Returns whether it succeeded or failed
    7
    test/tp_test.ex Test [1/3]
    test "runs multiple functions" do
    f1 = fn(x, y, z) -> {:ok, [x*x, y*y, z*z]} end
    f2 = fn(x, y, z) -> {:ok, [z, y, x]} end
    assert Tp.run([], [1, 2, 3]) == {:ok, [1, 2, 3]}
    assert Tp.run([f2], [7, 8, 9]) ==
    {:ok, [9, 8, 7]}
    assert Tp.run([f1, f2], [2, 3, 5]) ==
    {:ok, [25, 9, 4]}
    end
    

    View Slide

89. Returns whether it succeeded or failed
    7
    lib/tp.ex
    def run([], params), do: params
    def run([h | t], params) do
    case apply(h, params) do
    {:error, details} -> {:error, details}
    new_params -> run(t, new_params)
    end
    end
    

    View Slide

90. Returns whether it succeeded or failed
    7
    lib/tp.ex
    def run([], params), do: params
    def run([h | t], params) do
    case apply(h, params) do
    {:error, details} -> {:error, details}
    {:ok, new_params} -> run(t, new_params)
    end
    end
    

    View Slide

91. Returns whether it succeeded or failed
    7
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:error, details} -> {:error, details}
    {:ok, new_params} -> run(t, new_params)
    end
    end
    

    View Slide

92. Returns whether it succeeded or failed
    7
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:ok, new_params} -> run(t, new_params)
    {:error, details} -> {:error, details}
    end
    end
    

    View Slide

93. Rolls back when it fails
    8
    

    View Slide

94. Rolls back when it fails
    8
    test "rolls back when it fails" do
    State.start("initial")
    f1 = fn(x) -> State.put(x); {:ok, [x]} end
    f2 = fn(_) -> {:error, "BOOM!"} end
    Tp.run([f1, f2], [0])
    assert State.get == "initial"
    State.stop
    end
    test/tp_test.ex Test [4/4]
    

    View Slide

95. Rolls back when it fails
    8
    test "rolls back when it fails" do
    State.start("initial")
    f1 = fn(x) ->
    state = State.get
    State.put(x)
    {:ok, [x], fn() -> State.put(state) end}
    end
    f2 = fn(_) -> {:error, "BOOM!"} end
    Tp.run([f1, f2], [0])
    assert State.get == "initial"
    State.stop
    end
    test/tp_test.ex Test [4/4]
    

    View Slide

96. Rolls back when it fails
    8
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:ok, new_params} -> run(t, new_params)
    {:error, details} -> {:error, details}
    end
    end
    

    View Slide

97. Rolls back when it fails
    8
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:ok, new_params} ->
    run(t, new_params)
    {:error, details} ->
    {:error, details}
    end
    end
    

    View Slide

98. Rolls back when it fails
    8
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:ok, new_params} ->
    run(t, new_params)
    {:error, details} ->
    rollback.()
    {:error, details}
    end
    end
    

    View Slide

99. Rolls back when it fails
    8
    lib/tp.ex
    def run([], params), do: {:ok, params}
    def run([h | t], params) do
    case apply(h, params) do
    {:ok, new_params} ->
    run(t, new_params)
    {:ok, new_params, new_rollback} ->
    run(t, new_params)
    {:error, details} ->
    rollback.()
    {:error, details}
    end
    end
    

    View Slide

100. Rolls back when it fails
     8
     lib/tp.ex
     def run([], params, _), do: {:ok, params}
     def run([h | t], params, rollback) do
     case apply(h, params) do
     {:ok, new_params} ->
     run(t, new_params, rollback)
     {:ok, new_params, new_rollback} ->
     run(t, new_params, new_rollback)
     {:error, details} ->
     rollback.()
     {:error, details}
     end
     end
     

     View Slide

101. Rolls back when it fails
     8
     lib/tp.ex
     defp do_run([], params, _), do: {:ok, params}
     defp do_run([h | t], params, rollback) do
     case apply(h, params) do
     {:ok, new_params} ->
     run(t, new_params, rollback)
     {:ok, new_params, new_rollback} ->
     run(t, new_params, new_rollback)
     {:error, details} ->
     rollback.()
     {:error, details}
     end
     end
     

     View Slide

102. Rolls back when it fails
     8
     lib/tp.ex
     def run(functions, params) do
     default_rollback = fn -> nil end
     do_run(functions, params, default_rollback)
     end
     

     View Slide

103. Rolls back when it fails
     9
     

     View Slide

104. Rolls back when it fails
     9
     test/tp_test.ex Test [4/4]
     test "rolls back when it fails" do
     State.start("initial")
     f1 = fn(x) ->
     state = State.get
     {State.put(x), [x], fn() -> State.put(state) end}
     end
     f2 = fn(_) -> {:error, "BOOM!"} end
     Tp.run([f1, f2], [0])
     assert State.get == "initial"
     State.stop
     end
     

     View Slide

105. Rolls back when it fails
     9
     test/tp_test.ex Test [4/4]
     test "rolls back when it fails" do
     State.start("initial")
     f1 = fn(x) ->
     state = State.get
     {State.put(x), [x], fn() -> State.put(state) end}
     end
     f2 = fn(_) -> {:error, "BOOM!"} end
     Tp.run([f1, f1, f2], [0])
     assert State.get == "initial"
     State.stop
     end
     

     View Slide

106. Rolls back when it fails
     8
     lib/tp.ex
     def run(functions, params) do
     default_rollback = fn -> nil end
     do_run(functions, params, default_rollback)
     end
     

     View Slide

107. Rolls back when it fails
     9
     lib/tp.ex
     def run(functions, params) do
     do_run(functions, params, _rollback = [])
     end
     

     View Slide

108. Rolls back when it fails
     9
     lib/tp.ex
     defp do_run([], params, _), do: {:ok, params}
     defp do_run([h | t], params, rollback) do
     case apply(h, params) do
     {:ok, new_params} ->
     run(t, new_params, rollback)
     {:ok, new_params, new_rollback} ->
     run(t, new_params, new_rollback)
     {:error, details} ->
     rollback.()
     {:error, details}
     end
     end
     

     View Slide

109. Rolls back when it fails
     9
     lib/tp.ex
     defp do_run([], params, _), do: {:ok, params}
     defp do_run([h | t], params, rollback) do
     case apply(h, params) do
     {:ok, new_params} ->
     run(t, new_params, rollback)
     {:ok, new_params, new_rollback} ->
     run(t, new_params, [new_rollback | rollback])
     {:error, details} ->
     rollback.()
     {:error, details}
     end
     end
     

     View Slide

110. Rolls back when it fails
     9
     lib/tp.ex
     defp do_run([], params, _), do: {:ok, params}
     defp do_run([h | t], params, rollback) do
     case apply(h, params) do
     {:ok, new_params} ->
     run(t, new_params, rollback)
     {:ok, new_params, new_rollback} ->
     run(t, new_params, [new_rollback | rollback])
     {:error, details} ->
     Enum.each(rollback, fn(f) -> f.() end)
     {:error, details}
     end
     end
     

     View Slide

111. it works!!
     

     View Slide

112. Tp.run([
     &add_alias_record/2,
     &add_cname_record/2,
     &configure_cname_file/2,
     &save_connection/2,
     ], [connection, …])
     

     View Slide

113. what’s
     point
     your
     ?
     

     View Slide

114. practice
     

     View Slide

115. be smart
     

     View Slide

116. thank you
     

     View Slide

117. questions
     

     View Slide