Ecto Without a DB: Many Meanings of Map

Conference talk video: https://www.youtube.com/watch?v=k_xDi7zAcNM

At a high level Ecto is about 3 main concepts: 1) managing connections to a database, 2) generating SQL, 3) defining and validating schema structs. This talk ignores the first two and focuses on the last part. Ecto 3.0 was refactored to separate core ecto from ecto_sql, but even Ecto 2.0 can be used without a database.

Peek behind the curtain a bit on how Ecto schemas and changesets work to gain intuition on what's possible and troubleshooting ideas when things don't go as you expect. Changesets are remarkably general and can be used to abstract an HTML form from an underlying data storage layout, or to manage advanced search criteria, or even as an anti-corruption layer for not-completely-trusted data coming into your system. We'll explore these use cases with practical code you could start using in your projects tomorrow.

Transcript

1. Ecto Without a DB: Many Meanings of Map Greg Vaughn
2. • @gvaughn or @gregvaughn on GitHub, Twitter, Slack, ElixirForum, etc. • Elixiring since 2013 • Professionally since 2017 Who Dat
3. ORM? ORM? ORM What's Ecto?
4. Main Parts of Ecto ✔ ✔
5. iex environment $ mix phx.new ecto_without_a_db --module EctoNoDb --no-ecto Added to mix.exs in `deps/0` function: {:ecto, "~> 3.0"}, {:phoenix_ecto, "~> 4.0"} $ mix deps.get $ iex -S mix
6. Structs are maps with superpowers Every journey starts with a map
7. Anatomy of a Struct iex(1)> defmodule A, do: defstruct [a: 1, b: 2, c: 3] {:module, A, <<... iex(2)> inspect %A{} "%A{a: 1, b: 2, c: 3}" iex(3)> inspect %A{}, structs: false "%{__struct__: A, a: 1, b: 2, c: 3}" iex(4)> A.__info__(:functions) [__struct__: 0, __struct__: 1] iex(5)> inspect A.__struct__(), structs: false "%{__struct__: A, a: 1, b: 2, c: 3}"
8. Schemas are Structs with superpowers2
9. Anatomy of an Embedded Schema iex(1)> defmodule B do ...(1)> use Ecto.Schema; embedded_schema do ...(1)> field(:a, :integer); field(:b, :string); field(:c, :float) ...(1)> end ...(1)> end {:module, B, <<... iex(2)> inspect %B{}, structs: false "%{__struct__: B, a: nil, b: nil, c: nil, id: nil}" iex(3)> B.__info__(:functions) [__changeset__: 0, __schema__: 1, __schema__: 2, __struct__: 0, __struct__: 1] iex(4)> B.__schema__(:fields) [:id, :a, :b, :c] iex(5)> B.__changeset__() %{a: :integer, b: :string, c: :float, id: :binary_id}
10. Changesets are Command Objects Changesets are Command Objects Structs
11. Anatomy of a Changeset iex(1)> inspect %Ecto.Changeset{} #Ecto.Changeset< action: nil, changes: %{}, errors: [], data: nil, valid?: false > iex(2)> inspect %Ecto.Changeset{}, structs: false "%{__struct__: Ecto.Changeset, action: nil, changes: %{}, constraints: [], data: nil, empty_values: [\"\"], errors: [], filters: %{}, params: nil, prepare: [], repo: nil, repo_opts: [], required: [], types: nil, valid?: false, validations: [] }"
12. Anatomy of a Changeset iex(3)> c = Ecto.Changeset.cast(%B{a: 0}, %{a: 1, b: "Lonestar", c: 3.0}, [:a, :b]) #Ecto.Changeset< action: nil, changes: %{a: 1, b: "Lonestar"}, errors: [], data: #B<>, valid?: true > iex(4)> inspect c, structs: false "%{__struct__: Ecto.Changeset, action: nil, changes: %{a: 1, b: \"Lonestar\"}, constraints: [], empty_values: [\"\"], errors: [], filters: %{}, prepare: [], repo: nil, repo_opts: [], required: [], valid?: true, validations: [] data: %{__struct__: B, a: 0, b: nil, c: nil, id: nil}, params: %{\"a\" => 1, \"b\" => \"Lonestar\", \"c\" => 3.0}, types: %{a: :integer, b: :string, c: :float, id: :binary_id}, }"
13. Back to Struct-land iex(1)> {:ok, b} = Ecto.Changeset.apply_action(c, :update) {:ok, %B{a: 1, b: "Lonestar", c: nil, id: nil}} iex(2)> Ecto.Changeset.cast(%B{}, %{a: 3.14}, [:a, :b]) |> ...(2)> Ecto.Changeset.apply_action(:update) {:error, #Ecto.Changeset< action: :update, changes: %{}, errors: [a: {"is invalid", [type: :integer, validation: :cast]}], data: #B<>, valid?: false >}
14. Elmo loves "M" for "mapping!" Elmo loves Ecto because it rhymes with "Elmo!"
15. The Phoenix Protocols
16. Transform to Form iex(1)> {:error, c} = Ecto.Changeset.cast(%B{}, %{a: 3.14}, [:a, :b]) |> Ecto.Changeset.apply_action(:update) ... iex(2)> Phoenix.HTML.Form.form_for(c, "http://somewhere/clever") %Phoenix.HTML.Form{ action: "http://somewhere/clever", data: %B{a: nil, b: nil, c: nil, id: nil}, errors: [a: {"is invalid", [type: :integer, validation: :cast]}], hidden: [], id: "b", impl: Phoenix.HTML.FormData.Ecto.Changeset, index: nil, name: "b", options: [method: "post"], params: %{"a" => 3.14}, source: #Ecto.Changeset< action: :update, changes: %{}, errors: [a: {"is invalid", [type: :integer, validation: :cast]}], data: #B<>, valid?: false > }
17. Example: Advanced Search defmodule EctoNoDb.EventSearchController do use EctoNoDbWeb, :controller def index(conn, %{"q" => query_params}) do ... end end
18. Example: Advanced Search iex(1)> query_params = %{"name" => "BBQ", "city" => "Austin", "start" => "2019-02-28", "end" => "2019-03-03"} iex(2)> types = %{name: :string, city: :string, start: :date, end: :date} iex(3)> c = Ecto.Changeset.cast({%{}, types}, query_params, Map.keys(types)) #Ecto.Changeset< action: nil, changes: %{ city: "Austin", end: ~D[2019-03-03], name: "BBQ", start: ~D[2019-02-28] }, errors: [], data: %{}, valid?: true >
19. Example: Advanced Search iex(4)> inspect c, structs: false "%{__struct__: Ecto.Changeset, ... without fields shown above ... constraints: [], empty_values: [\"\"], filters: %{}, prepare: [], repo: nil, repo_opts: [], required: [], validations: [], params: %{\"city\" => \"Austin\", \"end\" => \"2019-03-03\", \"name\" => \"BBQ\", \"start\" => \"2019-02-28\"}, types: %{city: :string, end: :date, name: :string, start: :date} }" iex(5)> Ecto.Changeset.apply_action(c, :update) {:ok, %{city: "Austin", end: ~D[2019-03-03], name: "BBQ", start: ~D[2019-02-28]}}
20. Example: Advanced Search defmodule EctoNoDb.EventSearchController do use EctoNoDbWeb, :controller @query_types = %{name: :string, city: :string, start: :date, end: :date} def index(conn, %{"q" => query_params}) do with {:ok, query} <- cast_params(query_params, @query_types), {:ok, results} <- NotInScopeForThisTalk.get_results(query) do ... do your thang ... end end defp cast_params(query_params, type_map) do {%{}, type_map} |> Ecto.Changeset.cast(query_params, Map.keys(type_map)) |> Ecto.Changeset.apply_action(:update) end end
21. Anti-Corruption Layer “If your application needs to deal with a database or another application whose model is undesirable or inapplicable to the model you want within your own application, use an AnticorruptionLayer to translate to/from that model and yours.” – c2 wiki
22. Anti-Corruption Layer API Client Anti Corruption Layer Domain Logic Typed Struct
23. Anti-Corruption Layer defmodule EctoNoDb.Event do use Ecto.Schema import Ecto.Changeset embedded_schema do field(:name, :string) field(:venue_name, :string) field(:venue_city, :string) field(:occurs_at, :naive_datetime) end def new(raw_map) do %__MODULE__{} |> cast(raw_map, [:name, :venue_name, :venue_city, :occurs_at]) |> apply_action(:update) end end
24. Anti-Corruption Layer defmodule EctoNoDb.Event do use Ecto.Schema import Ecto.Changeset embedded_schema do field(:name, :string) field(:occurs_at, :naive_datetime) embeds_one(:venue, EctoNoDb.Venue) embeds_one(:performer, EctoNoDb.Performer) end def new(raw_map) do %__MODULE__{} |> cast(raw_map, [:name, :occurs_at]) |> cast_embed(:venue) |> cast_embed(:performer) |> apply_action(:update) end end
25. Anti-Corruption Layer defmodule EctoNoDb.Event do ... def new(raw_map) do %__MODULE__{} |> changeset(raw_map) |> apply_action(:update) end defp changeset(base, raw_map) do base |> cast(raw_map, [:name, :occurs_at]) |> cast_embed(:venue) |> cast_embed(:performer) end ... end
26. Meta-Anti-Corruption Layer defmodule EctoNoDb.Event do use EctoNoDb.AntiCorruptionSchema embedded_schema do field(:name, :string) field(:occurs_at, :naive_datetime) embeds_one(:venue, EctoNoDb.Venue) embeds_one(:performer, EctoNoDb.Performer) end end defmodule EctoNoDb.Venue do use EctoNoDb.AntiCorruptionSchema embedded_schema do field(:name, :string) field(:city, :string) end end
27. Meta-Anti-Corruption Layer defmodule EctoNoDb.AntiCorruptionSchema do defmacro __using__(_opts) do quote do use Ecto.Schema @primary_key false def new(raw_map), do: unquote(__MODULE__).generic_new(raw_map, __MODULE__) def changeset(base, params), do: unquote(__MODULE__).generic_changeset(base, params) defoverridable new: 1, changeset: 2 end end def generic_new(raw_map, struct_module) do struct(struct_module) |> struct_module.changeset(raw_map) |> Ecto.Changeset.apply_action(:update) end def generic_changeset(base, raw_map) do struct_mod = base.__struct__ embeds = struct_mod.__schema__(:embeds) allowed = struct_mod.__schema__(:fields) -- embeds chg = Ecto.Changeset.cast(base, raw_map, allowed) Enum.reduce(embeds, chg, &Ecto.Changeset.cast_embed(&2, &1)) end end
28. Thank You • Takeaways • Maps -> Structs -> Schemas -> Changesets -> Structs/Maps • Understanding one level below where you are focused • Ideas for when to use typed structs in your domain • gvaughn@gmail.com • Twitter: @gregvaughn • Elixir Slack: @gregvaughn • Elixir Forum: gregvaughn • Questions?