benchmark.livemd

Benchmark map/struct access syntaxes

Mix.install([
  {:benchee_dsl, "~> 0.5"}
  # {:benchee_markdown, "~> 0.3"}
])

Section

defmodule UnderTest do
  defstruct [:a, :b]

  {:function, :unpack, 1, 26,
   [
     {:line, 12},
     {:label, 25},
     {:func_info, {:atom, Test}, {:atom, :unpack}, 1},
     {:label, 26},
     {:test, :is_map, {:f, 25}, [x: 0]},
     {:get_map_elements, {:f, 25},
      {:tr, {:x, 0}, {{:t_map, :any, :any}, 0, 18_446_744_073_709_551_615}},
      {:list, [atom: :b, x: 3, atom: :a, x: 2, atom: :__struct__, x: 1]}},
     {:test, :is_eq_exact, {:f, 25}, [x: 1, atom: Test]},
     {:line, 13},
     {:gc_bif, :+, {:f, 0}, 4, [x: 2, x: 3], {:x, 0}},
     :return
   ]}

  def unpack(%__MODULE__{a: a, b: b}) do
    a + b
  end

  {:function, :match, 1, 20,
   [
     {:line, 6},
     {:label, 19},
     {:func_info, {:atom, Test}, {:atom, :match}, 1},
     {:label, 20},
     {:test, :is_map, {:f, 19}, [x: 0]},
     {:get_map_elements, {:f, 19},
      {:tr, {:x, 0}, {{:t_map, :any, :any}, 0, 18_446_744_073_709_551_615}},
      {:list, [atom: :__struct__, x: 1]}},
     {:test, :is_eq_exact, {:f, 19}, [x: 1, atom: Test]},
     {:get_map_elements, {:f, 21},
      {:tr, {:x, 0}, {{:t_map, :any, :any}, 0, 18_446_744_073_709_551_615}},
      {:list, [atom: :b, x: 2, atom: :a, x: 1]}},
     {:line, 7},
     {:gc_bif, :+, {:f, 0}, 3, [x: 1, x: 2], {:x, 0}},
     :return,
     {:label, 21},
     {:line, 8},
     {:badmatch, {:x, 0}}
   ]}

  def match(%__MODULE__{} = struct) do
    %__MODULE__{a: a, b: b} = struct
    a + b
  end

  {:function, :dottet_access, 1, 15,
   [
     {:line, 2},
     {:label, 14},
     {:func_info, {:atom, Test}, {:atom, :dottet_access}, 1},
     {:label, 15},
     {:test, :is_map, {:f, 14}, [x: 0]},
     {:get_map_elements, {:f, 14},
      {:tr, {:x, 0}, {{:t_map, :any, :any}, 0, 18_446_744_073_709_551_615}},
      {:list, [atom: :__struct__, x: 1]}},
     {:test, :is_eq_exact, {:f, 14}, [x: 1, atom: Test]},
     {:get_map_elements, {:f, 17},
      {:tr, {:x, 0}, {{:t_map, :any, :any}, 0, 18_446_744_073_709_551_615}},
      {:list, [atom: :a, x: 1]}},
     {:get_map_elements, {:f, 16},
      {:tr, {:x, 0}, {{:t_map, :any, :any}, 0, 18_446_744_073_709_551_615}},
      {:list, [atom: :b, x: 2]}},
     {:line, 3},
     {:gc_bif, :+, {:f, 0}, 3, [x: 1, x: 2], {:x, 0}},
     :return,
     {:label, 16},
     {:test_heap, 4, 1},
     {:put_tuple2, {:x, 0}, {:list, [atom: :badkey, atom: :b, x: 0]}},
     {:call_ext_only, 1, {:extfunc, :erlang, :error, 1}},
     {:label, 17},
     {:test_heap, 4, 1},
     {:put_tuple2, {:x, 0}, {:list, [atom: :badkey, atom: :a, x: 0]}},
     {:call_ext_only, 1, {:extfunc, :erlang, :error, 1}}
   ]}

  def dottet_access(%__MODULE__{} = struct) do
    struct.a + struct.b
  end

  {:function, :mixed, 1, 23,
   [
     {:line, 9},
     {:label, 22},
     {:func_info, {:atom, Test}, {:atom, :mixed}, 1},
     {:label, 23},
     {:test, :is_map, {:f, 22}, [x: 0]},
     {:get_map_elements, {:f, 22},
      {:tr, {:x, 0}, {{:t_map, :any, :any}, 0, 18_446_744_073_709_551_615}},
      {:list, [atom: :__struct__, x: 1]}},
     {:test, :is_eq_exact, {:f, 22}, [x: 1, atom: Test]},
     {:get_map_elements, {:f, 24},
      {:tr, {:x, 0}, {{:t_map, :any, :any}, 0, 18_446_744_073_709_551_615}},
      {:list, [atom: :b, x: 2, atom: :a, x: 1]}},
     {:line, 10},
     {:gc_bif, :+, {:f, 0}, 3, [x: 1, x: 2], {:x, 0}},
     :return,
     {:label, 24},
     {:line, 11},
     {:badmatch, {:x, 0}}
   ]}

  def mixed(%__MODULE__{} = struct) do
    %{a: a, b: b} = struct
    a + b
  end

  {:function, :map_access, 1, 18,
   [
     {:line, 4},
     {:label, 17},
     {:func_info, {:atom, Test}, {:atom, :map_access}, 1},
     {:label, 18},
     {:allocate, 1, 1},
     {:move, {:x, 0}, {:y, 0}},
     {:move, {:atom, :a}, {:x, 1}},
     {:line, 5},
     {:call_ext, 2, {:extfunc, Map, :get, 2}},
     {:swap, {:y, 0}, {:x, 0}},
     {:move, {:atom, :b}, {:x, 1}},
     {:call_ext, 2, {:extfunc, Map, :get, 2}},
     {:gc_bif, :+, {:f, 0}, 1, [y: 0, x: 0], {:x, 0}},
     {:deallocate, 1},
     :return
   ]}

  def map_access(struct) do
    Map.get(struct, :a) + Map.get(struct, :b)
  end
end

{:module, name, _binary, _bindings} =
  defmodule Benchmark do
    use BencheeDsl.Benchmark
    config(warmup: 5, time: 10, pre_check: true, print: [configuration: false])
    inputs(%{"input" => %UnderTest{a: 1, b: 2}})

    job(dottet_access(struct)) do
      UnderTest.dottet_access(struct)
    end

    job(unpack(struct)) do
      UnderTest.unpack(struct)
    end

    job(map_access(struct)) do
      UnderTest.map_access(struct)
    end

    job(match(struct)) do
      UnderTest.match(struct)
    end

    job(mixed(struct)) do
      UnderTest.mixed(struct)
    end
  end

BencheeDsl.Livebook.benchee_config() |> name.run() |> BencheeDsl.Livebook.render()

gistfile1.txt

##### With input input #####
Name                    ips        average  deviation         median         99th %
unpack               2.67 M      374.11 ns  ±1629.96%         357 ns         532 ns
match                2.60 M      384.25 ns  ±1755.78%         359 ns         581 ns
mixed                2.57 M      388.85 ns  ±1691.96%         360 ns         635 ns
dottet_access        2.55 M      392.23 ns  ±1685.90%         367 ns         648 ns
map_access           2.47 M      404.81 ns  ±1596.56%         378 ns         656 ns

Comparison: 
unpack               2.67 M
match                2.60 M - 1.03x slower +10.14 ns
mixed                2.57 M - 1.04x slower +14.73 ns
dottet_access        2.55 M - 1.05x slower +18.11 ns
map_access           2.47 M - 1.08x slower +30.70 ns

I didn't want to mention this in the last PR.

  def unpack(%__MODULE__{a: a, b: b}) do
    a + b
  end

  def dottet_access(%__MODULE__{} = struct) do
    struct.a + struct.b
  end

It seems that each dot access to a struct compiles to a single :get_map_elements instruction. So unpack/1 gets all map fields in one go and dottet_access/1 call :get_map_elements three times. Executing three instructions instead of one explains 5% difference in run time.

That behavior actually can be explained. The Erlang's compiler never tracks if a map actually has specific fields or not, so it has to generate error handling code:

     {:test_heap, 4, 1},
     {:put_tuple2, {:x, 0}, {:list, [atom: :badkey, atom: :b, x: 0]}},
     {:call_ext_only, 1, {:extfunc, :erlang, :error, 1}},
     {:label, 17},
     {:test_heap, 4, 1},
     {:put_tuple2, {:x, 0}, {:list, [atom: :badkey, atom: :a, x: 0]}},
     {:call_ext_only, 1, {:extfunc, :erlang, :error, 1}}

Which throws :badkey error like that:

** (KeyError) key :a not found in: %{}
    iex:3: Test.test2/1
    iex:2: (file)

On the other hand when matching happens in function's head it doesn't matter if :get_map_elements fails to extract all requested fields - execution flow goes to the next head and so on until it out of options:

** (FunctionClauseError) no function clause matching in Test.test/1    
    
    The following arguments were given to Test.test/1:
    
        # 1
        %{}

What do you think @mtrudel? Is it worth to thin down ThousandIsland.Socket module in exchange for using more verbose syntax?

Very interesting! Why don't we try this on a couple of the hot paths in Thousand Island & see if it makes a real world difference? The readability sacrifice isn't great, but it's honestly not THAT big of a difference, especially if it has substantial perf impacts (I'd call a 5% difference substantial in this case).

Sure, will do!