Benchee Demos

benchee-demos.livemd

Benchee Demos

Mix.install([
  {:benchee, "~> 1.3"},
  {:kino_benchee, "~> 0.1.0"},
  {:jason, "~> 1.4"},
  {:jiffy, "~> 1.1"},
  {:benchee_markdown, "~> 0.3.3"}
])

Intro to Benchee

Benchee is the most common benchmarking library in the Elixir ecosystem.

The most basic way to invoke Benchee is to provide a map to the Benchee.run/1 function.

The keys are the name of the test case, while the values are the implementation to be tested.

It's recommended to use functions defined in a module (as opposed to inline functions) to benefit from optimizations within the runtime.

defmodule AddNumbersImplementations do
  def with_sleep do
    :timer.sleep(1)

    Enum.sum(1..100)
  end

  def no_sleep do
    Enum.sum(1..100)
  end
end

Benchee.run(%{
  "With Sleep" => &AddNumbersImplementations.with_sleep/0,
  "No Sleep" => &AddNumbersImplementations.no_sleep/0
})

Kino.nothing()

Benchmarking JSON Encoding

JSON was natively added to the beam virtual machine in OTP 27. If this notebook is running on an older version of OTP, we should not include the non-existent native JSON module in our benchmarks.

To detect this, we grab the otp_release value from the system and convert it to an integer. Note that List.to_integer/1 has to be used since the OTP release response is an Erlang charlist, not a string.

current_otp_version =
  :erlang.system_info(:otp_release)
  |> List.to_integer()

has_native_json_module = current_otp_version >= 27

For a benchmark of JSON encoding, we construct a single object containing strings, numbers, a nested object, and a list.

To test libraries at different payload sizes, we construct a list of objects with a specified length. In order to avoid any possible optimizations from identical repeated objects, we put an incrementing id value on each object in the list.

single_object = %{
  "hello" => "world",
  "example" => 123,
  "nested" => %{
    "key" => "value"
  },
  "list" => [1, 2, 3]
}

make_list_of_objects = fn count ->
  1..count
  |> Enum.map(fn x ->
    Map.put(single_object, "id", x)
  end)
end

json_encoding_benchee_jobs = %{
  "Jason" => &Jason.encode_to_iodata!/1,
  "Jiffy" => &:jiffy.encode/1
}

# Add native JSON to the comparison if it is supported by our OTP version.
json_encoding_benchee_jobs =
  if has_native_json_module do
    Map.put(json_encoding_benchee_jobs, "Native JSON", &:json.encode/1)
  else
    json_encoding_benchee_jobs
  end

result =
  Benchee.run(
    json_encoding_benchee_jobs,
    inputs: %{
      "Single Object" => single_object,
      "List of 10 Objects" => make_list_of_objects.(10),
      "List of 100 Objects" => make_list_of_objects.(100),
      "List of 1,000 Objects" => make_list_of_objects.(1_000),
      "List of 10,000 Objects" => make_list_of_objects.(10_000)
    }
  )

Kino.nothing()

Benchmarking JSON Decoding

json_decoding_benchee_jobs = %{
  "Jason" => &Jason.decode!/1,
  "Jiffy" => &:jiffy.decode/1
}

# Add native JSON to the comparison if it is supported by our OTP version.
json_decoding_benchee_jobs =
  if has_native_json_module do
    Map.put(json_decoding_benchee_jobs, "Native JSON", &:json.decode/1)
  else
    json_decoding_benchee_jobs
  end

default_json_decoding_input = ~s({"hello": "world"})

decoding_input =
  Kino.Input.textarea("JSON to Decode", monospace: true, default: default_json_decoding_input)

Benchee.run(
  json_decoding_benchee_jobs,
  inputs: %{
    "Provided Input" => Kino.Input.read(decoding_input)
  },
  memory_time: 5
)
|> Benchee.Formatters.Markdown.render()
|> Kino.Markdown.new()

Sorting Algorithms

Example sorting algorithm implementations come from Rosetta Code.

defmodule Sort do
  # An implementation of bubble sort
  def bsort(list) when is_list(list) do
    t = bsort_iter(list)

    if t == list, do: t, else: bsort(t)
  end

  defp bsort_iter([x, y | t]) when x > y, do: [y | bsort_iter([x | t])]
  defp bsort_iter([x, y | t]), do: [x | bsort_iter([y | t])]
  defp bsort_iter(list), do: list

  # An implementation of quick sort
  def qsort([]), do: []

  def qsort([h | t]) do
    {lesser, greater} = Enum.split_with(t, &(&1 < h))
    qsort(lesser) ++ [h] ++ qsort(greater)
  end

  # An implementation of selection sort
  def selection_sort(list) when is_list(list), do: selection_sort(list, [])

  defp selection_sort([], sorted), do: sorted

  defp selection_sort(list, sorted) do
    max = Enum.max(list)
    selection_sort(List.delete(list, max), [max | sorted])
  end
end

defmodule SortingScenarios do
  @random_list 1..10000 |> Enum.to_list() |> Enum.shuffle()

  def bubble_sort() do
    Sort.bsort(@random_list)
  end

  def quick_sort() do
    Sort.qsort(@random_list)
  end

  def selection_sort() do
    Sort.selection_sort(@random_list)
  end
end

Benchee.run(
  %{
    "bubble sort" => &SortingScenarios.bubble_sort/0,
    "quick sort" => &SortingScenarios.quick_sort/0,
    "selection sort" => &SortingScenarios.selection_sort/0
  },
  memory_time: 5,
  reduction_time: 5
)

# Kino.nothing()