Oban batching slow test example

oban_test_example.ex

# my_app.ex
defmodule MyApp do
  alias MyApp.Jobs.TestJob

  def test_work(use_batch?) do
    1..100
    |> Enum.map(fn i -> %{job_number: i} end)
    |> then(fn args_list ->
      if use_batch? do
        TestJob.new_batch(args_list, batch_callback_args: %{batch: :my_batch})
      else
        Enum.map(args_list, &TestJob.new/1)
      end
    end)
    |> Oban.insert_all()
  end
end


# my_app/jobs/test_job.ex
defmodule MyApp.Jobs.TestJob do
  use Oban.Pro.Workers.Batch

  @impl Oban.Pro.Worker
  def process(%Job{args: %{"job_number" => _job_number}}) do
    # Quick queries to the database, processing etc
    Process.sleep(25)

    :ok
  end

  @impl Batch
  def handle_completed(%Job{meta: %{"batch_id" => batch_id}}) do
    {:ok, batch_id}
  end
end


# my_app_test.exs
defmodule MyAppTest do
  use MyApp.DataCase, async: true

  require Logger

  test "uses batch and goes slow" do
    start = DateTime.utc_now()

    MyApp.test_work(true)

    diff = DateTime.diff(DateTime.utc_now(), start)
    Logger.warn("Took #{diff} seconds to process jobs using batch")
  end

  test "doesn't use batch and goes fast" do
    start = DateTime.utc_now()

    MyApp.test_work(false)

    diff = DateTime.diff(DateTime.utc_now(), start)
    Logger.warn("Took #{diff} seconds to process jobs inline")
  end
end

This is the result of running mix test:

Here's a modification of the test that allowed me to run it in the Pro repo directly (there aren't any changes to MyApp or TestJob):

defmodule Oban.Pro.Workers.BatchSpeedTest do
  use Oban.Pro.Case, async: true

  # my_app.ex
  defmodule MyApp do
    alias MyApp.Jobs.TestJob

    def test_work(use_batch?) do
      1..100
      |> Enum.map(fn i -> %{job_number: i} end)
      |> then(fn args_list ->
        if use_batch? do
          TestJob.new_batch(args_list, batch_callback_args: %{batch: :my_batch})
        else
          Enum.map(args_list, &TestJob.new/1)
        end
      end)
      |> Oban.insert_all()
    end
  end

  # my_app/jobs/test_job.ex
  defmodule MyApp.Jobs.TestJob do
    use Oban.Pro.Workers.Batch

    @impl Oban.Pro.Worker
    def process(%Job{args: %{"job_number" => _job_number}}) do
      # Quick queries to the database, processing etc
      Process.sleep(25)

      :ok
    end

    @impl Batch
    def handle_completed(%Job{meta: %{"batch_id" => batch_id}}) do
      {:ok, batch_id}
    end
  end

  require Logger

  setup do
    start_supervised_oban!(
      name: Oban,
      testing: :disabled,
      queues: [default: 10],
      stage_interval: 10
    )

    :ok
  end

  test "uses batch and goes slow" do
    {time, _} = :timer.tc(fn -> MyApp.test_work(true) end)

    await_complete()

    Logger.warning("Took #{time} microseconds to insert jobs using batch")

    stop_supervised!(Oban)
  end

  test "doesn't use batch and goes fast" do
    {time, _} = :timer.tc(fn -> MyApp.test_work(false) end)

    await_complete()

    Logger.warning("Took #{time} microseconds to insert jobs inline")

    stop_supervised!(Oban)
  end

  defp await_complete do
    with_backoff(fn ->
      assert ~w(completed) ==
               Oban.Job
               |> Repo.all()
               |> Enum.map(& &1.state)
               |> Enum.uniq()
    end)
  end
end

That inserts the jobs and waits for them all to complete (you should use a :manual testing mode and drain_jobs or run_batch to do this in testing, but I digress).

Here's the output showing that inline takes ~14ms and the batch takes ~44ms (waiting for the callback to trigger):

Something seems off with your database or application setup.