Benchmark various GraphQL resolving variants

README.md

Benchmark various GraphQL resolving variants

There exists confusion about how a GraphQL execution engine like graphql-js does its work. The engine will invoke a resolve function for every [scalar] field in a request.

• defaultFieldResolver: By default graphql-js will invoke its own defaultFieldResolver resolve function when no explicit resolver was specified when generating the executable schema. This resolver needs to do some reflection on the request in order to determine what data to return.
• explicitDefaultFieldResolver: We can also use graphql-js's defaultFieldResolver explicitly, which will skip a check for whether or not a field resolver function was specified when generating the executable schema.
• customFieldResolver: We can also specify our own field resolver function that does not need to perform any reflection, as we know what type to return.
• customAsyncFieldResolver: Custom field resolvers can also be async. This doesn't make their own execution time much worse, but scheduling follow-up work on the runloop inherently opens up the thread to perform other work in the meantime, meaning overall request time goes up some.
• defaultFieldResolverWithAsyncRootResolver: This is the variant we see most often in Teams, where all data is resolved in an async root-field resolver and then all nested data is handled through graphql-js' defaultFieldResolver.
• explicitDefaultFieldResolverWithAsyncRootResolver: This is included to show what adding a nested custom async field resolver to Teams' typicaly pattern would do with overall performance.

Run

npx https://gist.github.com/alloy/677dfbe5acbfd5696d5c6dd5df47f633

Results

defaultFieldResolver x 598,703 ops/sec ±0.88% (87 runs sampled), mean 1.67μs
explicitDefaultFieldResolver x 602,025 ops/sec ±0.24% (88 runs sampled), mean 1.66μs
defaultFieldResolverWithAsyncRootResolver x 468,960 ops/sec ±0.71% (90 runs sampled), mean 2.13μs
explicitDefaultFieldResolverWithAsyncRootResolver x 475,675 ops/sec ±0.70% (90 runs sampled), mean 2.10μs
customFieldResolver x 692,693 ops/sec ±0.26% (90 runs sampled), mean 1.44μs
customAsyncFieldResolver x 354,694 ops/sec ±2.16% (88 runs sampled), mean 2.82μs
Fastest is customFieldResolver

As can be seen, and logically deduced once understanding that fields are always resolved via function invocation, the nested synchronous customFieldResolver variant will technically always be fasted, as it does the least amount of work.

We can also see that doing async work is fastest when everything is resolved in the root-field. However, this throws out all benefits of GraphQL as it:

• introduces the problem where you may perform work that is not requested at all, and thus more costly; or
• introduces coupling of root-field resolvers to specific use-cases, and thus not allowing for fast product feature iteration.

More importantly

The overall overhead for any of these is negligible when considering the amount of work real-world resolvers perform. Real resolvers request data from databases, make HTTP calls, and other such things that take many multitudes of time. We can simulate that by passing a number of miliseconds of “work” that the resolvers should perform, for instance 30ms of work:

npx https://gist.github.com/alloy/677dfbe5acbfd5696d5c6dd5df47f633 30

…we get the following results:

defaultFieldResolver x 32.13 ops/sec ±0.15% (76 runs sampled), mean 31.13ms
explicitDefaultFieldResolver x 32.13 ops/sec ±0.17% (76 runs sampled), mean 31.13ms
defaultFieldResolverWithAsyncRootResolver x 32.14 ops/sec ±0.17% (76 runs sampled), mean 31.11ms
explicitDefaultFieldResolverWithAsyncRootResolver x 32.09 ops/sec ±0.29% (75 runs sampled), mean 31.16ms
customFieldResolver x 32.16 ops/sec ±0.16% (76 runs sampled), mean 31.10ms
customAsyncFieldResolver x 31.21 ops/sec ±0.20% (74 runs sampled), mean 32.05ms
Fastest is customFieldResolver,defaultFieldResolverWithAsyncRootResolver,defaultFieldResolver,explicitDefaultFieldResolver,explicitDefaultFieldResolverWithAsyncRootResolver

Note how the benchmark tool considers them pretty much equal in execution time ¯_(ツ)_/¯

index.js

#!/usr/bin/env node

const {
  GraphQLBoolean,
  GraphQLObjectType,
  GraphQLSchema,
  GraphQLString,
  defaultFieldResolver,
  execute,
  parse,
} = require("graphql");
const Benchmark = require("benchmark");
const deepEqual = require("deep-equal");
const invariant = require("invariant");

function delay(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function formatSeconds(seconds) {
  if (seconds < 0.001) {
    return (seconds * 1_000_000).toFixed(2) + "μs";
  } else if (seconds < 1) {
    return (seconds * 1_000).toFixed(2) + "ms";
  } else {
    return seconds + "s";
  }
}

function createSchema(explicitFieldResolver, rootFieldResolverAsyncWorkInMs) {
  return new GraphQLSchema({
    query: new GraphQLObjectType({
      name: "Query",
      fields: {
        conversation: {
          type: new GraphQLObjectType({
            name: "Conversation",
            fields: {
              id: { type: GraphQLString },
              isNewForUser: {
                type: GraphQLBoolean,
                resolve: explicitFieldResolver,
              },
            },
          }),
          resolve: () => {
            const conversation = {
              id: "1",
              isNewForUser: false,
            };
            return rootFieldResolverAsyncWorkInMs === undefined
              ? conversation
              : rootFieldResolverAsyncWorkInMs === 0
              ? Promise.resolve(conversation)
              : delay(rootFieldResolverAsyncWorkInMs).then(() => conversation);
          },
        },
      },
    }),
  });
}

async function main(workInMs) {
  const schemaVariants = {
    defaultFieldResolver: createSchema(undefined, workInMs),
    explicitDefaultFieldResolver: createSchema(defaultFieldResolver, workInMs),

    // These always make the root resolver async by defaulting to work=0
    defaultFieldResolverWithAsyncRootResolver: createSchema(
      undefined,
      workInMs || 0
    ),
    explicitDefaultFieldResolverWithAsyncRootResolver: createSchema(
      defaultFieldResolver,
      workInMs || 0
    ),

    customFieldResolver: createSchema(() => false, workInMs),
    customAsyncFieldResolver: createSchema(
      // work is split between root and nested field resolvers
      () =>
        workInMs
          ? delay(Math.floor(workInMs / 2)).then(() => false)
          : Promise.resolve(false),
      workInMs === undefined ? undefined : workInMs - Math.floor(workInMs / 2)
    ),
  };

  const document = parse(`
    query {
      conversation {
        id
        isNewForUser
      }
    }
  `);

  // Ensure all variants resolve to the same result
  await Promise.all(
    Object.keys(schemaVariants).map(async (variant) => {
      const result = await execute({
        schema: schemaVariants[variant],
        document,
      });
      invariant(
        deepEqual(result.data, {
          conversation: {
            id: "1",
            isNewForUser: false,
          },
        }),
        "Unexpected result:\n%s",
        JSON.stringify({ variant, result }, null, 2)
      );
    })
  );

  const suite = new Benchmark.Suite();

  Object.keys(schemaVariants).forEach((variant) => {
    suite.add(variant, {
      defer: true,
      fn: async (deferred) => {
        await execute({
          schema: schemaVariants[variant],
          document,
        });
        deferred.resolve();
      },
    });
  });

  suite
    .on("cycle", function (event) {
      console.log(
        String(event.target) +
          ", mean " +
          formatSeconds(event.target.stats.mean)
      );
    })
    .on("complete", function () {
      console.log("Fastest is " + this.filter("fastest").map("name"));
    })
    .run({ async: false });
}

main(process.argv[2]);

package.json

{
  "name": "graphql-field-resolvers",
  "author": "Eloy Durán <[email protected]>",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "bin": "./index.js",
  "scripts": {
    "start": "node index.js"
  },
  "license": "MIT",
  "dependencies": {
    "benchmark": "^2.1.4",
    "deep-equal": "^2.0.5",
    "graphql": "^16.5.0",
    "invariant": "^2.2.4"
  }
}