mapping

code-examples.md

Stripe uses this for encoding their deepObject-style form data:

https://github.com/stripe/stripe-node/blob/753b77a33e3a173b2fd6d59c80dd7be87b8bd141/lib/utils.js#L54

  stringifyRequestData: (data) => {
    return (
      qs
        .stringify(data, {
          serializeDate: (d) => Math.floor(d.getTime() / 1000),
        })
        // Don't use strict form encoding by changing the square bracket control
        // characters back to their literals. This is fine by the server, and
        // makes these parameter strings easier to read.
        .replace(/%5B/g, '[')
        .replace(/%5D/g, ']')
    );
  },

mapping.md

Example

ARGS

var args = {
  random: {
    fish: "SALMON",
    price: "999",
  },
  date_between: ["11111", "99999"],
  line_items: [
    {
      price: "100",
    },
    {
      price: "200",
    },
  ],
  expand: ["invoice", "price"],
};

var claims = {
  sub: "ACCOUNT_ID",
};

CONFIG

{
  service: "stripe",
  name: "rest:post",
  parameters: [
    {
      // overriding the default style
      in: "searchParams",
      name: "date_between",
      style: "pipeDelimited",
      mapping: [["get", { path: "args.date_between" }]],
    },
    {
      // demonstrating this is same as our current directive pipeline
      in: "searchParams",
      name: "dinner",
      mapping: [
        ["get", { path: "args.random.fish" }],
        ["toLowerCase", {}],
      ],
    },
    {
      // simple example
      in: "path",
      name: "account",
      mapping: [["get", { path: "claims.sub" }]],
    },
    {
      // using root indicator '$', a sort will place this first so we'll effectively extend (if value is an object) this with later values
      in: "form",
      name: "$",
      mapping: [["get", { path: "args.random" }]],
    },
    {
      // going to nest an object then extend it
      in: "form",
      name: "nested_and_extended",
      mapping: [["get", { path: "args.random" }]],
    },
    {
      // override explode and style settings, `nested_and_extended` will be extended rather than overwritten if it is an extendable type (Object)
      in: "form",
      name: "nested_and_extended",
      objectExtend: true,
      explode: false,
      style: "form",
      mapping: [["get", { path: "args.list_items[0]" }]],
    },
    {
      // array push notation, irrelevant here, but important on the next one
      in: "form",
      name: "line_items",
      mapping: [["get", { path: "args.line_items" }]],
    },
    {
      // array push option here prevents overwriting `line_items`. Returned array will be concatenated, anything else will be pushed. If the existing value is not an array it will be overwritten with the array output of this.
      in: "form",
      name: "line_items",
      arrayPush: true,
      mapping: [["get", { path: "args.random" }]],
    },
    {
      // this name syntax allows setting a property on each item in the array, this may not be a good idea (perhaps should be handled as a special directive)
      in: "form",
      name: "line_items[].coupon",
      mapping: [["set", { value: "COUPON_CODE" }]],
    },
    {
      // override default style and explode
      in: "form",
      name: "mode",
      style: "form",
      explode: false,
      value: ["subscription", "one-off"],
    },
    {
      // override style and nested object notation
      in: "form",
      name: "deep.path",
      style: "form",
      explode: false,
      value: "SPINDRIFT",
    },
    {
      // our existing Stripe example
      in: "form",
      name: "expand",
      mapping: [["get", { path: "args.expand" }]],
    },
    {
      // our existing Stripe example
      in: "form",
      name: "expand",
      arrayPush: true,
      value: "field_i_really_want",
    },
  ],
  parametersOptions: {
    form: {
      style: "deepObject",
      explode: true,
    },
    path: {
      style: "simple",
    },
    searchParams: {
      style: "deepObject",
      explode: true,
    },
  },
}

OUTPUT

POST /account/ACCOUNT_ID?date_between=11111|99999&dinner=salmon

fish=SALMON
price=999
nested_and_extended=fish,SALMON,price,100
line_items[0][price]=100
line_items[0][coupon]=COUPON_CODE
line_items[1][price]=200
line_items[1][coupon]=COUPON_CODE
line_items[2][fish]=SALMON
line_items[2][price]=999
line_items[2][coupon]=COUPON_CODE
mode=subscription,one-off
deep=path,SPINDRIFT
expand[]=invoice
expand[]=price
expand[]=field_i_really_want

What's new here

• parameters uses a new config format, moving away from tuples to arrays of config options
• parametersOptions has broad configuration options for how parameters are used. These pertain to serialization style and formatting. They are effectively defaults, which any individual parameter can override.
• The special parameter name $ referring to the root of the hypothetical object representation of the parameter.
• value which allows you to directly set (in the schema) the value of a key
• mapping which evaluates an array of directives to get a value
• objectExtend and arrayPush as explicit operations — presumably these could be set as default behaviors via the parametersOptions.

Extended use case

With this new format it becomes possible to do batch operations, say we had an earlier resolver return a list and we want to iterate over a particular property.

{
  service: "stripe",
  name: "rest:batch:get",
  parameters: [
    {
      in: "searchParams",
      name: "expand",
      value: ["prices", "invoices"],
    },
    {
      // batchKey identifies this as the parameter to iterate over to make multiple requests, schema validation could enforce having only one of these
      in: "path",
      name: "product",
      batchKey: true,
      mapping: [
        [
          "jsonPath",
          { path: "steps.listCustomerProducts.data.items[*].productId" },
        ],
      ],
    },
  ],
}

{
  listCustomerProducts: {
    data: {
      items: [
        {
          productId: 123,
        },
        {
          productId: 789,
        },
      ],
    },
  },
}

GET /products/123?expand[]=prices&expand[]=invoices
GET /products/789?expand[]=prices&expand[]=invoices

What's good about it?

• Parity of capabilities with OpenAPI — set style and processing options param-by-param
• Much more descriptive
• Likely to be handled better by default code formatters
• Easier to set values directly in the schema
• Greater flexibility for future uses
• Examples above are very REST focused, but the same principles work for building and mapping args objects in: args
• Looks less like the object you're presumably outputting — I think this is good as the cases become more complex
• Likely a bit easier to iterate our project schemas with, we simply add new in enums if we have a new structure to build
• A fairly easy shift from current approach — all current mappings are fully expressible with a relatively simple transform

What's less good about it?

• Much more verbose, likely to make schemas longer
• Repetition of key names, like in and name feels tiresome

9/13 Proposal

var args = {
  random: {
    fish: "SALMON",
    price: "999",
  },
  date_between: ["11111", "99999"],
  line_items: [
    {
      price: "100",
    },
    {
      price: "200",
    },
  ],
  expand: ["invoice", "price"],
};

var claims = {
  sub: "ACCOUNT_ID",
};

A json patch inspired approach:

{
  service: "stripe",
  name: "rest:post",
  searchParams: {
    ops: [
      {
        op: "set", // the default, doesn't need to be specific
        path: "date_between",
        mapping: [["get", { path: "args.date_between" }]],
      },
      {
        path: "dinner",
        mapping: [
          ["get", { path: "args.random.fish" }],
          ["toLowerCase", {}],
        ],
      },
    ],
    serialize: {
      date_between: {
        style: "pipeDelimited",
      },
    },
  },
  pathParams: {
    ops: [
      {
        path: "account",
        mapping: [["get", { path: "claims.sub" }]],
      },
    ],
  },
  form: {
    ops: [
      {
        path: "$",
        mapping: [["get", { path: "args.random" }]],
      },
      {
        //  Example of the flexibility of the `ops` approach, we can extend a base object, then remove props we might not want
        op: "remove",
        path: "price",
        mapping: [["get", { path: "args.random" }]],
      },
      {
        // extend will coerce this to an object, e.g., if args.random = 'foo' `nested_and_extended: { foo: 'foo' }`
        op: "extend",
        path: "nested_and_extended",
        mapping: [["get", { path: "args.random" }]],
      },
      {
        // object coercion and then object extend — basically the op ensures an object in the output
        op: "extend",
        path: "nested_and_extended",
        mapping: [["get", { path: "args.list_items[0]" }]],
      },
      {
        // If we explicitly set the empty brackets, coerce any result to an array.
        path: "line_items[]",
        // OR, maybe we use these two and avoid custom syntax
        op: "push",
        path: "line_items",
        mapping: [["get", { path: "args.line_items" }]],
      },
      {
        /* This is how we handle the concat/push behavior, similar to:
         * { line_items: [...args.line_items, isArray(args.random) ? ...args.random : args.random] }
         **/
        path: "line_items[]",
        // OR
        op: "push",
        path: "line_items",
        mapping: [["get", { path: "args.random" }]],
      },
      {
        // Syntax, inspired by jsonpath-plus, for operating on every item in an array
        path: "line_items[*].coupon",
        // This one trips me up, as I can't think of a "boring" syntax for describing this
        mapping: [["set", { value: "COUPON_CODE" }]],
      },
      {
        path: "mode",
        value: ["subscription", "one-off"],
      },
      {
        path: "deep.path",
        value: "SPINDRIFT",
      },
      {
        path: "expand[]",
        mapping: [["get", { path: "args.expand" }]],
      },
      {
        path: "expand[]",
        value: "field_i_really_want",
      },
    ],
    serialize: {
      nested_and_extended: { explode: false, style: "form" },
      mode: { explode: false, style: "form" },
      deep: { explode: false, style: "form" },
      expand: { explode: true, style: "deepObject" },
    },
  },
}

OUTPUT

POST /account/ACCOUNT_ID?date_between=11111|99999&dinner=salmon

fish=SALMON
nested_and_extended=fish,SALMON,price,100
line_items[0][price]=100
line_items[0][coupon]=COUPON_CODE
line_items[1][price]=200
line_items[1][coupon]=COUPON_CODE
line_items[2][fish]=SALMON
line_items[2][price]=999
line_items[2][coupon]=COUPON_CODE
mode=subscription,one-off
deep=path,SPINDRIFT
expand[]=invoice
expand[]=price
expand[]=field_i_really_want

Final Proposal

{
  "service": "stripe",
  "name": "rest:post",
  "searchParams": {
    "ops": [
      {
        "path": "date_between",
        "mapping": "args.date_between"
      },
      {
        "path": "dinner",
        "mapping": [
          ["get", { "path": "args.random.fish" }],
          ["toLowerCase", {}]
        ]
      }
    ],
    "serialize": {
      "paths": {
        "date_between": {
          "style": "pipeDelimited"
        }
      }
    }
  },
  "pathParams": {
    "ops": [
      {
        "path": "account",
        "mapping": "claims.sub"
      }
    ]
  },
  "form": {
    "ops": [
      {
        "path": "$",
        "mapping": "args.random"
      },
      {
        "path": "price",
        "op": "remove",
        "mapping": "args.random"
      },
      {
        "path": "nested_and_extended",
        "op": "extend",
        "mapping": "args.random"
      },
      {
        "path": "nested_and_extended",
        "op": "extend",
        "mapping": "args.list_items[0]"
      },
      {
        "path": "line_items",
        "op": "concat",
        "mapping": "args.line_items"
      },
      {
        "path": "line_items",
        "op": "concat",
        "mapping": "args.random"
      },
      {
        "path": "line_items[*].coupon",
        "value": "COUPON_CODE"
      },
      {
        "path": "line_items[:3].bar",
        "value": "BAR"
      },
      {
        "path": "foo[-2:].bar",
        "value": "BAR"
      },
      {
        "path": "mode",
        "value": ["subscription", "one-off"]
      },
      {
        "path": "deep.path",
        "value": "SPINDRIFT"
      },
      {
        "path": "expand",
        "op": "concat",
        "mapping": "args.expand"
      },
      {
        "path": "expand",
        "op": "concat",
        "value": "field_i_really_want"
      }
    ],
    "serialize": {
      "defaults": {
        "style": "form",
        "explode": true
      },
      "paths": {
        "nested_and_extended": { "explode": false, "style": "form" },
        "mode": { "explode": false, "style": "form" },
        "deep": { "explode": false, "style": "form" },
        "expand": { "explode": true, "style": "deepObject" }
      }
    }
  }
}

schema.json

{
  "definitions": {
    "ops": {
      "type": "object",
      "properties": {
        "path": {
          "type": "string"
        },
        "op": {
          "type": "string",
          "enum": ["set", "push", "extend", "remove"]
        },
        "value": {
          "type":["number","string","boolean","object","array", "null"]
        }        
      }
    }
  },
  "schema": {
    "type": "object",
    "properties": {}
  }
}

scratch.js

var _ = require("lodash/fp")
// var res = _.set(obj, 'bar.baz bam.blah', 'BLAH')
// var arr = ['foo', 'bar']
// var final = _.concat(['foo'], ['bar'])

// var obj = {foo: [{bar: 'BAR'}, {bar: 'LALA'}]};


var paths = [
    'foo[...].bar',
    'foo[.].bar',
    'foo[**].bar',
    'foo[*].bar',
    'foo[:-3].bar',
    'foo[:3].bar',
    'foo[-1:].bar',
    'foo[5:].bar',
    'foo[1:2:2].bar',
    'foo[1,3].bar',
    'foo[3].bar',
    'foo[bar].baz'
];

var myArray = ['A', 'B', 'C', 'D', 'E', 'F', 'G'];

var getFromStart = (arr, to) => {
    return _.slice(0, to, arr);
}

var getFromEnd = (arr, to) => {
    return _.slice(to, arr.length, arr);
}

function parsePath(path) {
  // Match candidate non-empty brackets
  // const candidate = path.match(/\[([0-9:,-]+|[*]{1})\]/);
  const splitPathLeft = path.split('[');

  // No brackets present
  if (!splitPathLeft[1]) {
    return {path, isIterable: false};
  }

  const pathBefore = splitPathLeft[0];
  const splitPathRight = splitPathLeft[1].split(']');

  // No closing bracket
  if (!splitPathRight[1]) {
    return {path, isIterable: false};
  }

  const pathAfter = splitPathRight[1].replace(/^\./, '');
  const expr = splitPathRight[0];

  // All children, cheapest test
  if (expr === '*') {
    return {
      path,
      isIterable: true,
      pathBefore,
      pathAfter,
      isAll: true
    };
  }

  // Has no numbers
  if (!/\d/.test(expr)) {
    return {path, isIterable: false};
  }

  // Has only numbers
  if (/^\d+$/m.test(expr)) {
    return {path, isIterable: false};
  }

  // Pluck notation, [index, index, ..., index]
  if (/^[0-9]+(,[0-9]+)+$/m.test(expr)) {
    const pluck = expr.split(',').map(Number);
    return {path, isIterable: true, pathBefore, pathAfter, pluck};
  }

  // Python slice notation, [start, stop, step]
  var sliceMatch = /(-?\d*):(-?\d*):?(\d*)/.exec(expr);
  if (sliceMatch) {
    var [, start, stop, step] = sliceMatch;
    return {
      path,
      isIterable: true,
      pathBefore,
      pathAfter,
      start: Number(start),
      stop: Number(stop),
      step: Number(step)
    };
  }

  return {path, isIterable: false};
}

for (var p of paths) {
    console.log(p, parsePath(p));
}

// var str = 'foo[*].bar';
// var iteratorIndex = str.indexOf('[*]');

// if (iteratorIndex > -1) {
//     var pathBefore = str.slice(0, iteratorIndex);
//     var pathAfter = str.slice(iteratorIndex + 3);
//     var before = _.get(pathBefore, obj);
//     console.log({pathBefore, before, pathAfter});
// }

// console.log(_.get('foo[-1]', obj));

Also objectExtend: true and arrayPush: true feel like they should be the defaults or there should be an op: 'set' | 'push' | 'extend' property on each line and we'd change the set: [..] key to ops: [..]

Also for comparison here is the previously proposed tuple format but organized by param and serialization split out:

const resolverConfig = {
  "service": "stripe",
  "name": "rest:post",
  "searchParams": {
    "set": [
      ["date_between", ["get", {"path": "args.date_between"}]],
      ["dinner", [["get", {"path": "args.random.fish"}], ["toLowerCase", {}]]]
    ],
    "serialize": {
      "date_between": {"style": "pipeDelimited"}
    }
  },
  "pathParams": [
    ["account", ["get", {"path": "claims.sub"}]]
  ],
  "form": {
    "set": [
      ["...", ["get", {"path": "args.random"}]],
      ["nested_and_extended", ["get", {"path": "args.random"}]],
      ["nested_and_extended", ["get", {"path": "args.list_items[0]"}]],
      ["line_items", ["get", {"path": "args.line_items"}]],
      ["line_items", ["get", {"path": "args.random"}]],
      ["line_items[].coupon", ["set", {"value": "COUPON_CODE"}]],
      ["mode", ["set", {"value": ["subscription", "one-off"]}]],
      ["deep.path", ["set", "SPINDRIFT"]],
      ["expand", ["get", {"path": "args.expand"}]],
      ["expand", ["set", {"value": "field_i_really_want"}]],
    ],
    "serialize": {
      "nested_and_extended": {"explode": false, "style": "form"},
      "mode": {"explode": false, "style": "form"},
      "deep": {"explode": false, "style": "form"}
    }
  }
}

• pathParams do have serialization options, per OpenAPI: https://swagger.io/docs/specification/serialization/ — crazy as some of that is, I think we should support it

• The split out by section definitely looks more readable and maybe more easily validated, though it could lead to more schema churn. We'd need to support different prop names based on the capabilities of the resolver as opposed to adding to the enums of a single prop, e.g., in: 'form' | 'path' | 'searchParams' is added to, but we maintain a fairly generic / consistent parameters object. I have no strong opinion on this point, just highlighting.

• I would still avoid tuples (sadly); the object format at the param level is capturing detail you haven't brought into the tuples, like the ability to set extend / push behaviors without necessarily creating a new language conveyed through the keys. I also think it is visually a bit easier to handle.

• I think it makes sense to keep the keys as simple as possible, to avoid creating an ad-hoc language which is also challenging to validate... With your approach of having an explicit set child property, I wonder if it makes sense to support nesting of functionality:

var args = {
  foo: [{price: '100'}, {price: '200}],
  bar: 'BAR'
}

{
  "form": {
    "set": [
      {
        "name": "parent",
        "set": [
          {
            "name": "$",
            "mapping": [["get", {"path": "args.foo"}]]
          },
          {
            "name": "child",
            "mapping": [["get", {"path": "args.bar"}]]
          }
        ]
      }
    ]
  }
}

{
  parent: [{price: '100', child: 'BAR'}, {price: '200', child: 'BAR'}]
}

Also objectExtend: true and arrayPush: true feel like they should be the defaults or there should be an op: 'set' | 'push' | 'extend' property on each line and we'd change the set: [..] key to ops: [..]

I don't quite follow this. Could you flesh out an example?

Also objectExtend: true and arrayPush: true feel like they should be the defaults or there should be an op: 'set' | 'push' | 'extend' property on each line and we'd change the set: [..] key to ops: [..]

I don't quite follow this. Could you flesh out an example?

const resolverConfig = {
  "service": "stripe",
  "name": "rest:post",
  "searchParams": {
    "ops": [
      {
        "name": "date_between",
        "mapping": [
          ["get", {"path": "args.date_between"}]
        ]
      },
      {
        "name": "dinner",
        "mapping": [
          ["get", {"path": "args.random.fish"}],
          ["toLowerCase", {}]
        ]
      }
    ],
    "serialize": {
      "date_between": {"style": "pipeDelimited"}
    }
  },
  "pathParams": [
    { // simple example
      "name": "account",
      "mapping": [
        ["get", {"path": "claims.sub"}]
      ]
    }
  ],
  "form": {
    "ops": [
      { // going to nest an object then extend it
        "name": "nested_and_extended",
        "mapping": [
          ["get", {"path": "args.random"}]
        ]
      },
      { // override explode and style settings, `nested_and_extended` will be extended rather than overwritten if it is an extendable type (Object)
        "name": "nested_and_extended",
        "op": "extend",
        "mapping": [
          ["get", {"path": "args.list_items[0]"}]
        ]
      },
      { // array push notation, irrelevant here, but important on the next one
        "name": "line_items",
        "mapping": [
          ["get", {"path": "args.line_items"}]
        ]
      },
      { // array push option here prevents overwriting `line_items`. Returned array will be concatenated, anything else will be pushed. If the existing value is not an array it will be overwritten with the array output of this.
        "name": "line_items",
        "op": "push",
        "mapping": [
          ["get", {"path": "args.random"}]
        ]
      },
      { // this name syntax allows setting a property on each item in the array, this may not be a good idea (perhaps should be handled as a special directive)
        "name": "line_items[].coupon",
        "mapping": [
          ["set", {"value": "COUPON_CODE"}]
        ]
      },
      { // override default style and explode
        "name": "mode",
        "value": ["subscription", "one-off"]
      },
      { // override style and nested object notation
        "name": "deep.path",
        "value": "SPINDRIFT"
      },
      { // our existing Stripe example
        "name": "expand",
        "mapping": [
          ["get", {"path": "args.expand"}]
        ]
      },
      { // our existing Stripe example
        "name": "expand",
        "op": "push",
        "value": "field_i_really_want"
      }
    ],
    "serialize": {
      "nested_and_extended": {"explode": false, "style": "form"},
      "mode": {"explode": false, "style": "form"},
      "deep": {"explode": false, "style": "form"}
    }
  }
}

I see, so like this: http://jsonpatch.com

I think considering the problem as similar to a json patch might be a good direction