Script to convert AWS Data Pipeline template to Cloudformation template with a sample Terraform module

converter-cli.js

const fs = require('fs');
const Converter = require('./converter.js');

const options = {
  name: process.argv[3],
  description: process.argv[4],
  activate: process.argv[5],
};
const converter = new Converter(JSON.parse(fs.readFileSync(process.argv[2])), options);

process.stdout.write(JSON.stringify(converter.cloudformation()));

converter.js

const parseField = (key, value) => {
  const formationField = { "Key": key };

  if (typeof value === 'string') {
    formationField.StringValue = value;
  } else if (value.ref) {
    formationField.RefValue = value.ref;
  } else {
    throw new Error('Field type is not supported');
  }

  return formationField;
};

const transformObject = (toTransform) => {
  const object = Object.assign({}, toTransform);
  if (object.id === 'Default' && object.name === 'Default' && !object.type) {
    object.type = 'Default';
  }

  const keys = Object.keys(object);
  const isArray = key => object[key] instanceof Array;

  const targetKeys = keys
    .filter(key => key !== 'id' && key !== 'name');

  const simpleFields = targetKeys
    .filter(key => !isArray(key))
    .map(key => parseField(key, object[key]));

  const arrayFields = targetKeys
    .filter(isArray)
    .reduce((res, key) => {
      const keyValues = object[key].map(value => parseField(key, value));
      return res.concat(keyValues);
    }, []);

  return {
    "Id": object.id,
    "Name": object.name,
    "Fields": simpleFields.concat(arrayFields),
  };
};

const transformParameter = parameter => ({
  "Id": parameter.id,
  "Attributes": Object.keys(parameter)
    .filter(key => key !== 'id')
    .map(key => ({
      "Key": key,
      "StringValue": parameter[key],
    })),
});

const normalizeName = name => name
  .replace(/[^a-z0-9]/gi, ' ')
  .replace(/\b\w/g, l => l.toUpperCase())
  .replace(/ /g, '');

class Converter {
  constructor(template, options) {
    this.template = template;
    this.options = Object.assign({}, options);
  }

  _values(values) {
    const obj = Object.assign({}, values, this.options.values);
    return Object.keys(obj)
      .map(key => ({ "Id": key, "StringValue": obj[key] }));
  }

  cloudformation() {
    const result = {
      "AWSTemplateFormatVersion": "2010-09-09",
      "Description": "Data Pipeline Template",
      "Resources": {},
    };

    result.Resources[normalizeName(this.options.name)] = {
      "Type": "AWS::DataPipeline::Pipeline",
      "Properties": {
        "Activate": this.options.activate,
        "Description": this.options.description,
        "Name": this.options.name,
        "PipelineObjects": this.template.objects.map(transformObject),
        "ParameterObjects": this.template.parameters.map(transformParameter),
        "ParameterValues": this._values(this.template.values),
      },
    };
    return result;
  }
}

module.exports = Converter;

main.tf

variable "name" {}
variable "description" {}
variable "activate" {}
variable "template" {}
variable "values" {
  type = "map"
  default = {}
}

data "external" "converted_template" {
  program = ["node", "${path.module}/terraform-data.js"]
  query = {
    template = "${var.template}",
    name = "${var.name}",
    description = "${var.description}",
    activate = "${var.activate}",
    values = "${jsonencode("${var.values}")}"
  }
}

resource "aws_cloudformation_stack" "data_pipeline" {
  name = "data-pipeline-${var.name}-stack",
  template_body = "${data.external.converted_template.result.json}",
  on_failure = "DELETE"
}

terraform-data.js

const readline = require('readline');
const Converter = require('./converter.js');

const rl = readline.createInterface({
  input: process.stdin,
  output: process.stdout,
  terminal: false,
});

rl.on('line', (data) => {
  const options = JSON.parse(data);
  options.activate = (!!options.activate).toString();
  options.values = JSON.parse(options.values);

  const converter = new Converter(JSON.parse(options.template), options);
  const template = JSON.stringify(converter.cloudformation());
  process.stdout.write(JSON.stringify({ json: template }));
});

Hi,
For your information there is an issue when we have backslashes in Data Pipeline JSON template. Finally to many backslashes ate added in CF template body and the step will fail.
Thx.

Thanks Artem, you saved my day!

I have been using this for quite a while. Really useful. Thx!