Script to convert AWS Data Pipeline template to Cloudformation template with a sample Terraform module

converter-cli.js

const fs = require('fs');
const Converter = require('./converter.js');

const options = {
  name: process.argv[3],
  description: process.argv[4],
  activate: process.argv[5],
};
const converter = new Converter(JSON.parse(fs.readFileSync(process.argv[2])), options);

process.stdout.write(JSON.stringify(converter.cloudformation()));

converter.js

const parseField = (key, value) => {
  const formationField = { "Key": key };

  if (typeof value === 'string') {
    formationField.StringValue = value;
  } else if (value.ref) {
    formationField.RefValue = value.ref;
  } else {
    throw new Error('Field type is not supported');
  }

  return formationField;
};

const transformObject = (toTransform) => {
  const object = Object.assign({}, toTransform);
  if (object.id === 'Default' && object.name === 'Default' && !object.type) {
    object.type = 'Default';
  }

  const keys = Object.keys(object);
  const isArray = key => object[key] instanceof Array;

  const targetKeys = keys
    .filter(key => key !== 'id' && key !== 'name');

  const simpleFields = targetKeys
    .filter(key => !isArray(key))
    .map(key => parseField(key, object[key]));

  const arrayFields = targetKeys
    .filter(isArray)
    .reduce((res, key) => {
      const keyValues = object[key].map(value => parseField(key, value));
      return res.concat(keyValues);
    }, []);

  return {
    "Id": object.id,
    "Name": object.name,
    "Fields": simpleFields.concat(arrayFields),
  };
};

const transformParameter = parameter => ({
  "Id": parameter.id,
  "Attributes": Object.keys(parameter)
    .filter(key => key !== 'id')
    .map(key => ({
      "Key": key,
      "StringValue": parameter[key],
    })),
});

const normalizeName = name => name
  .replace(/[^a-z0-9]/gi, ' ')
  .replace(/\b\w/g, l => l.toUpperCase())
  .replace(/ /g, '');

class Converter {
  constructor(template, options) {
    this.template = template;
    this.options = Object.assign({}, options);
  }

  _values(values) {
    const obj = Object.assign({}, values, this.options.values);
    return Object.keys(obj)
      .map(key => ({ "Id": key, "StringValue": obj[key] }));
  }

  cloudformation() {
    const result = {
      "AWSTemplateFormatVersion": "2010-09-09",
      "Description": "Data Pipeline Template",
      "Resources": {},
    };

    result.Resources[normalizeName(this.options.name)] = {
      "Type": "AWS::DataPipeline::Pipeline",
      "Properties": {
        "Activate": this.options.activate,
        "Description": this.options.description,
        "Name": this.options.name,
        "PipelineObjects": this.template.objects.map(transformObject),
        "ParameterObjects": this.template.parameters.map(transformParameter),
        "ParameterValues": this._values(this.template.values),
      },
    };
    return result;
  }
}

module.exports = Converter;

main.tf

variable "name" {}
variable "description" {}
variable "activate" {}
variable "template" {}
variable "values" {
  type = "map"
  default = {}
}

data "external" "converted_template" {
  program = ["node", "${path.module}/terraform-data.js"]
  query = {
    template = "${var.template}",
    name = "${var.name}",
    description = "${var.description}",
    activate = "${var.activate}",
    values = "${jsonencode("${var.values}")}"
  }
}

resource "aws_cloudformation_stack" "data_pipeline" {
  name = "data-pipeline-${var.name}-stack",
  template_body = "${data.external.converted_template.result.json}",
  on_failure = "DELETE"
}

terraform-data.js

const readline = require('readline');
const Converter = require('./converter.js');

const rl = readline.createInterface({
  input: process.stdin,
  output: process.stdout,
  terminal: false,
});

rl.on('line', (data) => {
  const options = JSON.parse(data);
  options.activate = (!!options.activate).toString();
  options.values = JSON.parse(options.values);

  const converter = new Converter(JSON.parse(options.template), options);
  const template = JSON.stringify(converter.cloudformation());
  process.stdout.write(JSON.stringify({ json: template }));
});

Thanks Artem, you saved my day!

I have been using this for quite a while. Really useful. Thx!