Use jinja2 templates to generate multiple terraform providers in a loop

.gitignore

# Doit database files
.doit.db.*

# Terraform files created from templates
_*.tf

dodo.py

from jinja2 import Environment, FileSystemLoader
from pathlib import Path


DEFAULT_REGION = "us-east-1"
REGIONS = [
    "ap-northeast-1",
    "ap-northeast-2",
    "ap-northeast-3",
    "ap-south-1",
    "ap-southeast-1",
    "ap-southeast-2",
    "ca-central-1",
    "eu-central-1",
    "eu-north-1",
    "eu-west-1",
    "eu-west-2",
    "eu-west-3",
    "sa-east-1",
    "us-east-1",
    "us-east-2",
    "us-west-1",
    "us-west-2",
]


def task_render():
    """ "Render jinja2 templates.

    Yields:
        Render task for each jinja2 template file.

    """
    cwd = Path(".")
    environment = Environment(loader=FileSystemLoader(cwd))
    j2_templates = cwd.glob("*.j2")

    def render_template(template_path):
        """Render template.

        Args:
            template_path: Path to jinja2 template file

        """
        template = environment.get_template(str(template_path))
        with open(f"_{template_path.stem}", "w") as output_file:
            output_file.write(template.render(default_region=DEFAULT_REGION, regions=REGIONS))

    for j2_template in j2_templates:
        target = f"_{j2_template.stem}"
        yield {
            "name": target,
            "actions": [
                (
                    render_template,
                    [j2_template],
                    {},
                ),
            ],
            "file_dep": [j2_template],
            "targets": [target],
            "clean": True,
        }

provider.tf.j2

{% for region in regions %}
provider "aws" {
{%- if region != default_region %}
  alias = "{{ region | replace("-", "_") }}"
{%- endif %}
  default_tags {
    tags = local.tags
  }
  region = "{{ region }}"
}
{% endfor %}

@nimblenitin In a python virtual environment, the dependencies could be installed with:
$ pip install doit jinja2

and the template could be rendered with:
$ doit

The _provider.tf output file would look as follows:

provider "aws" {
  alias = "ap_northeast_1"
  default_tags {
    tags = local.tags
  }
  region = "ap-northeast-1"
}

provider "aws" {
  alias = "ap_northeast_2"
  default_tags {
    tags = local.tags
  }
  region = "ap-northeast-2"
}

provider "aws" {
  alias = "ap_northeast_3"
  default_tags {
    tags = local.tags
  }
  region = "ap-northeast-3"
}

provider "aws" {
  alias = "ap_south_1"
  default_tags {
    tags = local.tags
  }
  region = "ap-south-1"
}

provider "aws" {
  alias = "ap_southeast_1"
  default_tags {
    tags = local.tags
  }
  region = "ap-southeast-1"
}

provider "aws" {
  alias = "ap_southeast_2"
  default_tags {
    tags = local.tags
  }
  region = "ap-southeast-2"
}

provider "aws" {
  alias = "ca_central_1"
  default_tags {
    tags = local.tags
  }
  region = "ca-central-1"
}

provider "aws" {
  alias = "eu_central_1"
  default_tags {
    tags = local.tags
  }
  region = "eu-central-1"
}

provider "aws" {
  alias = "eu_north_1"
  default_tags {
    tags = local.tags
  }
  region = "eu-north-1"
}

provider "aws" {
  alias = "eu_west_1"
  default_tags {
    tags = local.tags
  }
  region = "eu-west-1"
}

provider "aws" {
  alias = "eu_west_2"
  default_tags {
    tags = local.tags
  }
  region = "eu-west-2"
}

provider "aws" {
  alias = "eu_west_3"
  default_tags {
    tags = local.tags
  }
  region = "eu-west-3"
}

provider "aws" {
  alias = "sa_east_1"
  default_tags {
    tags = local.tags
  }
  region = "sa-east-1"
}

provider "aws" {
  default_tags {
    tags = local.tags
  }
  region = "us-east-1"
}

provider "aws" {
  alias = "us_east_2"
  default_tags {
    tags = local.tags
  }
  region = "us-east-2"
}

provider "aws" {
  alias = "us_west_1"
  default_tags {
    tags = local.tags
  }
  region = "us-west-1"
}

provider "aws" {
  alias = "us_west_2"
  default_tags {
    tags = local.tags
  }
  region = "us-west-2"
}

After that, you would be able to run terraform plan as usual.

Note that local.tags value is missing from the example code, but you can add your own or remove that from the jinja2 template if you prefer not to use any tag.

thanks @jcollado one more thing- I would be sharing this module as terraform registry to user, I would ideally be taking input like region from user and expect user to use to do terraform plan directly without running $ doit. Is that possible to just take input from user using the module and populate this template without user having to run the $ doit command and installing dependency?

@nimblenitin It might be possible to do some magic with a null resource and a local-exec provisioner. However, I believe that wouldn't be a cleaner solution because it probably requires running terraform twice (one to render the template and one to generate the plan with all the resources defined) and that would be more confusing that just running doit when the template is updated and then terraform.

Thanks appreciate your response.

Hey @jcollado I want to take input in the form of kind of dictionary with values- Profile, region and populate that in template, was trying but could not get anywhere with doit documentation. Below is what should populate. Can you please help with the modified dodo.py code for the same if possible? Also I did not understand why you have put this condition- if region != default_region. You could just mention the value of default_region in region list right?

{% for region in regions %}
provider "aws" {
{%- if region != default_region %}
alias = "{{ region | replace("-", "_") }}"
{%- endif %}
default_tags {
tags = local.tags
}
region = "{{ region }}"
profile = "{{ profiles }}"
}
{% endfor %}

hey @jcollado never mind I got it right. Appreciate you sharing the workaround :)

@jcollado so there is one more thing which I cannot get. I am going to be using a file as a data source for account_details below. Can you please tell me how I can make it work?

from jinja2 import Environment, FileSystemLoader
from pathlib import Path

account_details = [
    {'alias': 'MEMBER1', 'region': 'us-east-1', 'member_account_id': '0000000000', 'ccs_mem_account_id': '0000000000'},
    {'alias': 'MEMBER2', 'region': 'us-east-2', 'member_account_id': '0000000000', 'ccs_mem_account_id': '0000000000'}
]

def task_render():
    """ "Render jinja2 templates.
    Yields:
        Render task for each jinja2 template file.
    """
    cwd = Path(".")
    environment = Environment(loader=FileSystemLoader(cwd))
    j2_templates = cwd.glob("*.j2")

    def render_template(template_path):
        """Render template.
        Args:
            template_path: Path to jinja2 template file
        """
        template = environment.get_template(str(template_path))
        with open(f"{template_path.stem}", "w") as output_file:
            output_file.write(template.render(data=account_details))

    for j2_template in j2_templates:
        target = f"_{j2_template.stem}"
        yield {
            "name": target,
            "actions": [
                (
                    render_template,
                    [j2_template],
                    {},
                ),
            ],
            "file_dep": [j2_template],
            "targets": [target],
            "clean": True,
        }               render_template,
                    [j2_template],
                    {},
                ),
            ],
            "file_dep": [j2_template],
            "targets": [target],
            "clean": True,
        }

@nimblenitin It depends on the content of your template, but if you expect account_details to be available in the template file, you need to pass it to the template.render method. One way to do that would be as follows:

template.render(account_details=account_details)

Thanks. Will try it out