backup consul kv store with terraform (used for upgrading consul versions that didn't have backups)

copy_kv.sh

import subprocess
import json
import base64
import os

# Gather all key values
keys = json.loads(subprocess.check_output(['curl', '-sL', 'http://localhost:8500/v1/kv/?keys']))

# Map them to their values
kvs = {}
for k in keys:
    vals = json.loads(subprocess.check_output(['curl', '-sL', 'http://localhost:8500/v1/kv/{0}'.format(k)]))
    for v in vals:
        kvs[k] = base64.decodestring(v.get('Value') or '')

# Creates a consul_keys data source
data_out = ""
for k in kvs:
  name = os.path.basename(k.rstrip('\/'))
  data_out += """
  key {{
    name = "{}"
    path = "{}"
  }}""".format(name, k)

data_out = """
data "consul_keys" "get" {{{}
}}
""".format(data_out)

# Creates a consul_keys resource
resource_out = ""
for k in kvs:
  resource_out += """
  key {{
    path = "{}"
    value = {}
  }}""".format(k, json.dumps(kvs[k]))

resource_out = """
resource "consul_keys" "set" {{
  count = "${{var.set}}"{}
}}
""".format(resource_out)

# Creates a null_resource for tracking whether kv has changed
null_out = ""
for k in kvs:
  name = os.path.basename(k.rstrip('\/'))
  null_out += """
    {0} = "${{data.consul_keys.get.var.{0}}}" """.format(
  name)

null_out = """resource "null_resource" "null" {{
  triggers {{{}
  }}
}}""".format(null_out)

# Prints to stdout, so you can use output as a main.tf file
print("""variable "set" {
  default = 0
}""")
print(data_out)
print(resource_out)
print(null_out)

This is a dead simple, one-off python script with no external dependencies that should be compatible with servers running Python 2.7. It can be used to backup your consul KV store for consul servers older than 0.7.0 (haven't tested on newer yet), so you have confidence around not losing your KV store through some accidental mishap. Just run like this:

python copy_kv.py > main.tf
terraform apply

Then upgrade your server. If something goes wrong, run this:

terraform apply -var set=1

NOTE: Don't forget to configure your consul provider somewhere

# provider.tf
provider "consul" {
  address    = "consul-server.example.com:8500"
  datacenter = "dc1"
}