Import .kibana from 6.1 to 6.8

migration68.py

import json
import os
import subprocess
import re

# TARGET ES
ENV = 'localhost:9200'
# CREDENTIALS
USER = 'xxx'
PWD = 'xxx'
# PATH WHERE TO STORE AND MANIPULATE THE DATA
FOLDER_PATH = '/home/kibana-migrations/'

# DO NOT MODIFY AFTER HERE

DUMP_MAPPING_TEMPLATE = '''
NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump \
  --input=https://{}:{}@{} \
  --input-index=.kibana \
  --output={} \
  --limit=1000 \
  --type=mapping
'''

DUMP_DATA_TEMPLATE = '''
NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump \
  --input=https://{}:{}@{} \
  --input-index=.kibana \
  --output={} \
  --limit=1000 \
  --type=data
'''

UPLOAD_MAPPING_TEMPLATE = '''
NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump \
  --input={} \
  --output=https://{}:{}@{} \
  --output-index=.kibana_old \
  --limit=1000 \
  --type=mapping
'''

UPLOAD_DATA_TEMPLATE = '''
NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump \
  --input={} \
  --output=https://{}:{}@{} \
  --output-index=.kibana_old \
  --limit=1000 \
  --type=data
'''

DELETE_INDEX = '''
curl -XDELETE https://{}:{}@{}/.kibana -k
'''

SET_ALIAS_TEMPLATE = '''
curl -XPOST https://{}:{}@{}/_aliases -d '{{"actions":[{{"add":{{"index":".kibana_old","alias":".kibana"}}}}]}}' -H 'Content-Type: application/json' -k
'''

SHOW_ALIASES = '''
curl -XGET https://{}:{}@{}/_aliases?pretty -k
'''


def call(cmd):
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    for line in p.stdout.readlines():
        print(line),
    retval = p.wait()


def dump(mapping_file_path, data_file_path):
    cmd_dump_mapping = DUMP_MAPPING_TEMPLATE.format(USER, PWD, ENV, mapping_file_path)
    call(cmd_dump_mapping)

    cmd_dump_data = DUMP_DATA_TEMPLATE.format(USER, PWD, ENV, data_file_path)
    call(cmd_dump_data)


def remove_release_date(file_path, reg_exp, fallback_reg_exp=None):
    lines = []
    new_file_path = file_path.replace('.json', '_processed.json')
    with open(file_path, 'r') as f:
        content = f.readlines()
        for line in content:
            line_proc = line
            if '\"release_date' in line:
                line_proc = re.sub(reg_exp, '', line_proc)
                if '\"release_date' in line_proc and fallback_reg_exp:
                    line_proc = re.sub(fallback_reg_exp, '{', line_proc)
                if '\"release_date' in line_proc:
                    raise Exception

                if 'mapping' in file_path:
                    line_proc = line_proc.replace('{"doc":{"properties"', '{"doc":{"dynamic":"strict","properties"')
                json.loads(line_proc)
            lines.append(line_proc)

    with open(new_file_path, 'w') as f:
        for line in lines:
            f.write(line)

    return new_file_path


def main():
    """The script does the following:
    - 1) create a local directory in your machine
    - 2) dump in there the mappings and data from a target Kibana instance
    - 3) process the files downloaded to remove the release_date and set the mappings to strict
    - 4) print a set of commands to:
      - a) delete the .kibana index
      - b) upload the new mappings and data to the index .kibana_old in the target Kibana instance
      - c) set the alias .kibana to the index .kibana_old
      - d) check that the alias is set correctly

    The workflow is the following:
    - run the script
    - check that the mappings and data have been correctly downloaded
    - delete the current .kibana with 4a
    - upload the .kibana_old generated with 4b
    - set the alias .kibana to .kibana_old with 4c
    - check that the alias is set correctly with 4d
    - switch off the Kibana
    - upgrade to Kibana 6.8

    :return:
    """
    if os.path.isdir(FOLDER_PATH):
        print("directory exists")
        return

    os.mkdir(FOLDER_PATH)

    mapping_file_path = FOLDER_PATH + 'kibana_mapping.json'
    data_file_path = FOLDER_PATH + 'kibana.json'

    dump(mapping_file_path, data_file_path)
    new_mapping_file_path = remove_release_date(mapping_file_path, ',\"release_date\":\{\"type\".\"date\"\}')
    new_data_file_path = remove_release_date(data_file_path, ',\"release_date\":\".*\.[0-9][0-9][0-9]*\"',
                                             fallback_reg_exp='\{\"release_date\":\".*\.[0-9][0-9][0-9]*\",')

    print(DELETE_INDEX.format(USER, PWD, ENV))

    cmd_upload_mapping = UPLOAD_MAPPING_TEMPLATE.format(new_mapping_file_path, USER, PWD, ENV)
    print(cmd_upload_mapping)

    cmd_dump_data = UPLOAD_DATA_TEMPLATE.format(new_data_file_path, USER, PWD, ENV)
    print(cmd_dump_data)

    print(SET_ALIAS_TEMPLATE.format(USER, PWD, ENV))

    print(SHOW_ALIASES.format(USER, PWD, ENV))


if __name__ == '__main__':
    main()