Created
July 8, 2016 03:57
-
-
Save portante/2eb4268aba95da8a08918e0fbf265111 to your computer and use it in GitHub Desktop.
This is a simple script which creates index aliases for index names of the pattern: "<prefix>[.name].(sar|sosreport)-*", where the "name" is optional. We have daily indices for *.sar-* indices, and monthly indices for *.sosreport-* indices. This will create aliases for various relative groupings based on the table.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| # | |
| # es-create-index-aliases | |
| # | |
| # Create the index aliases, making it easier to reference groupings of the | |
| # time series data found there-in. | |
| # | |
| from __future__ import print_function | |
| import os, sys | |
| DEBUG = False | |
| try: | |
| indextype = sys.argv[1] | |
| except IndexError: | |
| print("Need an index type, either sar or sosreport", file=sys.stderr) | |
| sys.exit(1) | |
| else: | |
| if indextype not in ('sar', 'sosreport'): | |
| print("Invalied index type %s, specify either 'sar' or 'sosreport'" % (indextype,), file=sys.stderr) | |
| sys.exit(1) | |
| try: | |
| indexname = sys.argv[2] | |
| except IndexError: | |
| indexname = '' | |
| indextype_alias_sets = { | |
| 'sosreport': [ | |
| (3, '3month'), | |
| (6, '6month'), | |
| (9, '9month'), | |
| (12, '1year') | |
| ], | |
| 'sar': [ | |
| (7, '1week'), | |
| (14, '2week'), | |
| (28, '4week'), | |
| (56, '8week'), | |
| (3 * 28, '3month'), | |
| (6 * 28, '6month'), | |
| (9 * 28, '9month'), | |
| (365, '1year') | |
| ] | |
| } | |
| cfg_name = os.environ.get('ES_CONFIG_PATH') | |
| if cfg_name is None: | |
| print("Need ES_CONFIG_PATH environment variable defined", | |
| file=sys.stderr) | |
| sys.exit(1) | |
| import configparser | |
| config = configparser.ConfigParser() | |
| config.read(cfg_name) | |
| try: | |
| URL = config.get('Server', 'server') | |
| except configparser.NoSectionError: | |
| print("Need a [Server] section with host and port defined in %s" | |
| " configuration file" % cfg_name, file=sys.stderr) | |
| sys.exit(1) | |
| except configparser.NoOptionError: | |
| host = config.get('Server', 'host') | |
| port = config.get('Server', 'port') | |
| else: | |
| host, port = URL.rsplit(':', 1) | |
| hosts = [dict(host=host, port=port),] | |
| INDEX_PREFIX = config.get('Settings', 'index_prefix') | |
| # Silence logging messages from the elasticsearch client library | |
| import logging | |
| try: | |
| from logging import NullHandler | |
| except ImportError: | |
| class NullHandler(logging.Handler): | |
| def handle(self, record): | |
| pass | |
| def emit(self, record): | |
| pass | |
| def createLock(self): | |
| self.lock = None | |
| logging.getLogger('elasticsearch').addHandler(NullHandler()) | |
| from elasticsearch import Elasticsearch | |
| es = Elasticsearch(hosts) | |
| from datetime import datetime | |
| now = datetime.utcnow() | |
| def _create_alias(tname, tindex, doraise=False): | |
| try: | |
| es.indices.update_alias(index=tindex, name=tname) | |
| except Exception as err: | |
| if doraise: | |
| raise | |
| else: | |
| print(repr(err), file=sys.stderr) | |
| sys.exit(1) | |
| else: | |
| print("Created alias %s for %r" % (tname, tindex)) | |
| if indexname: | |
| INDEX_PREFIX += '.%s' % indexname | |
| index_pat = '%s.%s-*' % (INDEX_PREFIX, indextype,) | |
| try: | |
| res = es.indices.get_aliases(index_pat) | |
| except Exception as err: | |
| print(repr(err), file=sys.stderr) | |
| sys.exit(1) | |
| else: | |
| if not res: | |
| print("Nothing to do") | |
| sys.exit(0) | |
| index_list = list(res.keys()) | |
| index_list.sort(reverse=True) | |
| if DEBUG: | |
| import json | |
| print("Existing: ") | |
| print(json.dumps(res, indent=2, sort_keys=True)) | |
| proposed_removes = {} | |
| removes = {} | |
| adds = {} | |
| # Deconstruct the existing set of aliases, assuming that all aliases will get | |
| # removed and re-applied to an entirely new set of indexes. If later we find | |
| # we are adding an alias to an index that is in the remove set, we leave it as | |
| # is (deleting it from the remove set and not inserting it in the add set). | |
| for index in res.keys(): | |
| try: | |
| aliases = res[index]["aliases"].keys() | |
| except KeyError: | |
| continue | |
| for alias in aliases: | |
| try: | |
| removes[index][alias] = res[index]["aliases"][alias] | |
| except KeyError: | |
| removes[index] = { alias: res[index]["aliases"][alias] } | |
| def conditional_add(alias, index): | |
| global removes | |
| global adds | |
| found = False | |
| for idx in removes.keys(): | |
| if alias in removes[idx] and index == idx: | |
| found = True | |
| if found: | |
| # We found that this alias was already set and scheduled to be | |
| # removed, so just drop it from the remove set | |
| del removes[index][alias] | |
| if not removes[index]: | |
| del removes[index] | |
| else: | |
| try: | |
| adds[index][alias] = {} | |
| except KeyError: | |
| adds[index] = { alias: {} } | |
| # Reconstruct the aliases based on what currently exists | |
| latest_index = index_list[0] | |
| latest_alias = '%s.%s-latest' % (INDEX_PREFIX, indextype) | |
| conditional_add(latest_alias, latest_index) | |
| all_alias = '%s.%s' % (INDEX_PREFIX, indextype) | |
| for index in index_list: | |
| conditional_add(all_alias, index) | |
| for number, suffix in indextype_alias_sets[indextype]: | |
| sub_alias = '%s.%s-%s' % (INDEX_PREFIX, indextype, suffix) | |
| sub_index_list = index_list[:number] | |
| for index in sub_index_list: | |
| conditional_add(sub_alias, index) | |
| if DEBUG: | |
| print("Removes:") | |
| print(json.dumps(removes, indent=2, sort_keys=True)) | |
| print("Adds:") | |
| print(json.dumps(adds, indent=2, sort_keys=True)) | |
| # Now we have a set of things to remove and those to be added, so merge them | |
| # together in one set of actions. | |
| actions = [] | |
| for idx in removes.keys(): | |
| for ali in removes[idx].keys(): | |
| actions.append({ 'remove': { 'alias': ali, 'index': idx } }) | |
| for idx in adds.keys(): | |
| for ali in adds[idx].keys(): | |
| actions.append({ 'add': { 'alias': ali, 'index': idx } }) | |
| if not actions: | |
| print("Aliases all up-to-date") | |
| sys.exit(0) | |
| if DEBUG: | |
| print("Actions:") | |
| print(json.dumps(actions, indent=2, sort_keys=True)) | |
| try: | |
| res = es.indices.update_aliases(body = { "actions": actions }) | |
| except Exception as err: | |
| print(repr(err), file=sys.stderr) | |
| sys.exit(1) | |
| else: | |
| sys.exit(0) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment