Skip to content

Instantly share code, notes, and snippets.

@portante
Created July 8, 2016 03:57
Show Gist options
  • Select an option

  • Save portante/2eb4268aba95da8a08918e0fbf265111 to your computer and use it in GitHub Desktop.

Select an option

Save portante/2eb4268aba95da8a08918e0fbf265111 to your computer and use it in GitHub Desktop.
This is a simple script which creates index aliases for index names of the pattern: "<prefix>[.name].(sar|sosreport)-*", where the "name" is optional. We have daily indices for *.sar-* indices, and monthly indices for *.sosreport-* indices. This will create aliases for various relative groupings based on the table.
#!/usr/bin/env python3
#
# es-create-index-aliases
#
# Create the index aliases, making it easier to reference groupings of the
# time series data found there-in.
#
from __future__ import print_function
import os, sys
DEBUG = False
try:
indextype = sys.argv[1]
except IndexError:
print("Need an index type, either sar or sosreport", file=sys.stderr)
sys.exit(1)
else:
if indextype not in ('sar', 'sosreport'):
print("Invalied index type %s, specify either 'sar' or 'sosreport'" % (indextype,), file=sys.stderr)
sys.exit(1)
try:
indexname = sys.argv[2]
except IndexError:
indexname = ''
indextype_alias_sets = {
'sosreport': [
(3, '3month'),
(6, '6month'),
(9, '9month'),
(12, '1year')
],
'sar': [
(7, '1week'),
(14, '2week'),
(28, '4week'),
(56, '8week'),
(3 * 28, '3month'),
(6 * 28, '6month'),
(9 * 28, '9month'),
(365, '1year')
]
}
cfg_name = os.environ.get('ES_CONFIG_PATH')
if cfg_name is None:
print("Need ES_CONFIG_PATH environment variable defined",
file=sys.stderr)
sys.exit(1)
import configparser
config = configparser.ConfigParser()
config.read(cfg_name)
try:
URL = config.get('Server', 'server')
except configparser.NoSectionError:
print("Need a [Server] section with host and port defined in %s"
" configuration file" % cfg_name, file=sys.stderr)
sys.exit(1)
except configparser.NoOptionError:
host = config.get('Server', 'host')
port = config.get('Server', 'port')
else:
host, port = URL.rsplit(':', 1)
hosts = [dict(host=host, port=port),]
INDEX_PREFIX = config.get('Settings', 'index_prefix')
# Silence logging messages from the elasticsearch client library
import logging
try:
from logging import NullHandler
except ImportError:
class NullHandler(logging.Handler):
def handle(self, record):
pass
def emit(self, record):
pass
def createLock(self):
self.lock = None
logging.getLogger('elasticsearch').addHandler(NullHandler())
from elasticsearch import Elasticsearch
es = Elasticsearch(hosts)
from datetime import datetime
now = datetime.utcnow()
def _create_alias(tname, tindex, doraise=False):
try:
es.indices.update_alias(index=tindex, name=tname)
except Exception as err:
if doraise:
raise
else:
print(repr(err), file=sys.stderr)
sys.exit(1)
else:
print("Created alias %s for %r" % (tname, tindex))
if indexname:
INDEX_PREFIX += '.%s' % indexname
index_pat = '%s.%s-*' % (INDEX_PREFIX, indextype,)
try:
res = es.indices.get_aliases(index_pat)
except Exception as err:
print(repr(err), file=sys.stderr)
sys.exit(1)
else:
if not res:
print("Nothing to do")
sys.exit(0)
index_list = list(res.keys())
index_list.sort(reverse=True)
if DEBUG:
import json
print("Existing: ")
print(json.dumps(res, indent=2, sort_keys=True))
proposed_removes = {}
removes = {}
adds = {}
# Deconstruct the existing set of aliases, assuming that all aliases will get
# removed and re-applied to an entirely new set of indexes. If later we find
# we are adding an alias to an index that is in the remove set, we leave it as
# is (deleting it from the remove set and not inserting it in the add set).
for index in res.keys():
try:
aliases = res[index]["aliases"].keys()
except KeyError:
continue
for alias in aliases:
try:
removes[index][alias] = res[index]["aliases"][alias]
except KeyError:
removes[index] = { alias: res[index]["aliases"][alias] }
def conditional_add(alias, index):
global removes
global adds
found = False
for idx in removes.keys():
if alias in removes[idx] and index == idx:
found = True
if found:
# We found that this alias was already set and scheduled to be
# removed, so just drop it from the remove set
del removes[index][alias]
if not removes[index]:
del removes[index]
else:
try:
adds[index][alias] = {}
except KeyError:
adds[index] = { alias: {} }
# Reconstruct the aliases based on what currently exists
latest_index = index_list[0]
latest_alias = '%s.%s-latest' % (INDEX_PREFIX, indextype)
conditional_add(latest_alias, latest_index)
all_alias = '%s.%s' % (INDEX_PREFIX, indextype)
for index in index_list:
conditional_add(all_alias, index)
for number, suffix in indextype_alias_sets[indextype]:
sub_alias = '%s.%s-%s' % (INDEX_PREFIX, indextype, suffix)
sub_index_list = index_list[:number]
for index in sub_index_list:
conditional_add(sub_alias, index)
if DEBUG:
print("Removes:")
print(json.dumps(removes, indent=2, sort_keys=True))
print("Adds:")
print(json.dumps(adds, indent=2, sort_keys=True))
# Now we have a set of things to remove and those to be added, so merge them
# together in one set of actions.
actions = []
for idx in removes.keys():
for ali in removes[idx].keys():
actions.append({ 'remove': { 'alias': ali, 'index': idx } })
for idx in adds.keys():
for ali in adds[idx].keys():
actions.append({ 'add': { 'alias': ali, 'index': idx } })
if not actions:
print("Aliases all up-to-date")
sys.exit(0)
if DEBUG:
print("Actions:")
print(json.dumps(actions, indent=2, sort_keys=True))
try:
res = es.indices.update_aliases(body = { "actions": actions })
except Exception as err:
print(repr(err), file=sys.stderr)
sys.exit(1)
else:
sys.exit(0)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment