Skip to content

Instantly share code, notes, and snippets.

@julrich

julrich/init-and-rollover-test.sh

Last active November 10, 2016 12:46
Show Gist options
  • Save julrich/7dcc96797e3931ceca55e639fa3229ae to your computer and use it in GitHub Desktop.
Save julrich/7dcc96797e3931ceca55e639fa3229ae to your computer and use it in GitHub Desktop.
Download ZIP
Index Rollover Test
Raw
gistfile1.txt
Simple test case to create an index, and test the rollover api.
Should move shards to single node before starting the shrink. Unfortunately, the settings seem to be updated correctly for the index, but no relocation is taking place.
Raw
init-and-rollover-test.sh
#!/usr/bin/env bash
DEBUG=true;
echo "RESETTINGS INDICES AND TEMPLATES:";
curl --silent -XDELETE "http://elasticsearch.services.ruhmesmeile.local:9200/_all" | jq .;
curl --silent -XDELETE "http://elasticsearch.services.ruhmesmeile.local:9200/_template/*" | jq .;
echo "";
echo "";
if [ "$DEBUG" = true ] ; then
echo "FOLLOWING LINE SHOULD BE EMPTY (NO SHARDS IN CLUSTER)";
curl --silent -XGET "http://elasticsearch.services.ruhmesmeile.local:9200/_cat/shards"
echo "";
fi
echo "RE-ADDING INDICES AND TEMPLATES:";
curl --silent -XPUT "http://elasticsearch.services.ruhmesmeile.local:9200/_template/hot-rmlogs" -d'
{
"template": "hot-rmlogs-*",
"settings": {
"number_of_shards": 5,
"number_of_replicas": 1,
"routing.allocation.include.box_type": "hot",
"routing.allocation.total_shards_per_node": 2
},
"aliases": {
"hot-rmlogs": {},
"search-rmlogs": {}
}
}' | jq .;
curl --silent -XPUT "http://elasticsearch.services.ruhmesmeile.local:9200/%3Chot-rmlogs-%7Bnow%2Fd%7D-000001%3E" | jq .;
echo "";
if [ "$DEBUG" = true ] ; then
echo "FRESH INDEX SETTINGS:";
curl --silent -XGET "http://elasticsearch.services.ruhmesmeile.local:9200/%3Chot-rmlogs-%7Bnow%2Fd%7D-000001%3E" | jq .;
echo "";
fi
echo "CREATING DOCUMENTS:";
for num in {1..100}
do
curl --silent -XPUT "http://elasticsearch.services.ruhmesmeile.local:9200/hot-rmlogs/logs/$num" -d'
{
"user" : "kimchy",
"post_date" : "2009-11-15T14:12:12",
"message" : "trying out Elasticsearch"
}' | jq .result;
done
if [ "$DEBUG" = true ] ; then
echo "INDICES AND SHARDS IN CLUSTER BEFORE ROLLOVER:";
curl --silent -XGET "http://elasticsearch.services.ruhmesmeile.local:9200/_cat/indices";
curl --silent -XGET "http://elasticsearch.services.ruhmesmeile.local:9200/_cat/shards";
echo "";
fi
ROLLOVER=$(curl --silent -XPOST "http://elasticsearch.services.ruhmesmeile.local:9200/hot-rmlogs/_rollover" -d'
{
"conditions": {
"max_docs": 100,
"max_age": "1m"
}
}');
# 12582912
if [ "$DEBUG" = true ] ; then
echo "ROLLOVER: $(echo "$ROLLOVER" | jq .)";
fi
ROLLED_OVER=$(echo $ROLLOVER | jq .rolled_over);
if [ "$DEBUG" = true ] ; then
echo "ROLLED_OVER: $ROLLED_OVER";
echo "";
fi
if [ "$ROLLED_OVER" = true ] ; then
OLD_INDEX=$(echo $ROLLOVER | jq --raw-output .old_index);
NEW_INDEX=$(echo $ROLLOVER | jq --raw-output .new_index);
if [ "$DEBUG" = true ] ; then
echo "OLD_INDEX: $OLD_INDEX";
echo "NEW_INDEX: $NEW_INDEX";
fi
COLD_INDEX=$(echo $OLD_INDEX | sed 's/hot/cold/g');
if [ "$DEBUG" = true ] ; then
echo "COLD_INDEX: $COLD_INDEX";
echo "";
echo "INDEX SETTINGS BEFORE UPDATE:";
curl --silent -XGET "http://elasticsearch.services.ruhmesmeile.local:9200/$OLD_INDEX" | jq .;
echo "";
fi
echo "INDEX SETTINGS UPDATED:";
curl --silent -XPUT "http://elasticsearch.services.ruhmesmeile.local:9200/$OLD_INDEX/_settings" -d'
{
"index.blocks.write": true,
"index.routing.allocation.require._name": "momcorpfdc",
"index.routing.allocation.exclude.box_type": "hot",
"index.routing.allocation.include.box_type": "cold",
"index.routing.allocation.total_shards_per_node": -1
}' | jq .;
echo "";
if [ "$DEBUG" = true ] ; then
echo "INDEX SETTINGS AFTER UPDATE:";
curl --silent -XGET "http://elasticsearch.services.ruhmesmeile.local:9200/$OLD_INDEX" | jq .;
echo "";
echo "CLUSTER HEALTH:";
curl --silent -XGET "http://elasticsearch.services.ruhmesmeile.local:9200/_cluster/health/" | jq .;
echo "";
echo "INDEX HEALTH:";
curl --silent -XGET "http://elasticsearch.services.ruhmesmeile.local:9200/_cluster/health/$OLD_INDEX" | jq .;
echo "";
echo "INDEX HEALTH AFTER WAIT:";
curl --silent -XGET "http://elasticsearch.services.ruhmesmeile.local:9200/_cluster/health/$OLD_INDEX?wait_for_no_relocating_shards" | jq .;
echo "";
echo "INDICES AND SHARDS IN CLUSTER AFTER ROLLOVER";
curl --silent -XGET "http://elasticsearch.services.ruhmesmeile.local:9200/_cat/indices";
curl --silent -XGET "http://elasticsearch.services.ruhmesmeile.local:9200/_cat/shards";
echo "";
fi
fi
Raw
result-of-call.log
RESETTINGS INDICES AND TEMPLATES:
{
"acknowledged": true
}
{
"acknowledged": true
}
FOLLOWING LINE SHOULD BE EMPTY (NO SHARDS IN CLUSTER)
RE-ADDING INDICES AND TEMPLATES:
{
"acknowledged": true
}
{
"acknowledged": true,
"shards_acknowledged": true
}
FRESH INDEX SETTINGS:
{
"hot-rmlogs-2016.11.10-000001": {
"aliases": {
"hot-rmlogs": {},
"search-rmlogs": {}
},
"mappings": {},
"settings": {
"index": {
"routing": {
"allocation": {
"include": {
"box_type": "hot"
},
"total_shards_per_node": "2"
}
},
"number_of_shards": "5",
"provided_name": "<hot-rmlogs-{now/d}-000001>",
"creation_date": "1478781787977",
"number_of_replicas": "1",
"uuid": "j2ZEhu4rTTGhTAvnEmURfQ",
"version": {
"created": "5000099"
}
}
}
}
}
CREATING DOCUMENTS:
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
"created"
INDICES AND SHARDS IN CLUSTER BEFORE ROLLOVER:
green open hot-rmlogs-2016.11.10-000001 j2ZEhu4rTTGhTAvnEmURfQ 5 1 100 0 141.5kb 68.6kb
hot-rmlogs-2016.11.10-000001 1 p STARTED 22 13.7kb 10.1.94.3 9becf4adbb74
hot-rmlogs-2016.11.10-000001 1 r STARTED 22 13.7kb 10.1.105.3 808b6c7026e0
hot-rmlogs-2016.11.10-000001 3 p STARTED 18 13.6kb 10.1.1.3 8bd47c300619
hot-rmlogs-2016.11.10-000001 3 r STARTED 18 13.6kb 10.1.19.3 dd9c0fe41c67
hot-rmlogs-2016.11.10-000001 4 r STARTED 23 13.9kb 10.1.87.3 8d272d79eda5
hot-rmlogs-2016.11.10-000001 4 p STARTED 23 13.9kb 10.1.19.3 dd9c0fe41c67
hot-rmlogs-2016.11.10-000001 2 p STARTED 17 13.6kb 10.1.105.3 808b6c7026e0
hot-rmlogs-2016.11.10-000001 2 r STARTED 17 17.8kb 10.1.1.3 8bd47c300619
hot-rmlogs-2016.11.10-000001 0 p STARTED 20 13.6kb 10.1.87.3 8d272d79eda5
hot-rmlogs-2016.11.10-000001 0 r STARTED 20 13.6kb 10.1.94.3 9becf4adbb74
ROLLOVER: {
"old_index": "hot-rmlogs-2016.11.10-000001",
"new_index": "hot-rmlogs-2016.11.10-000002",
"rolled_over": true,
"dry_run": false,
"acknowledged": true,
"shards_acknowledged": true,
"conditions": {
"[max_age: 1m]": false,
"[max_docs: 100]": true
}
}
ROLLED_OVER: true
OLD_INDEX: hot-rmlogs-2016.11.10-000001
NEW_INDEX: hot-rmlogs-2016.11.10-000002
COLD_INDEX: cold-rmlogs-2016.11.10-000001
INDEX SETTINGS BEFORE UPDATE:
{
"hot-rmlogs-2016.11.10-000001": {
"aliases": {
"search-rmlogs": {}
},
"mappings": {
"logs": {
"properties": {
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"post_date": {
"type": "date"
},
"user": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
},
"settings": {
"index": {
"routing": {
"allocation": {
"include": {
"box_type": "hot"
},
"total_shards_per_node": "2"
}
},
"number_of_shards": "5",
"provided_name": "<hot-rmlogs-{now/d}-000001>",
"creation_date": "1478781787977",
"number_of_replicas": "1",
"uuid": "j2ZEhu4rTTGhTAvnEmURfQ",
"version": {
"created": "5000099"
}
}
}
}
}
INDEX SETTINGS UPDATED:
{
"acknowledged": true
}
INDEX SETTINGS AFTER UPDATE:
{
"hot-rmlogs-2016.11.10-000001": {
"aliases": {
"search-rmlogs": {}
},
"mappings": {
"logs": {
"properties": {
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"post_date": {
"type": "date"
},
"user": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
},
"settings": {
"index": {
"routing": {
"allocation": {
"include": {
"box_type": "cold"
},
"exclude": {
"box_type": "hot"
},
"require": {
"_name": "momcorpfdc"
},
"total_shards_per_node": "-1"
}
},
"number_of_shards": "5",
"blocks": {
"write": "true"
},
"provided_name": "<hot-rmlogs-{now/d}-000001>",
"creation_date": "1478781787977",
"number_of_replicas": "1",
"uuid": "j2ZEhu4rTTGhTAvnEmURfQ",
"version": {
"created": "5000099"
}
}
}
}
}
CLUSTER HEALTH:
{
"cluster_name": "rm-logging",
"status": "yellow",
"timed_out": false,
"number_of_nodes": 7,
"number_of_data_nodes": 7,
"active_primary_shards": 10,
"active_shards": 19,
"relocating_shards": 0,
"initializing_shards": 1,
"unassigned_shards": 0,
"delayed_unassigned_shards": 0,
"number_of_pending_tasks": 0,
"number_of_in_flight_fetch": 0,
"task_max_waiting_in_queue_millis": 0,
"active_shards_percent_as_number": 95
}
INDEX HEALTH:
{
"cluster_name": "rm-logging",
"status": "green",
"timed_out": false,
"number_of_nodes": 7,
"number_of_data_nodes": 7,
"active_primary_shards": 5,
"active_shards": 10,
"relocating_shards": 0,
"initializing_shards": 0,
"unassigned_shards": 0,
"delayed_unassigned_shards": 0,
"number_of_pending_tasks": 0,
"number_of_in_flight_fetch": 0,
"task_max_waiting_in_queue_millis": 0,
"active_shards_percent_as_number": 100
}
INDEX HEALTH AFTER WAIT:
{
"cluster_name": "rm-logging",
"status": "green",
"timed_out": false,
"number_of_nodes": 7,
"number_of_data_nodes": 7,
"active_primary_shards": 5,
"active_shards": 10,
"relocating_shards": 0,
"initializing_shards": 0,
"unassigned_shards": 0,
"delayed_unassigned_shards": 0,
"number_of_pending_tasks": 0,
"number_of_in_flight_fetch": 0,
"task_max_waiting_in_queue_millis": 0,
"active_shards_percent_as_number": 100
}
INDICES AND SHARDS IN CLUSTER AFTER ROLLOVER
yellow open hot-rmlogs-2016.11.10-000002 cysMF7x9RKys9Pv9i7OkIA 5 1 0 0 1.1kb 650b
green open hot-rmlogs-2016.11.10-000001 j2ZEhu4rTTGhTAvnEmURfQ 5 1 100 0 141.5kb 68.6kb
hot-rmlogs-2016.11.10-000002 1 p STARTED 0 130b 10.1.94.3 9becf4adbb74
hot-rmlogs-2016.11.10-000002 1 r STARTED 0 130b 10.1.105.3 808b6c7026e0
hot-rmlogs-2016.11.10-000002 3 p STARTED 0 130b 10.1.1.3 8bd47c300619
hot-rmlogs-2016.11.10-000002 3 r STARTED 0 130b 10.1.19.3 dd9c0fe41c67
hot-rmlogs-2016.11.10-000002 4 r STARTED 0 130b 10.1.87.3 8d272d79eda5
hot-rmlogs-2016.11.10-000002 4 p STARTED 0 130b 10.1.19.3 dd9c0fe41c67
hot-rmlogs-2016.11.10-000002 2 p STARTED 0 130b 10.1.105.3 808b6c7026e0
hot-rmlogs-2016.11.10-000002 2 r INITIALIZING 10.1.1.3 8bd47c300619
hot-rmlogs-2016.11.10-000002 0 p STARTED 0 130b 10.1.87.3 8d272d79eda5
hot-rmlogs-2016.11.10-000002 0 r STARTED 0 130b 10.1.94.3 9becf4adbb74
hot-rmlogs-2016.11.10-000001 1 p STARTED 22 13.7kb 10.1.94.3 9becf4adbb74
hot-rmlogs-2016.11.10-000001 1 r STARTED 22 13.7kb 10.1.105.3 808b6c7026e0
hot-rmlogs-2016.11.10-000001 3 p STARTED 18 13.6kb 10.1.1.3 8bd47c300619
hot-rmlogs-2016.11.10-000001 3 r STARTED 18 13.6kb 10.1.19.3 dd9c0fe41c67
hot-rmlogs-2016.11.10-000001 4 r STARTED 23 13.9kb 10.1.87.3 8d272d79eda5
hot-rmlogs-2016.11.10-000001 4 p STARTED 23 13.9kb 10.1.19.3 dd9c0fe41c67
hot-rmlogs-2016.11.10-000001 2 p STARTED 17 13.6kb 10.1.105.3 808b6c7026e0
hot-rmlogs-2016.11.10-000001 2 r STARTED 17 17.8kb 10.1.1.3 8bd47c300619
hot-rmlogs-2016.11.10-000001 0 p STARTED 20 13.6kb 10.1.87.3 8d272d79eda5
hot-rmlogs-2016.11.10-000001 0 r STARTED 20 13.6kb 10.1.94.3 9becf4adbb74
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment