Last active
August 29, 2015 14:01
-
-
Save poohsen/a3d3bb319010bf0c5648 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| all of a sudden, at 2:00 in the night, I started getting these errors: | |
| [2014-05-09 13:35:43,032][DEBUG][action.bulk ] [Bushmaster] [.marvel-2014.05.09][0] failed to execute bulk item (index) index {[.marvel-2014.05.09][index_stats][zToUMDvuSxS0wVAPKB236A], source[{"@timestamp":"2014-05-09T11:35:43.024Z","cluster_name":"elasticsearch","index":".marvel-2014.05.09","_source_node":{"id":"62b9fpf0TTC8D3VH8NWgFg","name":"Bushmaster","transport_address":"inet[79.99.186.108/79.99.186.108:9300]","ip":"79.99.186.108","host":"79.99.186.108","ip_port":"79.99.186.108:9300","master_node":true,"data_node":true,"master":true},"primaries":{"docs":{"count":3368,"deleted":0},"store":{"size_in_bytes":16965332,"throttle_time_in_millis":73833},"indexing":{"index_total":3369,"index_time_in_millis":7681,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":73,"query_time_in_millis":179,"query_current":0,"fetch_total":71,"fetch_time_in_millis":23,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":83,"total_time_in_millis":138969,"total_docs":142155,"total_size_in_bytes":460876713},"refresh":{"total":901,"total_time_in_millis":89815},"flush":{"total":3,"total_time_in_millis":975},"warmer":{"current":0,"total":832,"total_time_in_millis":365},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":112423},"translog":{"operations":701,"size_in_bytes":1172594}},"total":{"docs":{"count":3368,"deleted":0},"store":{"size_in_bytes":16965332,"throttle_time_in_millis":73833},"indexing":{"index_total":3369,"index_time_in_millis":7681,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":73,"query_time_in_millis":179,"query_current":0,"fetch_total":71,"fetch_time_in_millis":23,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":83,"total_time_in_millis":138969,"total_docs":142155,"total_size_in_bytes":460876713},"refresh":{"total":901,"total_time_in_millis":89815},"flush":{"total":3,"total_time_in_millis":975},"warmer":{"current":0,"total":832,"total_time_in_millis":365},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":112423},"translog":{"operations":701,"size_in_bytes":1172594}}}]} | |
| org.elasticsearch.index.mapper.MapperParsingException: failed to parse [index] | |
| at org.elasticsearch.index.mapper.core.AbstractFieldMapper.parse(AbstractFieldMapper.java:418) | |
| at org.elasticsearch.index.mapper.object.ObjectMapper.serializeValue(ObjectMapper.java:616) | |
| at org.elasticsearch.index.mapper.object.ObjectMapper.parse(ObjectMapper.java:469) | |
| at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:515) | |
| at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:462) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.prepareCreate(InternalIndexShard.java:363) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:427) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:160) | |
| at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:556) | |
| at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:426) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.mapper.MapperParsingException: failed to parse date field [.marvel-2014.05.09], tried both date format [dateOptionalTime], and timestamp number with locale [] | |
| at org.elasticsearch.index.mapper.core.DateFieldMapper.parseStringValue(DateFieldMapper.java:562) | |
| at org.elasticsearch.index.mapper.core.DateFieldMapper.innerParseCreateField(DateFieldMapper.java:490) | |
| at org.elasticsearch.index.mapper.core.NumberFieldMapper.parseCreateField(NumberFieldMapper.java:215) | |
| at org.elasticsearch.index.mapper.core.AbstractFieldMapper.parse(AbstractFieldMapper.java:408) | |
| ... 12 more | |
| Caused by: java.lang.IllegalArgumentException: Invalid format: ".marvel-2014.05.09" | |
| at org.elasticsearch.common.joda.time.format.DateTimeFormatter.parseMillis(DateTimeFormatter.java:754) | |
| at org.elasticsearch.index.mapper.core.DateFieldMapper.parseStringValue(DateFieldMapper.java:556) | |
| ... 15 more | |
| [2014-05-09 13:35:43,035][ERROR][marvel.agent.exporter ] [Bushmaster] create failure (index:[.marvel-2014.05.09] type: [index_stats]): MapperParsingException[failed to parse [index]]; nested: MapperParsingException[failed to parse date field [.marvel-2014.05.08], tried both date format [dateOptionalTime], and timestamp number with locale []]; nested: IllegalArgumentException[Invalid format: ".marvel-2014.05.08"]; | |
| [2014-05-09 13:35:43,037][ERROR][marvel.agent.exporter ] [Bushmaster] create failure (index:[.marvel-2014.05.09] type: [index_stats]): MapperParsingException[failed to parse [index]]; nested: MapperParsingException[failed to parse date field [.marvel-2014.05.09], tried both date format [dateOptionalTime], and timestamp number with locale []]; nested: IllegalArgumentException[Invalid format: ".marvel-2014.05.09"]; | |
| Trying to make a sense of it, I looked at the index_stats mapping: | |
| GET /.marvel-2014.05.09/_mapping/index_stats | |
| gives me this: | |
| { | |
| ".marvel-2014.05.09": { | |
| "mappings": { | |
| "index_stats": { | |
| "dynamic_templates": [ | |
| { | |
| "string_fields": { | |
| "mapping": { | |
| "type": "multi_field", | |
| "fields": { | |
| "raw": { | |
| "index": "not_analyzed", | |
| "ignore_above": 256, | |
| "type": "string" | |
| }, | |
| "{name}": { | |
| "index": "analyzed", | |
| "omit_norms": true, | |
| "type": "string" | |
| } | |
| } | |
| }, | |
| "match": "*", | |
| "match_mapping_type": "string" | |
| } | |
| } | |
| ], | |
| "properties": { | |
| "@timestamp": { | |
| "type": "date", | |
| "format": "dateOptionalTime" | |
| }, | |
| "_source_node": { | |
| "properties": { | |
| "data_node": { | |
| "type": "boolean" | |
| }, | |
| "host": { | |
| "type": "string", | |
| "norms": { | |
| "enabled": false | |
| }, | |
| "fields": { | |
| "raw": { | |
| "type": "string", | |
| "index": "not_analyzed", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "id": { | |
| "type": "string", | |
| "norms": { | |
| "enabled": false | |
| }, | |
| "fields": { | |
| "raw": { | |
| "type": "string", | |
| "index": "not_analyzed", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "ip": { | |
| "type": "string", | |
| "norms": { | |
| "enabled": false | |
| }, | |
| "fields": { | |
| "raw": { | |
| "type": "string", | |
| "index": "not_analyzed", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "ip_port": { | |
| "type": "string", | |
| "norms": { | |
| "enabled": false | |
| }, | |
| "fields": { | |
| "raw": { | |
| "type": "string", | |
| "index": "not_analyzed", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "master": { | |
| "type": "boolean" | |
| }, | |
| "master_node": { | |
| "type": "boolean" | |
| }, | |
| "name": { | |
| "type": "string", | |
| "norms": { | |
| "enabled": false | |
| }, | |
| "fields": { | |
| "raw": { | |
| "type": "string", | |
| "index": "not_analyzed", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "transport_address": { | |
| "type": "string", | |
| "norms": { | |
| "enabled": false | |
| }, | |
| "fields": { | |
| "raw": { | |
| "type": "string", | |
| "index": "not_analyzed", | |
| "ignore_above": 256 | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "cluster_name": { | |
| "type": "string", | |
| "norms": { | |
| "enabled": false | |
| }, | |
| "fields": { | |
| "raw": { | |
| "type": "string", | |
| "index": "not_analyzed", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "index": { | |
| "type": "date", | |
| "format": "dateOptionalTime" | |
| }, | |
| "primaries": { | |
| "properties": { | |
| "completion": { | |
| "properties": { | |
| "size_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "docs": { | |
| "properties": { | |
| "count": { | |
| "type": "long" | |
| }, | |
| "deleted": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "fielddata": { | |
| "properties": { | |
| "evictions": { | |
| "type": "long" | |
| }, | |
| "memory_size_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "filter_cache": { | |
| "properties": { | |
| "evictions": { | |
| "type": "long" | |
| }, | |
| "memory_size_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "flush": { | |
| "properties": { | |
| "total": { | |
| "type": "long" | |
| }, | |
| "total_time_in_millis": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "get": { | |
| "properties": { | |
| "current": { | |
| "type": "long" | |
| }, | |
| "exists_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "exists_total": { | |
| "type": "long" | |
| }, | |
| "missing_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "missing_total": { | |
| "type": "long" | |
| }, | |
| "time_in_millis": { | |
| "type": "long" | |
| }, | |
| "total": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "id_cache": { | |
| "properties": { | |
| "memory_size_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "indexing": { | |
| "properties": { | |
| "delete_current": { | |
| "type": "long" | |
| }, | |
| "delete_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "delete_total": { | |
| "type": "long" | |
| }, | |
| "index_current": { | |
| "type": "long" | |
| }, | |
| "index_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "index_total": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "merges": { | |
| "properties": { | |
| "current": { | |
| "type": "long" | |
| }, | |
| "current_docs": { | |
| "type": "long" | |
| }, | |
| "current_size_in_bytes": { | |
| "type": "long" | |
| }, | |
| "total": { | |
| "type": "long" | |
| }, | |
| "total_docs": { | |
| "type": "long" | |
| }, | |
| "total_size_in_bytes": { | |
| "type": "long" | |
| }, | |
| "total_time_in_millis": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "percolate": { | |
| "properties": { | |
| "current": { | |
| "type": "long" | |
| }, | |
| "memory_size": { | |
| "type": "string", | |
| "norms": { | |
| "enabled": false | |
| }, | |
| "fields": { | |
| "raw": { | |
| "type": "string", | |
| "index": "not_analyzed", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "memory_size_in_bytes": { | |
| "type": "long" | |
| }, | |
| "queries": { | |
| "type": "long" | |
| }, | |
| "time_in_millis": { | |
| "type": "long" | |
| }, | |
| "total": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "refresh": { | |
| "properties": { | |
| "total": { | |
| "type": "long" | |
| }, | |
| "total_time_in_millis": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "search": { | |
| "properties": { | |
| "fetch_current": { | |
| "type": "long" | |
| }, | |
| "fetch_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "fetch_total": { | |
| "type": "long" | |
| }, | |
| "open_contexts": { | |
| "type": "long" | |
| }, | |
| "query_current": { | |
| "type": "long" | |
| }, | |
| "query_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "query_total": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "segments": { | |
| "properties": { | |
| "count": { | |
| "type": "long" | |
| }, | |
| "memory_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "store": { | |
| "properties": { | |
| "size_in_bytes": { | |
| "type": "long" | |
| }, | |
| "throttle_time_in_millis": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "translog": { | |
| "properties": { | |
| "operations": { | |
| "type": "long" | |
| }, | |
| "size_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "warmer": { | |
| "properties": { | |
| "current": { | |
| "type": "long" | |
| }, | |
| "total": { | |
| "type": "long" | |
| }, | |
| "total_time_in_millis": { | |
| "type": "long" | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "total": { | |
| "properties": { | |
| "completion": { | |
| "properties": { | |
| "size_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "docs": { | |
| "properties": { | |
| "count": { | |
| "type": "long" | |
| }, | |
| "deleted": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "fielddata": { | |
| "properties": { | |
| "evictions": { | |
| "type": "long" | |
| }, | |
| "memory_size_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "filter_cache": { | |
| "properties": { | |
| "evictions": { | |
| "type": "long" | |
| }, | |
| "memory_size_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "flush": { | |
| "properties": { | |
| "total": { | |
| "type": "long" | |
| }, | |
| "total_time_in_millis": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "get": { | |
| "properties": { | |
| "current": { | |
| "type": "long" | |
| }, | |
| "exists_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "exists_total": { | |
| "type": "long" | |
| }, | |
| "missing_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "missing_total": { | |
| "type": "long" | |
| }, | |
| "time_in_millis": { | |
| "type": "long" | |
| }, | |
| "total": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "id_cache": { | |
| "properties": { | |
| "memory_size_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "indexing": { | |
| "properties": { | |
| "delete_current": { | |
| "type": "long" | |
| }, | |
| "delete_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "delete_total": { | |
| "type": "long" | |
| }, | |
| "index_current": { | |
| "type": "long" | |
| }, | |
| "index_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "index_total": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "merges": { | |
| "properties": { | |
| "current": { | |
| "type": "long" | |
| }, | |
| "current_docs": { | |
| "type": "long" | |
| }, | |
| "current_size_in_bytes": { | |
| "type": "long" | |
| }, | |
| "total": { | |
| "type": "long" | |
| }, | |
| "total_docs": { | |
| "type": "long" | |
| }, | |
| "total_size_in_bytes": { | |
| "type": "long" | |
| }, | |
| "total_time_in_millis": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "percolate": { | |
| "properties": { | |
| "current": { | |
| "type": "long" | |
| }, | |
| "memory_size": { | |
| "type": "string", | |
| "norms": { | |
| "enabled": false | |
| }, | |
| "fields": { | |
| "raw": { | |
| "type": "string", | |
| "index": "not_analyzed", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "memory_size_in_bytes": { | |
| "type": "long" | |
| }, | |
| "queries": { | |
| "type": "long" | |
| }, | |
| "time_in_millis": { | |
| "type": "long" | |
| }, | |
| "total": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "refresh": { | |
| "properties": { | |
| "total": { | |
| "type": "long" | |
| }, | |
| "total_time_in_millis": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "search": { | |
| "properties": { | |
| "fetch_current": { | |
| "type": "long" | |
| }, | |
| "fetch_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "fetch_total": { | |
| "type": "long" | |
| }, | |
| "open_contexts": { | |
| "type": "long" | |
| }, | |
| "query_current": { | |
| "type": "long" | |
| }, | |
| "query_time_in_millis": { | |
| "type": "long" | |
| }, | |
| "query_total": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "segments": { | |
| "properties": { | |
| "count": { | |
| "type": "long" | |
| }, | |
| "memory_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "store": { | |
| "properties": { | |
| "size_in_bytes": { | |
| "type": "long" | |
| }, | |
| "throttle_time_in_millis": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "translog": { | |
| "properties": { | |
| "operations": { | |
| "type": "long" | |
| }, | |
| "size_in_bytes": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "warmer": { | |
| "properties": { | |
| "current": { | |
| "type": "long" | |
| }, | |
| "total": { | |
| "type": "long" | |
| }, | |
| "total_time_in_millis": { | |
| "type": "long" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| it contains this part: | |
| ... | |
| "index": { | |
| "type": "date", | |
| "format": "dateOptionalTime" | |
| }, | |
| ... | |
| So it looks like ES made a mistake when auto-mapping marvel index_stats documents. | |
| I then tried to recreate the index with by copying the mapping above and changing this field, then deleting and creating the index: | |
| ... | |
| "index": { | |
| "type": "date", | |
| "format": "yyyy-MM-dd||.marvel-yyyy-MM-dd" | |
| }, | |
| ... | |
| This gives me a much smaller mapping for a few seconds: | |
| { | |
| ".marvel-2014.05.12": { | |
| "mappings": { | |
| "index_stats": { | |
| "dynamic_templates": [ | |
| { | |
| "string_fields": { | |
| "mapping": { | |
| "type": "multi_field", | |
| "fields": { | |
| "raw": { | |
| "index": "not_analyzed", | |
| "ignore_above": 256, | |
| "type": "string" | |
| }, | |
| "{name}": { | |
| "index": "analyzed", | |
| "omit_norms": true, | |
| "type": "string" | |
| } | |
| } | |
| }, | |
| "match": "*", | |
| "match_mapping_type": "string" | |
| } | |
| } | |
| ], | |
| "properties": { | |
| "total": { | |
| "properties": { | |
| "percolate": { | |
| "properties": { | |
| "memory_size_in_bytes": { | |
| "type": "long" | |
| }, | |
| "queries": { | |
| "type": "long" | |
| }, | |
| "time_in_millis": { | |
| "type": "long" | |
| }, | |
| "total": { | |
| "type": "long" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| then, Marvel changes the mapping again, and I end up with the same errors as in the beginning. | |
| So I have to questions here: | |
| 1) Why can't Marvel handle it's own mappings? Is this a problem with Marvel or with the ES auto-mapping? | |
| 2) Why can't I create the mapping properly by just getting the mapping, changing it a little and the putting it back? It doesn't even work when I just get the mapping, delete the index and the create it again without changing anything in the mapping. I still get the "crippled" mapping above. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment