Skip to content

Instantly share code, notes, and snippets.

@s-tajima

s-tajima/td-agent.conf

Created February 28, 2014 04:01
Show Gist options
  • Save s-tajima/9265001 to your computer and use it in GitHub Desktop.
Save s-tajima/9265001 to your computer and use it in GitHub Desktop.
Download ZIP
Configs for benchmarking of ElasticSearch compress.
Raw
td-agent.conf
<match *>
type copy
<store>
type forest
subtype elasticsearch
<template>
id es-test01.${tag}
include_tag_key true
tag_key @tag
host 127.0.0.1
port 9200
logstash_format true
logstash_prefix test01
logstash_dateformat %Y.%m.%d
type_name ${tag_parts[2]}.${tag_parts[3]}
flush_interval 10s
buffer_type file
buffer_path /var/log/td-agent/buffer/es-test01.${tag}
</template>
</store>
<store>
type forest
subtype elasticsearch
<template>
id es-test02.${tag}
include_tag_key true
tag_key @tag
host 127.0.0.1
port 9200
logstash_format true
logstash_prefix test02
logstash_dateformat %Y.%m.%d
type_name ${tag_parts[2]}.${tag_parts[3]}
flush_interval 10s
buffer_type file
buffer_path /var/log/td-agent/buffer/es-test02.${tag}
</template>
</store>
<store>
type forest
subtype elasticsearch
<template>
id es-test03.${tag}
include_tag_key true
tag_key @tag
host 127.0.0.1
port 9200
logstash_format true
logstash_prefix test03
logstash_dateformat %Y.%m.%d
type_name ${tag_parts[2]}.${tag_parts[3]}
flush_interval 10s
buffer_type file
buffer_path /var/log/td-agent/buffer/es-test03.${tag}
</template>
</store>
</match>
Raw
template_test01.json
{
"order": 1,
"template": "test01-*",
"settings": {
},
"mappings": {
"_default_": {
"properties": {
"@timestamp": { "index": "not_analyzed", "type": "date" },
"_log_name": { "index": "not_analyzed", "type": "string" },
"_hostname": { "index": "not_analyzed", "type": "string" },
"canonicalname": { "index": "not_analyzed", "type": "string" },
"host": { "type": "ip" },
"ident": { "index": "not_analyzed", "type": "string" },
"user": { "index": "not_analyzed", "type": "string" },
"method": { "index": "not_analyzed", "type": "string" },
"path": { "index": "not_analyzed", "type": "string" },
"protocol": { "index": "not_analyzed", "type": "string" },
"status": { "index": "not_analyzed", "type": "string" },
"size": { "index": "not_analyzed", "type": "string" },
"reqtime_microsec": { "index": "not_analyzed", "type": "integer" },
"referer": { "type": "multi_field", "fields": { "request_uri": { "type": "string", "index" : "analyzed" }, "full": { "type": "string", "index" : "not_analyzed" } } },
"agent": { "type": "multi_field", "fields": { "request_uri": { "type": "string", "index" : "analyzed" }, "full": { "type": "string", "index" : "not_analyzed" } } },
"forwardedfor": { "index": "not_analyzed", "type": "string" }
}
}
}
}
Raw
template_test02.json
{
"order": 2,
"template": "test02-*",
"settings": {
},
"mappings": {
"_default_": {
"_source": { "compress": true },
"properties": {
"@timestamp": { "index": "not_analyzed", "type": "date" },
"_log_name": { "index": "not_analyzed", "type": "string" },
"_hostname": { "index": "not_analyzed", "type": "string" },
"canonicalname": { "index": "not_analyzed", "type": "string" },
"host": { "type": "ip" },
"ident": { "index": "not_analyzed", "type": "string" },
"user": { "index": "not_analyzed", "type": "string" },
"method": { "index": "not_analyzed", "type": "string" },
"path": { "index": "not_analyzed", "type": "string" },
"protocol": { "index": "not_analyzed", "type": "string" },
"status": { "index": "not_analyzed", "type": "string" },
"size": { "index": "not_analyzed", "type": "string" },
"reqtime_microsec": { "index": "not_analyzed", "type": "integer" },
"referer": { "type": "multi_field", "fields": { "request_uri": { "type": "string", "index" : "analyzed" }, "full": { "type": "string", "index" : "not_analyzed" } } },
"agent": { "type": "multi_field", "fields": { "request_uri": { "type": "string", "index" : "analyzed" }, "full": { "type": "string", "index" : "not_analyzed" } } },
"forwardedfor": { "index": "not_analyzed", "type": "string" }
}
}
}
}
Raw
template_test03.json
{
"order": 3,
"template": "test03-*",
"settings": {
"index.store.compress.stored" : "true"
},
"mappings": {
"_default_": {
"properties": {
"@timestamp": { "index": "not_analyzed", "type": "date" },
"_log_name": { "index": "not_analyzed", "type": "string" },
"_hostname": { "index": "not_analyzed", "type": "string" },
"canonicalname": { "index": "not_analyzed", "type": "string" },
"host": { "type": "ip" },
"ident": { "index": "not_analyzed", "type": "string" },
"user": { "index": "not_analyzed", "type": "string" },
"method": { "index": "not_analyzed", "type": "string" },
"path": { "index": "not_analyzed", "type": "string" },
"protocol": { "index": "not_analyzed", "type": "string" },
"status": { "index": "not_analyzed", "type": "string" },
"size": { "index": "not_analyzed", "type": "string" },
"reqtime_microsec": { "index": "not_analyzed", "type": "integer" },
"referer": { "type": "multi_field", "fields": { "request_uri": { "type": "string", "index" : "analyzed" }, "full": { "type": "string", "index" : "not_analyzed" } } },
"agent": { "type": "multi_field", "fields": { "request_uri": { "type": "string", "index" : "analyzed" }, "full": { "type": "string", "index" : "not_analyzed" } } },
"forwardedfor": { "index": "not_analyzed", "type": "string" }
}
}
}
}
Raw
test01_index_metadata.txt
{
state: open
settings: {
index: {
uuid: EHFxdZJrRLanRmz2YBm3zA
number_of_replicas: 1
number_of_shards: 5
version: {
created: 1000099
}
}
}
mappings: {
_default_: {
properties: {
protocol: {
index: not_analyzed
type: string
}
host: {
type: ip
}
status: {
index: not_analyzed
type: string
}
reqtime_microsec: {
type: integer
}
agent: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
size: {
index: not_analyzed
type: string
}
_log_name: {
index: not_analyzed
type: string
}
@timestamp: {
format: dateOptionalTime
type: date
}
forwardedfor: {
index: not_analyzed
type: string
}
ident: {
index: not_analyzed
type: string
}
path: {
index: not_analyzed
type: string
}
method: {
index: not_analyzed
type: string
}
_hostname: {
index: not_analyzed
type: string
}
referer: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
user: {
index: not_analyzed
type: string
}
canonicalname: {
index: not_analyzed
type: string
}
}
}
httpd_androbk.error: {
properties: {
protocol: {
index: not_analyzed
type: string
}
host: {
type: ip
}
status: {
index: not_analyzed
type: string
}
reqtime_microsec: {
type: integer
}
agent: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
size: {
index: not_analyzed
type: string
}
_log_name: {
index: not_analyzed
type: string
}
@tag: {
type: string
}
message: {
type: string
}
@timestamp: {
format: dateOptionalTime
type: date
}
forwardedfor: {
index: not_analyzed
type: string
}
ident: {
index: not_analyzed
type: string
}
path: {
index: not_analyzed
type: string
}
method: {
index: not_analyzed
type: string
}
referer: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
_hostname: {
index: not_analyzed
type: string
}
user: {
index: not_analyzed
type: string
}
canonicalname: {
index: not_analyzed
type: string
}
}
}
httpd_androbk.access: {
properties: {
protocol: {
index: not_analyzed
type: string
}
host: {
type: ip
}
status: {
index: not_analyzed
type: string
}
reqtime_microsec: {
type: integer
}
agent: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
size: {
index: not_analyzed
type: string
}
_log_name: {
index: not_analyzed
type: string
}
@tag: {
type: string
}
message: {
type: string
}
@timestamp: {
format: dateOptionalTime
type: date
}
forwardedfor: {
index: not_analyzed
type: string
}
ident: {
index: not_analyzed
type: string
}
path: {
index: not_analyzed
type: string
}
method: {
index: not_analyzed
type: string
}
referer: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
_hostname: {
index: not_analyzed
type: string
}
user: {
index: not_analyzed
type: string
}
canonicalname: {
index: not_analyzed
type: string
}
}
}
}
aliases: [ ]
}
Raw
test02_index_metadata.txt
{
state: open
settings: {
index: {
uuid: DcIQYtA2RDeu6OHvdT5r9Q
number_of_replicas: 1
number_of_shards: 5
version: {
created: 1000099
}
}
}
mappings: {
_default_: {
_source: {
compress: true
}
properties: {
protocol: {
index: not_analyzed
type: string
}
host: {
type: ip
}
status: {
index: not_analyzed
type: string
}
reqtime_microsec: {
type: integer
}
agent: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
size: {
index: not_analyzed
type: string
}
_log_name: {
index: not_analyzed
type: string
}
@timestamp: {
format: dateOptionalTime
type: date
}
forwardedfor: {
index: not_analyzed
type: string
}
ident: {
index: not_analyzed
type: string
}
path: {
index: not_analyzed
type: string
}
method: {
index: not_analyzed
type: string
}
_hostname: {
index: not_analyzed
type: string
}
referer: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
user: {
index: not_analyzed
type: string
}
canonicalname: {
index: not_analyzed
type: string
}
}
}
httpd_androbk.error: {
_source: {
compress: true
}
properties: {
protocol: {
index: not_analyzed
type: string
}
host: {
type: ip
}
status: {
index: not_analyzed
type: string
}
reqtime_microsec: {
type: integer
}
agent: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
size: {
index: not_analyzed
type: string
}
_log_name: {
index: not_analyzed
type: string
}
@tag: {
type: string
}
message: {
type: string
}
@timestamp: {
format: dateOptionalTime
type: date
}
forwardedfor: {
index: not_analyzed
type: string
}
ident: {
index: not_analyzed
type: string
}
path: {
index: not_analyzed
type: string
}
method: {
index: not_analyzed
type: string
}
referer: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
_hostname: {
index: not_analyzed
type: string
}
user: {
index: not_analyzed
type: string
}
canonicalname: {
index: not_analyzed
type: string
}
}
}
httpd_androbk.access: {
_source: {
compress: true
}
properties: {
protocol: {
index: not_analyzed
type: string
}
host: {
type: ip
}
status: {
index: not_analyzed
type: string
}
reqtime_microsec: {
type: integer
}
agent: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
size: {
index: not_analyzed
type: string
}
_log_name: {
index: not_analyzed
type: string
}
@tag: {
type: string
}
message: {
type: string
}
@timestamp: {
format: dateOptionalTime
type: date
}
forwardedfor: {
index: not_analyzed
type: string
}
ident: {
index: not_analyzed
type: string
}
path: {
index: not_analyzed
type: string
}
method: {
index: not_analyzed
type: string
}
referer: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
_hostname: {
index: not_analyzed
type: string
}
user: {
index: not_analyzed
type: string
}
canonicalname: {
index: not_analyzed
type: string
}
}
}
}
aliases: [ ]
}
Raw
test03_index_metadata.txt
{
state: open
settings: {
index: {
uuid: -wEKXtLXQSO1hhXWsW5Sfw
store: {
compress: {
stored: true
}
}
number_of_replicas: 1
number_of_shards: 5
version: {
created: 1000099
}
}
}
mappings: {
_default_: {
properties: {
protocol: {
index: not_analyzed
type: string
}
host: {
type: ip
}
status: {
index: not_analyzed
type: string
}
reqtime_microsec: {
type: integer
}
agent: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
size: {
index: not_analyzed
type: string
}
_log_name: {
index: not_analyzed
type: string
}
@timestamp: {
format: dateOptionalTime
type: date
}
forwardedfor: {
index: not_analyzed
type: string
}
ident: {
index: not_analyzed
type: string
}
path: {
index: not_analyzed
type: string
}
method: {
index: not_analyzed
type: string
}
_hostname: {
index: not_analyzed
type: string
}
referer: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
user: {
index: not_analyzed
type: string
}
canonicalname: {
index: not_analyzed
type: string
}
}
}
httpd_androbk.error: {
properties: {
protocol: {
index: not_analyzed
type: string
}
host: {
type: ip
}
status: {
index: not_analyzed
type: string
}
reqtime_microsec: {
type: integer
}
agent: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
size: {
index: not_analyzed
type: string
}
_log_name: {
index: not_analyzed
type: string
}
@tag: {
type: string
}
message: {
type: string
}
@timestamp: {
format: dateOptionalTime
type: date
}
forwardedfor: {
index: not_analyzed
type: string
}
ident: {
index: not_analyzed
type: string
}
path: {
index: not_analyzed
type: string
}
method: {
index: not_analyzed
type: string
}
referer: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
_hostname: {
index: not_analyzed
type: string
}
user: {
index: not_analyzed
type: string
}
canonicalname: {
index: not_analyzed
type: string
}
}
}
httpd_androbk.access: {
properties: {
protocol: {
index: not_analyzed
type: string
}
host: {
type: ip
}
status: {
index: not_analyzed
type: string
}
reqtime_microsec: {
type: integer
}
agent: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
size: {
index: not_analyzed
type: string
}
_log_name: {
index: not_analyzed
type: string
}
@tag: {
type: string
}
message: {
type: string
}
@timestamp: {
format: dateOptionalTime
type: date
}
forwardedfor: {
index: not_analyzed
type: string
}
ident: {
index: not_analyzed
type: string
}
path: {
index: not_analyzed
type: string
}
method: {
index: not_analyzed
type: string
}
referer: {
index: no
type: string
fields: {
full: {
index: not_analyzed
type: string
}
request_uri: {
type: string
}
}
}
_hostname: {
index: not_analyzed
type: string
}
user: {
index: not_analyzed
type: string
}
canonicalname: {
index: not_analyzed
type: string
}
}
}
}
aliases: [ ]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment