-
-
Save hidepin/873d84d86dcc9c2c0f9a to your computer and use it in GitHub Desktop.
| { | |
| "template" : "apache_log*", | |
| "mappings" : { | |
| "www3" : { | |
| "properties" : { | |
| "remote-host" : { | |
| "type" : "string" | |
| }, | |
| "request-header-Referer" : { | |
| "type" : "string" | |
| }, | |
| "request-header-User-Agent" : { | |
| "type" : "multi_field", | |
| "fields" : { | |
| "request-header-User-Agent" : { "type" : "string", "index": "analyzed" }, | |
| "full" : { "type" : "string", "index": "not_analyzed" } | |
| } | |
| }, | |
| "request-line" : { | |
| "type" : "string" | |
| }, | |
| "request-process-time-us" : { | |
| "type" : "long" | |
| }, | |
| "request-time" : { | |
| "type" : "date", | |
| "format" : "dateOptionalTime" | |
| }, | |
| "response-status" : { | |
| "type" : "long" | |
| }, | |
| "response-total-bytes" : { | |
| "type" : "long" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } |
embulk bundle update && bundle exec embulk preview config.yml
cd - && ./gradlew gem && cd - && embulk bundle update && bundle exec embulk preview config.yml
gem 'embulk-parser-apache-custom-log', path: '/home/hidepin/embulk/embulk-parser-apache-custom-log'
gem 'embulk-output-elasticsearch'
{
"template" : "apache_log_",
"settings" : {
"index" : {
"analysis" : {
"analyzer" : {
"method" : {
"type" : "pattern",
"pattern" : " ."
},
"url" : {
"type" : "pattern",
"pattern" : "^[A-Z]+|?."
},
"api" : {
"type" : "pattern",
"pattern" : "?._"
}
}
}
}
},
"mappings" : {
"www1" : {
"properties" : {
"remote-host" : {
"type" : "string",
"index" : "not_analyzed"
},
"request-header-Referer" : {
"type" : "string",
"index" : "not_analyzed"
},
"request-header-User-Agent" : {
"type" : "multi_field",
"fields" : {
"request-header-User-Agent" : {
"type" : "string",
"index" : "analyzed"
},
"full" : {
"type" : "string",
"index" : "not_analyzed"
}
}
},
"request-line" : {
"type" : "multi_field",
"fields" : {
"request-line" : {
"type" : "string",
"index" : "analyzed"
},
"method" : {
"type" : "string",
"index" : "analyzed",
"analyzer" : "method"
},
"url" : {
"type" : "string",
"index" : "analyzed",
"analyzer" : "url"
},
"api" : {
"type" : "string",
"index" : "analyzed",
"analyzer" : "api"
},
"full" : {
"type" : "string",
"index" : "not_analyzed"
}
}
},
"request-process-time-us" : {
"type" : "long"
},
"request-time" : {
"type" : "date",
"format" : "dateOptionalTime"
},
"response-status" : {
"type" : "long"
},
"response-total-bytes" : {
"type" : "long"
}
}
},
curl -XPUT http://192.168.0.56:9200/apache_log -d @multi.json
cat config.yml [~/embulk/weblog]
in:
type: file
path_prefix: /home/hidepin/embulk/www1/access.log
parser:
type: apache-custom-log
format: "%h %l %u %{[%d/%b/%Y:%T]}t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %D"
default_timezone: 'Asia/Tokyo'
charset: UTF-8
newline: CRLF
out: {type: stdout}
out:
type: elasticsearch
nodes:
- {host: 127.0.0.1, port: 9300}
index: apache_log
index_type: www1