elasticsearchのテンプレート

gistfile1.txt

{
    "template" : "apache_log*",
    "mappings" : {
      "www3" : {
        "properties" : {
          "remote-host" : {
            "type" : "string"
          },
          "request-header-Referer" : {
            "type" : "string"
          },
          "request-header-User-Agent" : {
              "type" : "multi_field",
              "fields" : {
                  "request-header-User-Agent" : { "type" : "string", "index": "analyzed" },
                  "full" : { "type" : "string", "index": "not_analyzed" }
              }
          },
          "request-line" : {
            "type" : "string"
          },
          "request-process-time-us" : {
            "type" : "long"
          },
          "request-time" : {
            "type" : "date",
            "format" : "dateOptionalTime"
          },
          "response-status" : {
            "type" : "long"
          },
          "response-total-bytes" : {
            "type" : "long"
          }
        }
      }
    }
  }
}

curl -XPUT http://192.168.0.56:9200/_template/apache_template -d @apache_log.json

cat config.yml [~/embulk/weblog]
in:
type: file
path_prefix: /home/hidepin/embulk/www1/access.log
parser:
type: apache-custom-log
format: "%h %l %u %{[%d/%b/%Y:%T]}t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %D"
default_timezone: 'Asia/Tokyo'
charset: UTF-8
newline: CRLF

out: {type: stdout}

out:
type: elasticsearch
nodes:
- {host: 127.0.0.1, port: 9300}
index: apache_log
index_type: www1

embulk bundle update && bundle exec embulk preview config.yml

cd - && ./gradlew gem && cd - && embulk bundle update && bundle exec embulk preview config.yml

gem 'embulk-parser-apache-custom-log', path: '/home/hidepin/embulk/embulk-parser-apache-custom-log'
gem 'embulk-output-elasticsearch'

{
"template" : "apache_log_",
"settings" : {
"index" : {
"analysis" : {
"analyzer" : {
"method" : {
"type" : "pattern",
"pattern" : " ."
},
"url" : {
"type" : "pattern",
"pattern" : "^[A-Z]+|?."
},
"api" : {
"type" : "pattern",
"pattern" : "?._"
}
}
}
}
},
"mappings" : {
"www1" : {
"properties" : {
"remote-host" : {
"type" : "string",
"index" : "not_analyzed"
},
"request-header-Referer" : {
"type" : "string",
"index" : "not_analyzed"
},
"request-header-User-Agent" : {
"type" : "multi_field",
"fields" : {
"request-header-User-Agent" : {
"type" : "string",
"index" : "analyzed"
},
"full" : {
"type" : "string",
"index" : "not_analyzed"
}
}
},
"request-line" : {
"type" : "multi_field",
"fields" : {
"request-line" : {
"type" : "string",
"index" : "analyzed"
},
"method" : {
"type" : "string",
"index" : "analyzed",
"analyzer" : "method"
},
"url" : {
"type" : "string",
"index" : "analyzed",
"analyzer" : "url"
},
"api" : {
"type" : "string",
"index" : "analyzed",
"analyzer" : "api"
},
"full" : {
"type" : "string",
"index" : "not_analyzed"
}
}
},
"request-process-time-us" : {
"type" : "long"
},
"request-time" : {
"type" : "date",
"format" : "dateOptionalTime"
},
"response-status" : {
"type" : "long"
},
"response-total-bytes" : {
"type" : "long"
}
}
},

curl -XPUT http://192.168.0.56:9200/apache_log -d @multi.json