command

#
# tw ( http://shokai.github.io/tw/ ) から fluentd を経由して Elasticsearch に投入する
# 
# tw 単体で起動すると↓みたいな JSON が出力される
# $ tw --stream --format=json
# {"id":000000000000000000,"user":"XXXX","text":"test test test","time":"2014-03-20 01:00:00 +0900","fav_count":null,"rt_count":null}
#
$ tw --stream --format=json | fluent-cat twitter.timeline

fluent.conf

<match twitter.timeline>
  type elasticsearch
  host localhost
  port 9200
  index_name twitter
  type_name timeline
</match>

mapping.json

{
  "twitter": {
    "mappings": {
      "timeline": {
        "properties": {
          "user": {
            "analyzer": "kuromoji_analyzer",
            "type": "string"
          },
          "time": {
            "include_in_all": false,
            "format": "dateOptionalTime",
            "store": true,
            "type": "date"
          },
          "text": {
            "analyzer": "kuromoji_analyzer",
            "store": true,
            "type": "string"
          },
          "rt_count": {
            "type": "integer"
          },
          "id": {
            "include_in_all": false,
            "store": true,
            "type": "long"
          },
          "fav_count": {
            "type": "integer"
          }
        },
        "_timestamp": {
          "path": "time",
          "enabled": true
        },
        "_id": {
          "path": "id"
        },
        "_all": {
          "analyzer": "kuromoji_analyzer"
        }
      }
    }
  }
}