Flume Configuration

flume-conf.properies

agent.channels = file-channel
agent.sources = rabbitmq-source
agent.sinks = hdfs-sink hbase-sink

agent.channels.file-channel.type = file
agent.channels.file-channel.checkpointDir = /etc/flume/data/checkpoint
agent.channels.file-channel.dataDirs = /etc/flume/data

agent.sources.rabbitmq-source.channels = file-channel
agent.sources.rabbitmq-source.type = org.apache.flume.source.rabbitmq.RabbitMQSource
agent.sources.rabbitmq-source.hostname = mq01.dev02.thunderboltlabs.com
agent.sources.rabbitmq-source.exchangename = beats.bus
agent.sources.rabbitmq-source.topics = #

agent.sinks.hdfs-sink.type = hdfs
agent.sinks.hdfs-sink.channel = file-channel
agent.sinks.hdfs-sink.hdfs.path = hdfs://hw1.dev.thunderboltlabs.com/tmp/beats-bus/%Y/%m/%d/%H
agent.sinks.hdfs-sink.hdfs.filePrefix = beats-bus
agent.sinks.hdfs-sink.hdfs.fileSuffix = .event
agent.sinks.hdfs-sink.hdfs.rollInterval=3600
agent.sinks.hdfs-sink.hdfs.rollCount=0
agent.sinks.hdfs-sink.hdfs.rollSize=0
agent.sinks.hdfs-sink.hdfs.fileType = SequenceFile
agent.sinks.hdfs-sink.hdfs.writeFormat = Text

agent.sinks.hbase-sink.type = org.apache.flume.sink.hbase.HBaseSink
agent.sinks.hbase-sink.channel = file-channel
agent.sinks.hbase-sink.table = events
agent.sinks.hbase-sink.columnFamily = beats-bus
agent.sinks.hbase-sink.serializer = org.apache.flume.sink.hbase.SimpleHbaseEventSerializer
agent.sinks.hbase-sink.serializer.rowPrefix = e
agent.sinks.hbase-sink.serializer.payloadColumn = payload
agent.sinks.hbase-sink.serializer.incrementColumn = icol