Flume Configuration files

flume-site-agent.xml

<!-- a1 (agent)  -->
<configuration>
  <property>
    <name>flume.master.servers</name>
    <value>$master_IP</value>
    <description>This is the address for the config servers status server (http)</description>
  </property>

  <property>
    <name>flume.collector.event.host</name>
    <value>$collector_IP</value>
    <description>This is the host name of the default "remote" collector.</description>
  </property>

  <property>
    <name>flume.collector.port</name>
    <value>35853</value>
    <description>This default tcp port that the collector listens to in order to receive events it is collecting.</description>
  </property>

  <property>
    <name>flume.agent.logdir</name>
    <value>/mnt/flume-${user.name}/agent</value>
    <description> This is the directory that write-ahead logging data
      or disk-failover data is collected from applications gets
      written to. The agent watches this directory.
    </description>
  </property>
</configuration>

flume-site-collector.xml

<!-- c1 (collector) /etc/flume/conf/flume-site.xml -->
<configuration>
  <property>
    <name>flume.master.servers</name>
    <value>$master</value>
    <description>This is the address for the config servers status server (http)</description>
  </property>

  <property>
    <name>flume.collector.event.host</name>
    <value>localhost</value>
    <description>This is the host name of the default "remote" collector.</description>
  </property>

  <property>
    <name>flume.collector.port</name>
    <value>35853</value>
    <description>This default tcp port that the collector listens to in order to receive events it is collecting.</description>
  </property>

  <property>
    <name>fs.default.name</name>
    <value>s3n://$account:$secret@$bucket</value>
  </property>

  <property>
    <name>fs.s3n.impl</name>
    <value>org.apache.hadoop.fs.s3native.NativeS3FileSystem</value>
  </property>
  
  <property>
    <name>fs.s3.awsAccessKeyId</name>
    <value>$account</value>
  </property>
  
  <property>
    <name>fs.s3.awsSecretAccessKey</name>
    <value>$secret</value>
  </property>

  <property>
    <name>fs.s3n.awsAccessKeyId</name>
    <value>$account</value>
  </property>
  
  <property>
    <name>fs.s3n.awsSecretAccessKey</name>
    <value>$secret</value>
  </property>

  <property>
    <name>flume.agent.logdir</name>
    <value>/mnt/flume-${user.name}/agent</value>
    <description> This is the directory that write-ahead logging data
      or disk-failover data is collected from applications gets
      written to. The agent watches this directory.
    </description>
  </property>
 
  <property>
    <name>flume.collector.dfs.dir</name>
    <value>file:///mnt/flume-${user.name}/collected</value>
    <description>This is a dfs directory that is the the final resting
    place for logs to be stored in.  This defaults to a local dir in
    /tmp but can be hadoop URI path that such as hdfs://namenode/path/
    </description>
  </property>  

  <property>
    <name>flume.collector.dfs.compress.gzip</name>
    <value>true</value>
    <description>Writes compressed output in gzip format to dfs. value is
     boolean type, i.e. true/false</description>
  </property>

  <property>
    <name>flume.collector.roll.millis</name>
    <value>60000</value>
    <description>The time (in milliseconds)
    between when hdfs files are closed and a new file is opened
    (rolled).
    </description>
  </property> 
</configuration>