hadoop configuration with ceph

core-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>

  <property>
    <name>hadoop.proxyuser.mapred.groups</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.mapred.hosts</name>
    <value>*</value>
  </property>

<!--
     Set-up the default filesystems
-->
  <property>
    <name>fs.default.name</name>
    <value>ceph://hobbit01.beocat.ksu.edu:6789/</value>
    <final />
  </property>

  <property>
    <name>fs.defaultFS</name>
    <value>ceph://hobbit01.beocat.ksu.edu:6789/</value>
    <final />
  </property>

<!--
     Configure non-default ceph options
-->
  <property>
    <name>ceph.auth.id</name>
    <value>hadoop</value>
    <final />
  </property>

  <property>
    <name>ceph.auth.keyring</name>
    <value>/etc/ceph/ceph.client.hadoop.keyring</value>
    <final />
  </property>

  <property>
    <name>ceph.data.pools</name>
    <value>hadoop</value>
    <final />
  </property>

  <property>
    <name>ceph.root.dir</name>
    <value>/hadoop</value>
    <final />
  </property>

  <property>
    <name>fs.AbstractFileSystem.ceph.impl</name>
    <value>org.apache.hadoop.fs.ceph.CephFs</value>
    <final />
  </property>

  <property>
    <name>fs.ceph.impl</name>
    <value>org.apache.hadoop.fs.ceph.CephFileSystem</value>
    <final />
  </property>

  <property>
    <name>ceph.replication</name>
    <value>2</value>
    <final />
  </property>
</configuration>

mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
  <property>
    <name>mapreduce.jobhistory.address</name>
    <value>gremlin00.beocat.ksu.edu:10020</value>
  </property>
  <property>
    <name>mapreduce.jobhistory.webapp.address</name>
    <value>gremlin00.beocat.ksu.edu:19888</value>
  </property>
  <property>
    <name>yarn.app.mapreduce.am.staging-dir</name>
    <value>/staging</value>
  </property>
</configuration>

yarn-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>

  <property>
    <name>yarn.resourcemanager.hostname</name>
    <value>gremlin00.beocat.ksu.edu</value>
  </property>

  <property>
    <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
  </property>

  <property>
    <name>yarn.app.mapreduce.am.staging-dir</name>
    <value>/staging</value>
  </property>

  <property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
  </property>

  <property>
    <description>Where to aggregate logs to.</description>
    <name>yarn.nodemanager.remote-app-log-dir</name>
    <value>ceph://hobbit01.beocat.ksu.edu:6789/var/log/hadoop-yarn/apps</value>
  </property>

  <property>
    <description>Classpath for typical applications.</description>
    <name>yarn.application.classpath</name>
    <value>
      $HADOOP_CONF_DIR,
      $HADOOP_COMMON_HOME/*,
      $HADOOP_COMMON_HOME/lib/*,
      $HADOOP_HDFS_HOME/*,
      $HADOOP_HDFS_HOME/lib/*,
      $HADOOP_MAPRED_HOME/*,
      $HADOOP_MAPRED_HOME/lib/*,
      $HADOOP_YARN_HOME/*,
      $HADOOP_YARN_HOME/lib/*
    </value>
  </property>

  <property>
    <description>Number of CPU cores that can be allocated for containers.</description>
    <name>yarn.nodemanager.resource.cpu-vcores</name>
    <value>12</value>
  </property>

  <property>
    <description>Amount of physical memory, in MB, that can be allocated for containers.</description>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>65536</value>
  </property>
</configuration>