mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>

  <property>
     <name>mapreduce.framework.name</name>
     <value>yarn</value>
  </property>

  <property>
     <name>mapreduce.jobhistory.address</name>
     <value>${master.address}:10020</value>
  </property>

  <property>
  <name>mapreduce.map.memory.mb</name>
  <value>4096</value>
  </property>

  <property>
  <name>mapreduce.reduce.memory.mb</name>
  <value>4096</value>
  </property>

  <property>
    <name>mapreduce.map.cpu.vcores</name>
    <value>1</value>
  </property>
  <property>
    <name>mapreduce.reduce.cpu.vcores</name>
    <value>1</value>
  </property>

  <property>
    <name>mapreduce.map.java.opts</name>
    <value>-server -Xmx3800m -Djava.net.preferIPv4Stack=true</value>
    <!-- <value>-server -Xmx1024m -Djava.net.preferIPv4Stack=true  -agentlib:hprof=file=/tmp/hprof/@[email protected],format=b</value> -->
  </property>

  <property>
    <name>mapreduce.reduce.java.opts</name>
    <value>-server -Xmx3800m -Djava.net.preferIPv4Stack=true</value>
    <!-- <value>-server -Xmx1024m -Djava.net.preferIPv4Stack=true  -agentlib:hprof=file=/tmp/hprof/@[email protected],format=b</value> -->
  </property>

  <property>
    <name>mapreduce.cluster.local.dir</name>
    <value>/hadoop1/mapred,/hadoop2/mapred</value>
  </property>


</configuration>
ozawa@ozamac:~/work

tez-site.xml

<?xml version="1.0"?>
<configuration>

<!-- Site specific YARN configuration properties -->
<property>
  <name>yarn.nodemanager.aux-services</name>
  <value>mapreduce_shuffle,spark_shuffle</value>
</property>

<property>
  <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
  <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>

<property>
  <name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
  <value>org.apache.spark.network.yarn.YarnShuffleService</value>
</property>

<property>
  <name>yarn.scheduler.minimum-allocation-mb</name>
  <value>512</value>
</property>

<property>
  <name>yarn.scheduler.maximum-allocation-mb</name>
  <value>32768</value>
</property>

<property>
  <name>yarn.nodemanager.resource.cpu-vcores</name>
  <value>16</value>
</property>

<property>
  <name>yarn.nodemanager.resource.memory-mb</name>
  <value>98304</value>
</property>

<property>
  <name>yarn.nodemanager.container-monitor.resource-calculator.class</name>
  <value>org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin</value>
</property>

<property>
  <name>yarn.resourcemanager.address</name>
  <value>${master.address}:8081</value>
</property>

<property>
  <name>yarn.resourcemanager.resource-tracker.address</name>
  <value>${master.address}:8082</value>
</property>

<property>
  <name>yarn.resourcemanager.scheduler.address</name>
  <value>${master.address}:8083</value>
</property>

<property>
  <name>yarn.resourcemanager.admin.address</name>
  <value>${master.address}:8084</value>
</property>

<property>
  <name>yarn.resourcemanager.webapp.address</name>
  <value>${master.address}:8088</value>
</property>

<property>
  <name>yarn.nodemanager.vmem-check-enabled</name>
  <value>false</value>
</property>

<property>
  <name>yarn.nodemanager.pmem-check-enabled</name>
  <value>true</value>
</property>

<property>
  <name>yarn.acl.enable</name>
  <value>false</value>
</property>

<property>
  <name>yarn.nodemanager.vmem-pmem-ratio</name>
  <value>1000</value>
</property>

<property>
  <name>yarn.log-aggregation-enable</name>
  <value>true</value>
</property>

<property>
  <name>yarn.log-aggregation.retain-seconds</name>
  <value>604800</value>
  <!-- one week -->
</property>

<property>
  <name>yarn.nodemanager.linux-container-executor.path</name>
  <value>/home/ubuntu/hadoop/bin/container-executor</value>
  <!-- one week -->
</property>

<!--
<property>
  <name>yarn.nodemanager.container-executor.class</name>
  <value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
</property>
-->

<property>
  <name>yarn.nodemanager.linux-container-executor.group</name>
  <value><!-- TODO your user group here. should match container-executor.cfg --></value>
</property>

<property>
  <description>Indicate to clients whether Timeline service is enabled or not.
  If enabled, the TimelineClient library used by end-users will post entities
  and events to the Timeline server.</description>
  <name>yarn.timeline-service.enabled</name>
  <value>true</value>
</property>

<property>
  <description>The setting that controls whether yarn system metrics is
  published on the timeline server or not by RM.</description>
  <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
  <value>true</value>
</property>

<property>
  <description>Indicate to clients whether to query generic application
  data from timeline history-service or not. If not enabled then application
  data is queried only from Resource Manager.</description>
  <name>yarn.timeline-service.generic-application-history.enabled</name>
  <value>true</value>
</property>

<!--
<property>
  <description>The hostname of the Timeline service web application.</description>
  <name>yarn.timeline-service.hostname</name>
  <value>${master.address}</value>
</property>
-->

<property>
  <description>Enables cross-origin support (CORS) for web services where
  cross-origin web response headers are needed. For example, javascript making
  a web services request to the timeline server.</description>
  <name>yarn.timeline-service.http-cross-origin.enabled</name>
  <value>true</value>
</property>
</configuration>

$ hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar randomtextwriter -Dmapreduce.framework.name=yarn-tez -Dtest.randomtextwrite.total_bytes=54975581388 wc500mb