yarn_preemption_capacity_scheduler

set tez.queue.name=user;
set tez.grouping.split-count=1000;
select transform (1) using '/tmp/sleep.sh 1000000000' from wikipedia limit 10;

set tez.queue.name=production;
set tez.grouping.split-count=1000;
select transform (1) using '/tmp/sleep.sh 1000000000' from wikipedia limit 10;

set hive.execution.engine=mr;
set mapred.job.queue.name=user;

set tez.grouping.split-count=1000;
set tez.queue.name=production;
select count(1) from randy_test;

select domain, count(1) from wikipedia group by domain;

yarn jar /usr/hdp/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar -jar /usr/hdp/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar -shell_command "sleep 1000" -num_containers 100 -queue production
yarn jar /usr/hdp/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar -jar /usr/hdp/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar -shell_command "sleep 1000" -num_containers 100 -queue user


cat sleep.sh
cat
sleep $1


yarn-site.xml
yarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill=15000
yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval=3000
yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round=0.1
yarn.resourcemanager.scheduler.monitor.policies=org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy
yarn.resourcemanager.monitor.capacity.preemption.max_ignored_over_capacity=0.1
yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor=0.2

capacity-scheduler:
yarn.scheduler.capacity.maximum-am-resource-percent=0.2
yarn.scheduler.capacity.maximum-applications=10000
yarn.scheduler.capacity.node-locality-delay=40
yarn.scheduler.capacity.queue-mappings-override.enable=false
yarn.scheduler.capacity.root.accessible-node-labels=*
yarn.scheduler.capacity.root.acl_administer_queue=*
yarn.scheduler.capacity.root.capacity=100
yarn.scheduler.capacity.root.default.acl_submit_applications=*
yarn.scheduler.capacity.root.default.capacity=30
yarn.scheduler.capacity.root.default.maximum-capacity=100
yarn.scheduler.capacity.root.default.state=RUNNING
yarn.scheduler.capacity.root.default.user-limit-factor=1
yarn.scheduler.capacity.root.hive-test.acl_administer_queue=*
yarn.scheduler.capacity.root.hive-test.acl_submit_applications=*
yarn.scheduler.capacity.root.hive-test.capacity=70
yarn.scheduler.capacity.root.hive-test.maximum-capacity=70
yarn.scheduler.capacity.root.hive-test.minimum-user-limit-percent=100
yarn.scheduler.capacity.root.hive-test.ordering-policy=fifo
yarn.scheduler.capacity.root.hive-test.production.acl_administer_queue=*
yarn.scheduler.capacity.root.hive-test.production.acl_submit_applications=*
yarn.scheduler.capacity.root.hive-test.production.capacity=60
yarn.scheduler.capacity.root.hive-test.production.maximum-capacity=100
yarn.scheduler.capacity.root.hive-test.production.minimum-user-limit-percent=100
yarn.scheduler.capacity.root.hive-test.production.ordering-policy=fifo
yarn.scheduler.capacity.root.hive-test.production.production1.acl_administer_queue=*
yarn.scheduler.capacity.root.hive-test.production.production1.acl_submit_applications=*
yarn.scheduler.capacity.root.hive-test.production.production1.capacity=33
yarn.scheduler.capacity.root.hive-test.production.production1.maximum-capacity=100
yarn.scheduler.capacity.root.hive-test.production.production1.minimum-user-limit-percent=100
yarn.scheduler.capacity.root.hive-test.production.production1.ordering-policy=fifo
yarn.scheduler.capacity.root.hive-test.production.production1.state=RUNNING
yarn.scheduler.capacity.root.hive-test.production.production1.user-limit-factor=10
yarn.scheduler.capacity.root.hive-test.production.production2.acl_administer_queue=*
yarn.scheduler.capacity.root.hive-test.production.production2.acl_submit_applications=*
yarn.scheduler.capacity.root.hive-test.production.production2.capacity=33
yarn.scheduler.capacity.root.hive-test.production.production2.maximum-capacity=100
yarn.scheduler.capacity.root.hive-test.production.production2.minimum-user-limit-percent=100
yarn.scheduler.capacity.root.hive-test.production.production2.ordering-policy=fifo
yarn.scheduler.capacity.root.hive-test.production.production2.state=RUNNING
yarn.scheduler.capacity.root.hive-test.production.production2.user-limit-factor=10
yarn.scheduler.capacity.root.hive-test.production.production3.acl_administer_queue=*
yarn.scheduler.capacity.root.hive-test.production.production3.acl_submit_applications=*
yarn.scheduler.capacity.root.hive-test.production.production3.capacity=34
yarn.scheduler.capacity.root.hive-test.production.production3.maximum-capacity=100
yarn.scheduler.capacity.root.hive-test.production.production3.minimum-user-limit-percent=100
yarn.scheduler.capacity.root.hive-test.production.production3.ordering-policy=fifo
yarn.scheduler.capacity.root.hive-test.production.production3.state=RUNNING
yarn.scheduler.capacity.root.hive-test.production.production3.user-limit-factor=10
yarn.scheduler.capacity.root.hive-test.production.queues=production1,production2,production3
yarn.scheduler.capacity.root.hive-test.production.state=RUNNING
yarn.scheduler.capacity.root.hive-test.production.user-limit-factor=10
yarn.scheduler.capacity.root.hive-test.queues=production,user
yarn.scheduler.capacity.root.hive-test.state=RUNNING
yarn.scheduler.capacity.root.hive-test.user-limit-factor=10
yarn.scheduler.capacity.root.hive-test.user.acl_administer_queue=*
yarn.scheduler.capacity.root.hive-test.user.acl_submit_applications=*
yarn.scheduler.capacity.root.hive-test.user.capacity=40
yarn.scheduler.capacity.root.hive-test.user.maximum-capacity=100
yarn.scheduler.capacity.root.hive-test.user.minimum-user-limit-percent=100
yarn.scheduler.capacity.root.hive-test.user.ordering-policy=fifo
yarn.scheduler.capacity.root.hive-test.user.state=RUNNING
yarn.scheduler.capacity.root.hive-test.user.user-limit-factor=10
yarn.scheduler.capacity.root.queues=default,hive-test