hive-recommendations.sql

set hive.auto.convert.join.noconditionaltask.size=200M;
set hive.auto.convert.join.noconditionaltask=true;
set hive.compute.query.using.stats=false;
set hive.exec.compress.intermediate=true;
set hive.exec.compress.output=true;
set hive.exec.dynamic.partition.mode=strict;
set hive.exec.dynamic.partition=true;
set hive.exec.max.created.files=100000;
set hive.exec.max.dynamic.partitions.pernode=1000;
set hive.exec.max.dynamic.partitions=10000;
set hive.exec.parallel=true; 
set hive.exec.reducers.bytes.per.reducer=67108864;
set hive.fetch.task.conversion.threshold=1073741824;
set hive.fetch.task.conversion=more;
set hive.groupby.orderby.position.alias=true;
set hive.hadoop.supports.splittable.combineinputformat=true; 
set hive.limit.pushdown.memory.usage=0.4;
set hive.map.aggr.hash.percentmemory=0.5;
set hive.map.aggr=true;
set hive.merge.mapfiles=true;
set hive.merge.mapredfiles=true;
set hive.merge.size.per.task=268435456;
set hive.merge.smallfiles.avgsize=268435456;
set hive.merge.sparkfiles=true;
set hive.optimize.bucketmapjoin.sortedmerge=false;
set hive.optimize.bucketmapjoin=true;
set hive.optimize.constant.propagation=true;
set hive.optimize.index.filter=true;
set hive.optimize.ppd=true;
set hive.optimize.reducededuplication.min.reducer=4;
set hive.optimize.reducededuplication=true;
set hive.optimize.sort.dynamic.partition=enabled;
set hive.optimize.sort.dynamic.partition=false;
set hive.plan.serialization.format=kryo;
set hive.ppd.remove.duplicatefilters=true;
set hive.smbjoin.cache.rows=10000;
set hive.stats.autogather=true;
set hive.stats.fetch.column.stats=true;
set mapred.min.split.size.per.node=134217728;
set mapred.min.split.size.per.rack=134217728;
set mapred.min.split.size=134217728;
set parquet.compression=SNAPPY;

#These are always job specific and need to be tuned per query - 

set mapreduce.job.reduces=20;
set mapreduce.map.java.opts=-Xmx20024m;
set mapreduce.map.memory.mb=20024;
set mapreduce.reduce.java.opts=-Xmx20024m;
set mapreduce.reduce.memory.mb=20024;

#These needs to be set no matter what - 
set mapred.job.queue.name=<queuename>;