Created
September 29, 2013 08:46
-
-
Save hellojinjie/6750572 to your computer and use it in GitHub Desktop.
NullPointException in UDAF
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package com.jaunty.hive.udf; | |
| import java.util.Map; | |
| import org.apache.hadoop.hive.ql.exec.Description; | |
| import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; | |
| import org.apache.hadoop.hive.ql.metadata.HiveException; | |
| import org.apache.hadoop.hive.ql.parse.SemanticException; | |
| import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; | |
| import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer; | |
| import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo; | |
| import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver2; | |
| import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; | |
| import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; | |
| import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; | |
| import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; | |
| import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; | |
| import com.google.common.collect.Maps; | |
| @Description(value="_FUNC_ calculate the cdn bytes loaded", name="cdnbytesloaded") | |
| public class GenericUDAFCdnBytesLoaded implements GenericUDAFResolver2 { | |
| @Override | |
| public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) | |
| throws SemanticException { | |
| throw new SemanticException( | |
| "This UDAF does not support the deprecated getEvaluator() method."); | |
| } | |
| @Override | |
| public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) | |
| throws SemanticException { | |
| ObjectInspector[] ois = info.getParameterObjectInspectors(); | |
| if (ois.length != 1) { | |
| throw new UDFArgumentLengthException("_FUNC_ take only one parameter"); | |
| } | |
| return new GenericUDAFCdnBytesLoadedEvaluator(); | |
| } | |
| public static class CdnAggregationBuffer implements AggregationBuffer { | |
| Map<String, Long> bytesLoaded; | |
| } | |
| public static class GenericUDAFCdnBytesLoadedEvaluator extends GenericUDAFEvaluator { | |
| private MapObjectInspector inputOI; | |
| private MapObjectInspector outputOI; | |
| private Map<String, String> cdnPatterns; | |
| { | |
| cdnPatterns = Maps.newHashMap(); | |
| cdnPatterns.put("cdnl3nl", "Level3"); | |
| cdnPatterns.put("cdnak", "Akamai"); | |
| cdnPatterns.put("cndllnwnl", "LimeLight"); | |
| cdnPatterns.put("cdncd", "CDNetworks"); | |
| } | |
| @Override | |
| public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { | |
| super.init(m, parameters); | |
| if (parameters.length != 1) { | |
| throw new UDFArgumentLengthException("In GenericUDAFCdnBytesLoadedEvaluator " | |
| + "parameters length is not 1"); | |
| } | |
| inputOI = (MapObjectInspector) parameters[0]; | |
| outputOI = (MapObjectInspector) ObjectInspectorUtils | |
| .getStandardObjectInspector(inputOI, ObjectInspectorCopyOption.JAVA); | |
| return outputOI; | |
| } | |
| @Override | |
| public AggregationBuffer getNewAggregationBuffer() throws HiveException { | |
| CdnAggregationBuffer buffer = new CdnAggregationBuffer(); | |
| buffer.bytesLoaded = Maps.newHashMap(); | |
| return buffer; | |
| } | |
| @Override | |
| public void reset(AggregationBuffer agg) throws HiveException { | |
| CdnAggregationBuffer buffer = (CdnAggregationBuffer) agg; | |
| buffer.bytesLoaded.clear(); | |
| } | |
| @Override | |
| public void iterate(AggregationBuffer agg, Object[] parameters) | |
| throws HiveException { | |
| if (parameters.length == 1 && parameters[0] != null) { | |
| merge(agg, parameters[0]); | |
| } | |
| } | |
| @Override | |
| public Object terminatePartial(AggregationBuffer agg) throws HiveException { | |
| return ((CdnAggregationBuffer) agg).bytesLoaded; | |
| } | |
| @Override | |
| public void merge(AggregationBuffer agg, Object partial) throws HiveException { | |
| if (partial == null) { | |
| return; | |
| } | |
| try { | |
| CdnAggregationBuffer buffer = (CdnAggregationBuffer) agg; | |
| @SuppressWarnings("unchecked") | |
| Map<String, Long> bytesLoaded = (Map<String, Long>) inputOI.getMap(partial); | |
| for (String key : bytesLoaded.keySet()) { | |
| boolean matched = false; | |
| for (String pattern : cdnPatterns.keySet()) { | |
| if (key.contains(pattern)) { | |
| String cdnName = cdnPatterns.get(pattern); | |
| Long loaded = buffer.bytesLoaded.get(cdnName); | |
| if (loaded == null) { | |
| loaded = bytesLoaded.get(key); | |
| } else { | |
| loaded = loaded > bytesLoaded.get(key) ? loaded : bytesLoaded.get(key); | |
| } | |
| buffer.bytesLoaded.put(cdnName, loaded); | |
| matched = true; | |
| break; | |
| } | |
| } | |
| if (matched == false) { | |
| Long loaded = buffer.bytesLoaded.get(key); | |
| if (loaded == null) { | |
| loaded = bytesLoaded.get(key); | |
| } else { | |
| loaded = loaded > bytesLoaded.get(key) ? loaded : bytesLoaded.get(key); | |
| } | |
| buffer.bytesLoaded.put(key, loaded); | |
| } | |
| } | |
| } catch (Exception e) { | |
| e.printStackTrace(); | |
| } | |
| } | |
| @Override | |
| public Object terminate(AggregationBuffer agg) throws HiveException { | |
| return ((CdnAggregationBuffer) agg).bytesLoaded; | |
| } | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Task Logs: 'attempt_201309271638_0025_m_000000_0' | |
| stdout logs | |
| stderr logs | |
| java.lang.reflect.InvocationTargetException | |
| Continuing ... | |
| java.lang.IllegalArgumentException: Unbound variable: GenericUDAFCdnBytesLoaded$GenericUDAFCdnBytesLoadedEvaluator0 | |
| Continuing ... | |
| syslog logs | |
| 2013-09-29 16:41:13,328 INFO org.apache.hadoop.util.NativeCodeLoader: Loaded the native-hadoop library | |
| 2013-09-29 16:41:13,509 INFO org.apache.hadoop.mapred.TaskRunner: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/distcache/6000556044884442336_-388038660_1759672009/172.16.0.237/tmp/hive-qos/hive_2013-09-29_16-42-52_508_3403558341003759013/-mr-10003/41af96bb-2da6-4a3d-b17c-6d99530c7239 <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/HIVE_PLAN41af96bb-2da6-4a3d-b17c-6d99530c7239 | |
| 2013-09-29 16:41:13,514 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/META-INF <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/META-INF | |
| 2013-09-29 16:41:13,515 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/hive-exec-log4j.properties <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/hive-exec-log4j.properties | |
| 2013-09-29 16:41:13,516 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/job.jar <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/job.jar | |
| 2013-09-29 16:41:13,517 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/org <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/org | |
| 2013-09-29 16:41:13,517 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/javolution <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/javolution | |
| 2013-09-29 16:41:13,518 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/javax <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/javax | |
| 2013-09-29 16:41:13,519 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/.job.jar.crc <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/.job.jar.crc | |
| 2013-09-29 16:41:13,520 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/javaewah <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/javaewah | |
| 2013-09-29 16:41:13,811 INFO org.apache.hadoop.util.ProcessTree: setsid exited with exit code 0 | |
| 2013-09-29 16:41:13,815 INFO org.apache.hadoop.mapred.Task: Using ResourceCalculatorPlugin : org.apache.hadoop.util.LinuxResourceCalculatorPlugin@5266f887 | |
| 2013-09-29 16:41:13,923 INFO org.apache.hadoop.mapred.MapTask: Processing split: Paths:/user/hive/warehouse/jj.db/ssuser/000000_0.avro:0+598480InputFormatClass: org.apache.hadoop.mapred.TextInputFormat | |
| 2013-09-29 16:41:13,965 WARN org.apache.hadoop.hive.conf.HiveConf: hive-site.xml not found on CLASSPATH | |
| 2013-09-29 16:41:14,166 WARN org.apache.hadoop.io.compress.snappy.LoadSnappy: Snappy native library is available | |
| 2013-09-29 16:41:14,166 INFO org.apache.hadoop.io.compress.snappy.LoadSnappy: Snappy native library loaded | |
| 2013-09-29 16:41:14,175 INFO org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader: Processing file hdfs://172.16.0.237:9000/user/hive/warehouse/jj.db/ssuser/000000_0.avro | |
| 2013-09-29 16:41:14,175 INFO org.apache.hadoop.mapred.MapTask: numReduceTasks: 1 | |
| 2013-09-29 16:41:14,185 INFO org.apache.hadoop.mapred.MapTask: io.sort.mb = 100 | |
| 2013-09-29 16:41:14,227 INFO org.apache.hadoop.mapred.MapTask: data buffer = 79691776/99614720 | |
| 2013-09-29 16:41:14,227 INFO org.apache.hadoop.mapred.MapTask: record buffer = 262144/327680 | |
| 2013-09-29 16:41:14,236 INFO ExecMapper: maximum memory = 186646528 | |
| 2013-09-29 16:41:14,236 INFO ExecMapper: conf classpath = [file:/home/qos/app/hadoop-1.2.1/conf/, file:/usr/java/jdk1.7.0_40/lib/tools.jar, file:/home/qos/app/hadoop-1.2.1/, file:/home/qos/app/hadoop-1.2.1/hadoop-core-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/asm-3.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/aspectjrt-1.6.11.jar, file:/home/qos/app/hadoop-1.2.1/lib/aspectjtools-1.6.11.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-beanutils-1.7.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-beanutils-core-1.8.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-cli-1.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-codec-1.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-collections-3.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-configuration-1.6.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-daemon-1.0.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-digester-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-el-1.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-httpclient-3.0.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-io-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-lang-2.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-logging-1.1.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-logging-api-1.0.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-math-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-net-3.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/core-3.1.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/derby-10.4.2.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-capacity-scheduler-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-fairscheduler-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-thriftfs-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hsqldb-1.8.0.10.jar, file:/home/qos/app/hadoop-1.2.1/lib/jackson-core-asl-1.8.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jackson-mapper-asl-1.8.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jasper-compiler-5.5.12.jar, file:/home/qos/app/hadoop-1.2.1/lib/jasper-runtime-5.5.12.jar, file:/home/qos/app/hadoop-1.2.1/lib/jdeb-0.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-core-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-json-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-server-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jets3t-0.6.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/jetty-6.1.26.jar, file:/home/qos/app/hadoop-1.2.1/lib/jetty-util-6.1.26.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsch-0.1.42.jar, file:/home/qos/app/hadoop-1.2.1/lib/junit-4.5.jar, file:/home/qos/app/hadoop-1.2.1/lib/kfs-0.2.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/log4j-1.2.15.jar, file:/home/qos/app/hadoop-1.2.1/lib/mockito-all-1.8.5.jar, file:/home/qos/app/hadoop-1.2.1/lib/oro-2.0.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/servlet-api-2.5-20081211.jar, file:/home/qos/app/hadoop-1.2.1/lib/slf4j-api-1.4.3.jar, file:/home/qos/app/hadoop-1.2.1/lib/slf4j-log4j12-1.4.3.jar, file:/home/qos/app/hadoop-1.2.1/lib/snappy-java-1.1.0-M4.jar, file:/home/qos/app/hadoop-1.2.1/lib/xmlenc-0.52.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsp-2.1/jsp-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsp-2.1/jsp-api-2.1.jar, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/classes, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/distcache/-6906649896783115865_-1711195192_1759672612/172.16.0.237/home/qos/app/hadoop/tmp/mapred/staging/qos/.staging/job_201309271638_0025/libjars/hive-builtins-0.10.0.jar/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/distcache/2983734166189404195_41183263_1759672650/172.16.0.237/home/qos/app/hadoop/tmp/mapred/staging/qos/.staging/job_201309271638_0025/libjars/hive-function-1.0-SNAPSHOT.jar/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/] | |
| 2013-09-29 16:41:14,236 INFO ExecMapper: thread classpath = [file:/home/qos/app/hadoop-1.2.1/conf/, file:/usr/java/jdk1.7.0_40/lib/tools.jar, file:/home/qos/app/hadoop-1.2.1/, file:/home/qos/app/hadoop-1.2.1/hadoop-core-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/asm-3.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/aspectjrt-1.6.11.jar, file:/home/qos/app/hadoop-1.2.1/lib/aspectjtools-1.6.11.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-beanutils-1.7.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-beanutils-core-1.8.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-cli-1.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-codec-1.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-collections-3.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-configuration-1.6.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-daemon-1.0.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-digester-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-el-1.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-httpclient-3.0.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-io-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-lang-2.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-logging-1.1.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-logging-api-1.0.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-math-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-net-3.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/core-3.1.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/derby-10.4.2.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-capacity-scheduler-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-fairscheduler-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-thriftfs-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hsqldb-1.8.0.10.jar, file:/home/qos/app/hadoop-1.2.1/lib/jackson-core-asl-1.8.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jackson-mapper-asl-1.8.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jasper-compiler-5.5.12.jar, file:/home/qos/app/hadoop-1.2.1/lib/jasper-runtime-5.5.12.jar, file:/home/qos/app/hadoop-1.2.1/lib/jdeb-0.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-core-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-json-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-server-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jets3t-0.6.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/jetty-6.1.26.jar, file:/home/qos/app/hadoop-1.2.1/lib/jetty-util-6.1.26.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsch-0.1.42.jar, file:/home/qos/app/hadoop-1.2.1/lib/junit-4.5.jar, file:/home/qos/app/hadoop-1.2.1/lib/kfs-0.2.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/log4j-1.2.15.jar, file:/home/qos/app/hadoop-1.2.1/lib/mockito-all-1.8.5.jar, file:/home/qos/app/hadoop-1.2.1/lib/oro-2.0.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/servlet-api-2.5-20081211.jar, file:/home/qos/app/hadoop-1.2.1/lib/slf4j-api-1.4.3.jar, file:/home/qos/app/hadoop-1.2.1/lib/slf4j-log4j12-1.4.3.jar, file:/home/qos/app/hadoop-1.2.1/lib/snappy-java-1.1.0-M4.jar, file:/home/qos/app/hadoop-1.2.1/lib/xmlenc-0.52.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsp-2.1/jsp-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsp-2.1/jsp-api-2.1.jar, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/classes, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/distcache/-6906649896783115865_-1711195192_1759672612/172.16.0.237/home/qos/app/hadoop/tmp/mapred/staging/qos/.staging/job_201309271638_0025/libjars/hive-builtins-0.10.0.jar/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/distcache/2983734166189404195_41183263_1759672650/172.16.0.237/home/qos/app/hadoop/tmp/mapred/staging/qos/.staging/job_201309271638_0025/libjars/hive-function-1.0-SNAPSHOT.jar/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/] | |
| 2013-09-29 16:41:14,280 INFO org.apache.hadoop.hive.ql.exec.MapOperator: Adding alias ssuser to work list for file hdfs://172.16.0.237:9000/user/hive/warehouse/jj.db/ssuser | |
| 2013-09-29 16:41:14,283 INFO org.apache.hadoop.hive.ql.exec.MapOperator: dump TS struct<siteid:string,day:string,time:string,ip:string,clientid:string,viewid:string,msgid:int,eventtype:string,playtime:int,devicetype:string,apptype:string,updateinterval:int,convention:string,appversion:string,streamurl:string,streamdescription:string,streamtype:int,streamlength:int,os:string,player:string,productid:string,networktype:string,bandwidth:int,bitrate:int,switchmethod:int,bytesloaded:map<string,bigint>,bytesloadeddelta:map<string,bigint>,dropframecount:int,windowmode:string,bufferlength:int,buffertime:int,isairplay:int,airplayduration:int,userid:string,progid:int,progtype:string,gameid:int,gamedate:string,hometeam:string,awayteam:string,code:int,dt:string> | |
| 2013-09-29 16:41:14,284 INFO ExecMapper: | |
| <MAP>Id =7 | |
| <Children> | |
| <TS>Id =3 | |
| <Children> | |
| <SEL>Id =2 | |
| <Children> | |
| <GBY>Id =1 | |
| <Children> | |
| <RS>Id =0 | |
| <Parent>Id = 1 null<\Parent> | |
| <\RS> | |
| <\Children> | |
| <Parent>Id = 2 null<\Parent> | |
| <\GBY> | |
| <\Children> | |
| <Parent>Id = 3 null<\Parent> | |
| <\SEL> | |
| <\Children> | |
| <Parent>Id = 7 null<\Parent> | |
| <\TS> | |
| <\Children> | |
| <\MAP> | |
| 2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.MapOperator: Initializing Self 7 MAP | |
| 2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.TableScanOperator: Initializing Self 3 TS | |
| 2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.TableScanOperator: Operator 3 TS initialized | |
| 2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.TableScanOperator: Initializing children of 3 TS | |
| 2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.SelectOperator: Initializing child 2 SEL | |
| 2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.SelectOperator: Initializing Self 2 SEL | |
| 2013-09-29 16:41:14,287 INFO org.apache.hadoop.hive.ql.exec.SelectOperator: SELECT struct<siteid:string,day:string,time:string,ip:string,clientid:string,viewid:string,msgid:int,eventtype:string,playtime:int,devicetype:string,apptype:string,updateinterval:int,convention:string,appversion:string,streamurl:string,streamdescription:string,streamtype:int,streamlength:int,os:string,player:string,productid:string,networktype:string,bandwidth:int,bitrate:int,switchmethod:int,bytesloaded:map<string,bigint>,bytesloadeddelta:map<string,bigint>,dropframecount:int,windowmode:string,bufferlength:int,buffertime:int,isairplay:int,airplayduration:int,userid:string,progid:int,progtype:string,gameid:int,gamedate:string,hometeam:string,awayteam:string,code:int,dt:string> | |
| 2013-09-29 16:41:14,289 INFO org.apache.hadoop.hive.ql.exec.SelectOperator: Operator 2 SEL initialized | |
| 2013-09-29 16:41:14,289 INFO org.apache.hadoop.hive.ql.exec.SelectOperator: Initializing children of 2 SEL | |
| 2013-09-29 16:41:14,289 INFO org.apache.hadoop.hive.ql.exec.GroupByOperator: Initializing child 1 GBY | |
| 2013-09-29 16:41:14,289 INFO org.apache.hadoop.hive.ql.exec.GroupByOperator: Initializing Self 1 GBY | |
| 2013-09-29 16:41:14,313 INFO org.apache.hadoop.mapred.TaskLogsTruncater: Initializing logs' truncater with mapRetainSize=-1 and reduceRetainSize=-1 | |
| 2013-09-29 16:41:14,332 INFO org.apache.hadoop.io.nativeio.NativeIO: Initialized cache for UID to User mapping with a cache timeout of 14400 seconds. | |
| 2013-09-29 16:41:14,333 INFO org.apache.hadoop.io.nativeio.NativeIO: Got UserName qos for UID 500 from the native implementation | |
| 2013-09-29 16:41:14,335 WARN org.apache.hadoop.mapred.Child: Error running child | |
| java.lang.RuntimeException: Error in configuring object | |
| at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93) | |
| at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64) | |
| at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117) | |
| at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:426) | |
| at org.apache.hadoop.mapred.MapTask.run(MapTask.java:366) | |
| at org.apache.hadoop.mapred.Child$4.run(Child.java:255) | |
| at java.security.AccessController.doPrivileged(Native Method) | |
| at javax.security.auth.Subject.doAs(Subject.java:415) | |
| at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190) | |
| at org.apache.hadoop.mapred.Child.main(Child.java:249) | |
| Caused by: java.lang.reflect.InvocationTargetException | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:606) | |
| at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88) | |
| ... 9 more | |
| Caused by: java.lang.RuntimeException: Error in configuring object | |
| at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93) | |
| at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64) | |
| at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117) | |
| at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) | |
| ... 14 more | |
| Caused by: java.lang.reflect.InvocationTargetException | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:606) | |
| at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88) | |
| ... 17 more | |
| Caused by: java.lang.RuntimeException: Map operator initialization failed | |
| at org.apache.hadoop.hive.ql.exec.ExecMapper.configure(ExecMapper.java:121) | |
| ... 22 more | |
| Caused by: java.lang.NullPointerException | |
| at org.apache.hadoop.hive.ql.exec.GroupByOperator.initializeOp(GroupByOperator.java:368) | |
| at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360) | |
| at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:436) | |
| at org.apache.hadoop.hive.ql.exec.Operator.initializeChildren(Operator.java:392) | |
| at org.apache.hadoop.hive.ql.exec.SelectOperator.initializeOp(SelectOperator.java:62) | |
| at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360) | |
| at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:436) | |
| at org.apache.hadoop.hive.ql.exec.Operator.initializeChildren(Operator.java:392) | |
| at org.apache.hadoop.hive.ql.exec.TableScanOperator.initializeOp(TableScanOperator.java:166) | |
| at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360) | |
| at org.apache.hadoop.hive.ql.exec.MapOperator.initializeOp(MapOperator.java:441) | |
| at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360) | |
| at org.apache.hadoop.hive.ql.exec.ExecMapper.configure(ExecMapper.java:98) | |
| ... 22 more | |
| 2013-09-29 16:41:14,338 INFO org.apache.hadoop.mapred.Task: Runnning cleanup for the task |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment