agibsonccc

	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at java.util.Collections$SynchronizedCollection.add(Collections.java:2035)
	at com.esotericsoftware.kryo.serializers.CollectionSerializer.read(CollectionSerializer.java:109)
	at com.esotericsoftware.kryo.serializers.CollectionSerializer.read(CollectionSerializer.java:18)
	at com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:648)
	at com.esotericsoftware.kryo.serializers.FieldSerializer$ObjectField.read(FieldSerializer.java:605)
	... 12 more
[ Executor task launch worker-0] Executor                       INFO  Finished task 1.0 in stage 3.0 (TID 13). 1771602 bytes result sent to driver
[       dispatcher-event-loop-2] TaskSchedulerImpl              DEBUG parentName: , name: TaskSet_3, runningTasks: 2

agibsonccc

  protected void buildModel(){
        switch (modelType) {
            case "MLP":
                // NSLKDD got .98% with 512, RMSProp & leakyrelu
                BasicMLPModel mlpmodel = new BasicMLPModel(
                        new int[]{nIn, 256, 256},
                        new int[]{256, 256, nOut},
                        iterations,
                        "leakyrelu",
                        WeightInit.XAVIER,

agibsonccc

[agibsonccc@localhost libnd4j]$ ./buildnativeoperations.sh  -c cuda
eval cmake
PACKAGING = none
BUILD = release
CHIP = cuda
ARCH = x86-64
CHIP_VERSION = 
GPU_COMPUTE_CAPABILITY = all
EXPERIMENTAL = no
LIBRARY TYPE = dynamic

agibsonccc

WARNING: The LD_LIBRARY_PATH variable is not set. Defaulting to a blank string.
Starting demo_spark-master_1
Starting demo_zookeeper_1
Starting demo_elasticsearch_1
Starting demo_cassandra_1
Starting demo_hdfsyarn_1
Starting demo_spark-slave_1
Starting demo_logstash_1
Starting demo_elastichq_1
Starting demo_streamsets_1

agibsonccc

16/10/20 08:27:52 INFO executor.Executor: Finished task 5.0 in stage 2.0 (TID 8). 2264 bytes result sent to driver
16/10/20 08:27:52 INFO scheduler.TaskSetManager: Finished task 5.0 in stage 2.0 (TID 8) in 51 ms on localhost (1/7)
16/10/20 08:28:01 INFO yarn.ApplicationMaster: Waiting for spark context initialization ... 
16/10/20 08:28:11 INFO yarn.ApplicationMaster: Waiting for spark context initialization ... 
16/10/20 08:28:21 INFO yarn.ApplicationMaster: Waiting for spark context initialization ... 
16/10/20 08:28:31 INFO yarn.ApplicationMaster: Waiting for spark context initialization ... 
16/10/20 08:28:41 INFO yarn.ApplicationMaster: Waiting for spark context initialization ... 
16/10/20 08:28:51 INFO yarn.ApplicationMaster: Waiting for spark context initialization ... 
16/10/20 08:29:01 ERROR yarn.ApplicationMaster: SparkContext did not initialize after waiting for 100000 ms. Please check earlier log output for errors. Failing the application.
16/10/20 08:29:01 INFO yarn.ApplicationMaster: Final app 

agibsonccc

{
  "initialSchema" : {
    "Schema" : {
      "columns" : [ {
        "String" : {
          "name" : "source ip"
        }
      }, {
        "Integer" : {
          "name" : "source port"

agibsonccc

            <dependency>
                <groupId>org.codehaus.jackson</groupId>
                <artifactId>jackson-mapper-asl</artifactId>
                <version>${codehaus.jackson.version}</version>
            </dependency>
            <dependency>
                <groupId>org.codehaus.jackson</groupId>
                <artifactId>jackson-core-asl</artifactId>
                <version>${codehaus.jackson.version}</version>
            </dependency>

agibsonccc

bin/spark-submit --class io.skymind.TestGPUInvocation  \
                 --conf spark.mesos.gpus.max=1 \
                  --master mesos://"$MASTER:7077"  \
                 --deploy-mode cluster \
                  --conf spark.mesos.role=spark \
                  --conf spark.driver.memory=2G \
                   --driver-java-options "-Dlog4j.configuration=file:///home/centos/dist/log4j.properties" \
                   --executor-memory 3G \
                   --conf spark.mesos.gpu.enabled=true \

agibsonccc

ureFreeSpace(2746) called with curMem=18694915, maxMem=4042889625
16/09/07 19:10:22 INFO MemoryStore: Block broadcast_970_piece0 stored as bytes in memory (estimated size 2.7 KB, free 3.7 GB)
16/09/07 19:10:22 INFO BlockManagerInfo: Added broadcast_970_piece0 in memory on localhost:41251 (size: 2.7 KB, free: 3.8 GB)
16/09/07 19:10:22 INFO SparkContext: Created broadcast 970 from broadcast at DAGScheduler.scala:861
16/09/07 19:10:22 INFO DAGScheduler: Submitting 8 missing tasks from ResultStage 641 (MapPartitionsRDD[1987] at mapPartitions at ParameterAveragingTrainingMaster.java:543)
16/09/07 19:10:22 INFO TaskSchedulerImpl: Adding task set 641.0 with 8 tasks
16/09/07 19:10:22 INFO TaskSetManager: Starting task 0.0 in stage 641.0 (TID 5128, localhost, PROCESS_LOCAL, 2089 bytes)
16/09/07 19:10:22 INFO TaskSetManager: Starting task 1.0 in stage 641.0 (TID 5129, localhost, PROCESS_LOCAL, 2089 bytes)
16/09/07 19:10:22 INFO TaskSetManager: Starting task 2.0 in stage 641.0 (TID 5130, localhost, PROCESS_LOCAL, 2089 b

agibsonccc

15:14:25.531 [main] DEBUG o.n.j.handler.impl.CudaZeroHandler - Creating bucketID: 1
15:14:25.545 [main] DEBUG o.d.n.l.c.s.SubsamplingLayer - CudnnSubsamplingHelper successfully loaded
15:14:25.547 [main] DEBUG o.d.n.l.convolution.ConvolutionLayer - CudnnConvolutionHelper successfully loaded
15:14:25.630 [main] DEBUG o.d.n.l.c.s.SubsamplingLayer - CudnnSubsamplingHelper successfully loaded
15:14:27.120 [main] WARN  o.n.j.handler.impl.CudaZeroHandler - Out of [DEVICE] memory, host memory will be used instead: deviceId: [0], requested bytes: [216536320]
15:14:27.500 [main] WARN  o.n.j.handler.impl.CudaZeroHandler - Out of [DEVICE] memory, host memory will be used instead: deviceId: [0], requested bytes: [216536320]
15:14:27.624 [main] WARN  o.n.j.handler.impl.CudaZeroHandler - Out of [DEVICE] memory, host memory will be used instead: deviceId: [0], requested bytes: [37669632]
Exception in thread "main" java.lang.RuntimeException: CUDA error = 2
	at org.deeplearning4j.nn.layers.convolution.CudnnConvolutionHelper.