nd4j_fail.log

#
# A fatal error has been detected by the Java Runtime Environment:
#
#  SIGSEGV (0xb) at pc=0x00007f549ae5b182, pid=16918, tid=140007211910912
#
# JRE version: Java(TM) SE Runtime Environment (8.0_91-b14) (build 1.8.0_91-b14)
# Java VM: Java HotSpot(TM) 64-Bit Server VM (25.91-b14 mixed mode linux-amd64 )
# Problematic frame:
# C  [libopenblas.so.0+0x1143182]  sgemm_incopy_SANDYBRIDGE+0x8d2
#
# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again
#
# If you would like to submit a bug report, please visit:
#   http://bugreport.java.com/bugreport/crash.jsp
# The crash happened outside the Java Virtual Machine in native code.
# See problematic frame for where to report the bug.
#

---------------  T H R E A D  ---------------

Current thread (0x00007f7200f4d000):  JavaThread "ParallelWrapper training thread 0" daemon [_thread_in_native, id=17010, stack(0x00007f55f811c000,0x00007f55f821d000)]

siginfo: si_signo: 11 (SIGSEGV), si_code: 2 (SEGV_ACCERR), si_addr: 0x00007f50c0f5e000

Registers:
RAX=0x0000000000000350, RBX=0x0000000000000000, RCX=0x0000000000000056, RDX=0x00007f53b440ed40
RSP=0x00007f55f8197bb0, RBP=0x00007f50c0f5dcac, RSI=0x00007f50c0df53f0, RDI=0x00007f50c10c6568
R8 =0x00007f50c122ee24, R9 =0x0000000000000000, R10=0x0000000000000000, R11=0x0000000000000000
R12=0x0000000000000000, R13=0x0000000000000000, R14=0x0000000000000000, R15=0x0000000000000000
RIP=0x00007f549ae5b182, EFLAGS=0x0000000000010206, CSGSFS=0x0000000000000033, ERR=0x0000000000000004
  TRAPNO=0x000000000000000e

Top of Stack: (sp=0x00007f55f8197bb0)
0x00007f55f8197bb0:   0000000000000000 0000000000000000
0x00007f55f8197bc0:   0000000000000000 0000000000000000
0x00007f55f8197bd0:   0000000000000000 0000000000000000
0x00007f55f8197be0:   0000000000000000 392aaf653a85c85d
0x00007f55f8197bf0:   3afcdebfba20223e b94f276e3a489205
0x00007f55f8197c00:   00000000005a22f0 00000000001688bc
0x00007f55f8197c10:   00007f5415bae708 00007f5415a45e4c
0x00007f55f8197c20:   0000000000000000 0000000000000060
0x00007f55f8197c30:   0000000000000000 00007f54158dcf90
0x00007f55f8197c40:   0000000000000180 00007f50c02b0e10
0x00007f55f8197c50:   000000000005a22f 0000000000000008
0x00007f55f8197c60:   000000000000000d 00007f55f821a8b0
0x00007f55f8197c70:   0000000000000000 00007f55f8197e70
0x00007f55f8197c80:   00007f549bf967a0 0000000000000001
0x00007f55f8197c90:   0000000000000010 00007f55f821a878
0x00007f55f8197ca0:   0000000000000000 00007f5499f0db9e
0x00007f55f8197cb0:   0000000000000064 00007f53b000ace0
0x00007f55f8197cc0:   0000000000000064 0000000000000000
0x00007f55f8197cd0:   0000000000000000 00007f53b000dee0
0x00007f55f8197ce0:   000000000000000d 0000000000000064
0x00007f55f8197cf0:   0000000000001900 0000000000000180
0x00007f55f8197d00:   0000000000000180 00007f53b440b000
0x00007f55f8197d10:   00007f55f8197e70 00007f55f821ad6c
0x00007f55f8197d20:   000000000000000d 0000000000000000
0x00007f55f8197d30:   00007f55f821a870 0000000000000000
0x00007f55f8197d40:   00007f55f821ac90 000000000000000d
0x00007f55f8197d50:   00007f53b000ace0 00007f55f8197e70
0x00007f55f8197d60:   0000000000006780 0000000000000000
0x00007f55f8197d70:   000000000005a22f 000000000000000d
0x00007f55f8197d80:   00007f5412bb2010 00007f50c0297010
0x00007f55f8197d90:   0000000000000020 0000000000000000
0x00007f55f8197da0:   000000000005a22f 00007f55f8197e70 

Instructions: (pc=0x00007f549ae5b182)
0x00007f549ae5b162:   89 ca 48 89 4c 24 b0 31 c0 48 89 5c 24 98 4c 89
0x00007f549ae5b172:   e1 0f 1f 44 00 00 44 8b 5c 06 04 44 8b 74 05 00
0x00007f549ae5b182:   44 8b 54 05 04 44 8b 2c 07 44 8b 4c 07 04 45 8b
0x00007f549ae5b192:   24 00 41 8b 5c 00 04 44 8b 3c 06 48 83 c0 08 44 

Register to memory mapping:

RAX=0x0000000000000350 is an unknown value
RBX=0x0000000000000000 is an unknown value
RCX=0x0000000000000056 is an unknown value
RDX=0x00007f53b440ed40 is an unknown value
RSP=0x00007f55f8197bb0 is pointing into the stack for thread: 0x00007f7200f4d000
RBP=0x00007f50c0f5dcac is an unknown value
RSI=0x00007f50c0df53f0 is an unknown value
RDI=0x00007f50c10c6568 is an unknown value
R8 =0x00007f50c122ee24 is an unknown value
R9 =0x0000000000000000 is an unknown value
R10=0x0000000000000000 is an unknown value
R11=0x0000000000000000 is an unknown value
R12=0x0000000000000000 is an unknown value
R13=0x0000000000000000 is an unknown value
R14=0x0000000000000000 is an unknown value
R15=0x0000000000000000 is an unknown value


Stack: [0x00007f55f811c000,0x00007f55f821d000],  sp=0x00007f55f8197bb0,  free space=494k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
C  [libopenblas.so.0+0x1143182]  sgemm_incopy_SANDYBRIDGE+0x8d2

Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)
j  org.bytedeco.javacpp.openblas.cblas_sgemm(IIIIIIFLorg/bytedeco/javacpp/FloatPointer;ILorg/bytedeco/javacpp/FloatPointer;IFLorg/bytedeco/javacpp/FloatPointer;I)V+0
j  org.nd4j.linalg.cpu.nativecpu.blas.CpuLevel3.sgemm(CCCIIIFLorg/nd4j/linalg/api/ndarray/INDArray;ILorg/nd4j/linalg/api/ndarray/INDArray;IFLorg/nd4j/linalg/api/ndarray/INDArray;I)V+80
j  org.nd4j.linalg.api.blas.impl.BaseLevel3.gemm(CCCDLorg/nd4j/linalg/api/ndarray/INDArray;Lorg/nd4j/linalg/api/ndarray/INDArray;DLorg/nd4j/linalg/api/ndarray/INDArray;)V+280
j  org.nd4j.linalg.api.ndarray.BaseNDArray.mmuli(Lorg/nd4j/linalg/api/ndarray/INDArray;Lorg/nd4j/linalg/api/ndarray/INDArray;)Lorg/nd4j/linalg/api/ndarray/INDArray;+330
j  org.deeplearning4j.nn.layers.feedforward.autoencoder.AutoEncoder.encode(Lorg/nd4j/linalg/api/ndarray/INDArray;ZLorg/deeplearning4j/nn/workspace/LayerWorkspaceMgr;)Lorg/nd4j/linalg/api/ndarray/INDArray;+58
j  org.deeplearning4j.nn.layers.feedforward.autoencoder.AutoEncoder.activate(Lorg/nd4j/linalg/api/ndarray/INDArray;ZLorg/deeplearning4j/nn/workspace/LayerWorkspaceMgr;)Lorg/nd4j/linalg/api/ndarray/INDArray;+10
j  org.deeplearning4j.nn.multilayer.MultiLayerNetwork.ffToLayerActivationsInWs(ILorg/deeplearning4j/nn/api/FwdPassType;ZLorg/nd4j/linalg/api/ndarray/INDArray;Lorg/nd4j/linalg/api/ndarray/INDArray;Lorg/nd4j/linalg/api/ndarray/INDArray;)Ljava/util/List;+328
j  org.deeplearning4j.nn.multilayer.MultiLayerNetwork.computeGradientAndScore()V+188
j  org.deeplearning4j.nn.multilayer.MultiLayerNetwork.computeGradientAndScore(Lorg/deeplearning4j/nn/workspace/LayerWorkspaceMgr;)V+1
j  org.deeplearning4j.optimize.solvers.BaseOptimizer.gradientAndScore(Lorg/deeplearning4j/nn/workspace/LayerWorkspaceMgr;)Lorg/nd4j/linalg/primitives/Pair;+13
j  org.deeplearning4j.optimize.solvers.StochasticGradientDescent.optimize(Lorg/deeplearning4j/nn/workspace/LayerWorkspaceMgr;)Z+2
j  org.deeplearning4j.optimize.Solver.optimize(Lorg/deeplearning4j/nn/workspace/LayerWorkspaceMgr;)V+9
j  org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(Lorg/nd4j/linalg/api/ndarray/INDArray;Lorg/nd4j/linalg/api/ndarray/INDArray;Lorg/nd4j/linalg/api/ndarray/INDArray;Lorg/nd4j/linalg/api/ndarray/INDArray;)V+286
j  org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(Lorg/nd4j/linalg/dataset/api/DataSet;)V+25
j  org.deeplearning4j.parallelism.trainer.DefaultTrainer.fit(Lorg/nd4j/linalg/dataset/api/DataSet;)V+54
j  org.deeplearning4j.parallelism.trainer.DefaultTrainer.run()V+591
j  java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V+95
j  java.util.concurrent.ThreadPoolExecutor$Worker.run()V+5
j  java.lang.Thread.run()V+11
v  ~StubRoutines::call_stub

---------------  P R O C E S S  ---------------

Java Threads: ( => current thread )
  0x00007f7201174000 JavaThread "ADSI prefetch thread" daemon [_thread_blocked, id=17026, stack(0x00007f53b640b000,0x00007f53b650c000)]
  0x00007f7201172000 JavaThread "ParallelWrapper training thread 15" daemon [_thread_blocked, id=17025, stack(0x00007f53b6b0c000,0x00007f53b6c0d000)]
  0x00007f7201170800 JavaThread "ParallelWrapper training thread 14" daemon [_thread_blocked, id=17024, stack(0x00007f53b6c0d000,0x00007f53b6d0e000)]
  0x00007f7200ee8000 JavaThread "ParallelWrapper training thread 13" daemon [_thread_blocked, id=17023, stack(0x00007f53b6d0e000,0x00007f53b6e0f000)]
  0x00007f7200ee6000 JavaThread "ParallelWrapper training thread 12" daemon [_thread_blocked, id=17022, stack(0x00007f53b6e0f000,0x00007f53b6f10000)]
  0x00007f7200ee4000 JavaThread "ParallelWrapper training thread 11" daemon [_thread_blocked, id=17021, stack(0x00007f53b6f10000,0x00007f53b7011000)]
  0x00007f7200ee2000 JavaThread "ParallelWrapper training thread 10" daemon [_thread_blocked, id=17020, stack(0x00007f53b7011000,0x00007f53b7112000)]
  0x00007f720117e000 JavaThread "ParallelWrapper training thread 9" daemon [_thread_blocked, id=17019, stack(0x00007f53b7112000,0x00007f53b7213000)]
  0x00007f720117c000 JavaThread "ParallelWrapper training thread 8" daemon [_thread_blocked, id=17018, stack(0x00007f53b7213000,0x00007f53b7314000)]
  0x00007f720117a800 JavaThread "ParallelWrapper training thread 7" daemon [_thread_blocked, id=17017, stack(0x00007f550003d000,0x00007f550013e000)]
  0x00007f7201178800 JavaThread "ParallelWrapper training thread 6" daemon [_thread_in_native, id=17016, stack(0x00007f550013e000,0x00007f550023f000)]
  0x00007f7200678000 JavaThread "ParallelWrapper training thread 5" daemon [_thread_blocked, id=17015, stack(0x00007f550023f000,0x00007f5500340000)]
  0x00007f7200676000 JavaThread "ParallelWrapper training thread 4" daemon [_thread_in_native, id=17014, stack(0x00007f5500340000,0x00007f5500441000)]
  0x00007f7200674800 JavaThread "ParallelWrapper training thread 3" daemon [_thread_blocked, id=17013, stack(0x00007f5500441000,0x00007f5500542000)]
  0x00007f7200672800 JavaThread "ParallelWrapper training thread 2" daemon [_thread_blocked, id=17012, stack(0x00007f5500542000,0x00007f5500643000)]
  0x00007f7200f4e000 JavaThread "ParallelWrapper training thread 1" daemon [_thread_blocked, id=17011, stack(0x00007f55f801b000,0x00007f55f811c000)]
=>0x00007f7200f4d000 JavaThread "ParallelWrapper training thread 0" daemon [_thread_in_native, id=17010, stack(0x00007f55f811c000,0x00007f55f821d000)]
  0x00007f720128a000 JavaThread "NativeRandomDeallocator thread 0" daemon [_thread_blocked, id=17002, stack(0x00007f55f841d000,0x00007f55f851e000)]
  0x00007f7200d31000 JavaThread "JavaCPP Deallocator" daemon [_thread_blocked, id=16970, stack(0x00007f55f9daa000,0x00007f55f9eab000)]
  0x00007f7200877800 JavaThread "Workspace deallocator thread" daemon [_thread_blocked, id=16969, stack(0x00007f55f9eab000,0x00007f55f9fac000)]
  0x00007f7200f7a000 JavaThread "main-EventThread" daemon [_thread_blocked, id=16967, stack(0x00007f55fb6d1000,0x00007f55fb7d2000)]
  0x00007f7200f79800 JavaThread "main-SendThread(kcs-evolution.know.know-center.at:10000)" daemon [_thread_in_native, id=16966, stack(0x00007f560126b000,0x00007f560136c000)]
  0x00007f7200431800 JavaThread "Service Thread" daemon [_thread_blocked, id=16962, stack(0x00007f560234a000,0x00007f560244b000)]
  0x00007f720042c000 JavaThread "C1 CompilerThread14" daemon [_thread_blocked, id=16961, stack(0x00007f560244b000,0x00007f560254c000)]
  0x00007f720042a000 JavaThread "C1 CompilerThread13" daemon [_thread_blocked, id=16960, stack(0x00007f560254c000,0x00007f560264d000)]
  0x00007f7200428000 JavaThread "C1 CompilerThread12" daemon [_thread_blocked, id=16959, stack(0x00007f560264d000,0x00007f560274e000)]
  0x00007f7200426800 JavaThread "C1 CompilerThread11" daemon [_thread_blocked, id=16958, stack(0x00007f560274e000,0x00007f560284f000)]
  0x00007f7200424000 JavaThread "C1 CompilerThread10" daemon [_thread_blocked, id=16957, stack(0x00007f560284f000,0x00007f5602950000)]
  0x00007f7200422000 JavaThread "C2 CompilerThread9" daemon [_thread_blocked, id=16956, stack(0x00007f5602950000,0x00007f5602a51000)]
  0x00007f7200420000 JavaThread "C2 CompilerThread8" daemon [_thread_blocked, id=16955, stack(0x00007f5602a51000,0x00007f5602b52000)]
  0x00007f720041d800 JavaThread "C2 CompilerThread7" daemon [_thread_blocked, id=16954, stack(0x00007f5602b52000,0x00007f5602c53000)]
  0x00007f720041b800 JavaThread "C2 CompilerThread6" daemon [_thread_blocked, id=16953, stack(0x00007f5602c53000,0x00007f5602d54000)]
  0x00007f7200419800 JavaThread "C2 CompilerThread5" daemon [_thread_blocked, id=16952, stack(0x00007f5602d54000,0x00007f5602e55000)]
  0x00007f720040f000 JavaThread "C2 CompilerThread4" daemon [_thread_blocked, id=16951, stack(0x00007f5602e55000,0x00007f5602f56000)]
  0x00007f720040d000 JavaThread "C2 CompilerThread3" daemon [_thread_blocked, id=16950, stack(0x00007f5602f56000,0x00007f5603057000)]
  0x00007f720040b000 JavaThread "C2 CompilerThread2" daemon [_thread_blocked, id=16949, stack(0x00007f5603057000,0x00007f5603158000)]
  0x00007f7200409000 JavaThread "C2 CompilerThread1" daemon [_thread_blocked, id=16948, stack(0x00007f5603158000,0x00007f5603259000)]
  0x00007f7200406000 JavaThread "C2 CompilerThread0" daemon [_thread_blocked, id=16947, stack(0x00007f5603259000,0x00007f560335a000)]
  0x00007f7200404800 JavaThread "Signal Dispatcher" daemon [_thread_blocked, id=16946, stack(0x00007f560335a000,0x00007f560345b000)]
  0x00007f72003d1800 JavaThread "Finalizer" daemon [_thread_blocked, id=16945, stack(0x00007f56035f6000,0x00007f56036f7000)]
  0x00007f72003cd000 JavaThread "Reference Handler" daemon [_thread_blocked, id=16944, stack(0x00007f56036f7000,0x00007f56037f8000)]
  0x00007f7200009800 JavaThread "main" [_thread_blocked, id=16919, stack(0x00007f7207d81000,0x00007f7207e82000)]

Other Threads:
  0x00007f72003c5000 VMThread [stack: 0x00007f56037f8000,0x00007f56038f9000] [id=16943]
  0x00007f7200434800 WatcherThread [stack: 0x00007f5602249000,0x00007f560234a000] [id=16963]

VM state:not at safepoint (normal execution)

VM Mutex/Monitor currently owned by a thread: None

Heap:
 PSYoungGen      total 716288K, used 481048K [0x00007f68c6580000, 0x00007f6911080000, 0x00007f71f1000000)
  eden space 629248K, 76% used [0x00007f68c6580000,0x00007f68e3b46318,0x00007f68ecc00000)
  from space 87040K, 0% used [0x00007f690bb80000,0x00007f690bb80000,0x00007f6911080000)
  to   space 87040K, 0% used [0x00007f6906680000,0x00007f6906680000,0x00007f690bb80000)
 ParOldGen       total 1701376K, used 101559K [0x00007f5671000000, 0x00007f56d8d80000, 0x00007f68c6580000)
  object space 1701376K, 5% used [0x00007f5671000000,0x00007f567732ddc0,0x00007f56d8d80000)
 Metaspace       used 50276K, capacity 52446K, committed 52480K, reserved 53248K

Card table byte_map: [0x00007f56633ff000,0x00007f5671000000] byte_map_base: 0x00007f16b8077000

Marking Bits: (ParMarkBitMap*) 0x00007f720727cd60
 Begin Bits: [0x00007f5510000000, 0x00007f557e000000)
 End Bits:   [0x00007f557e000000, 0x00007f55ec000000)

Polling page: 0x00007f7207e9c000

CodeCache: size=245760Kb used=17374Kb max_used=17385Kb free=228385Kb
 bounds [0x00007f71f1000000, 0x00007f71f2140000, 0x00007f7200000000]
 total_blobs=5299 nmethods=4226 adapters=982
 compilation: enabled

Compilation events (10 events):
Event: 113.240 Thread 0x00007f7200426800 4914       1       org.nd4j.linalg.api.buffer.BaseDataBuffer::setIndexer (6 bytes)
Event: 113.241 Thread 0x00007f7200426800 nmethod 4914 0x00007f71f1fb56d0 code [0x00007f71f1fb5820, 0x00007f71f1fb5950]
Event: 114.007 Thread 0x00007f720042a000 4915       3       java.lang.Double::doubleToLongBits (33 bytes)
Event: 114.008 Thread 0x00007f720042a000 nmethod 4915 0x00007f71f169d450 code [0x00007f71f169d5c0, 0x00007f71f169d7b0]
Event: 114.078 Thread 0x00007f720042c000 4916       3       java.util.concurrent.atomic.AtomicInteger::<init> (10 bytes)
Event: 114.078 Thread 0x00007f720042c000 nmethod 4916 0x00007f71f19c6c10 code [0x00007f71f19c6d80, 0x00007f71f19c6f90]
Event: 114.092 Thread 0x00007f7200424000 4917       3       org.nd4j.linalg.cpu.nativecpu.workspace.CpuWorkspace::clearExternalAllocations (142 bytes)
Event: 114.093 Thread 0x00007f7200424000 nmethod 4917 0x00007f71f19c4d90 code [0x00007f71f19c5040, 0x00007f71f19c6498]
Event: 114.604 Thread 0x00007f7200420000 4918       4       org.nd4j.linalg.api.ndarray.BaseNDArray::isAttached (76 bytes)