Skip to content

Instantly share code, notes, and snippets.

@acmiyaguchi

acmiyaguchi/stage_info.json

Last active August 24, 2020 19:43
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save acmiyaguchi/10e3eff87c9b20139ea7306a1e4de644 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save acmiyaguchi/10e3eff87c9b20139ea7306a1e4de644 to your computer and use it in GitHub Desktop.
Download ZIP
Spark event logs
Raw
stage_info.json
{
"Event":"SparkListenerStageCompleted",
"Stage Info":{
"Stage ID":2,
"Stage Attempt ID":2,
"Stage Name":"json at NativeMethodAccessorImpl.java:0",
"Number of Tasks":2,
"RDD Info":[
{
"RDD ID":20,
"Name":"MapPartitionsRDD",
"Scope":"{\"id\":\"52\",\"name\":\"Exchange\"}",
"Callsite":"json at NativeMethodAccessorImpl.java:0",
"Parent IDs":[
19
],
"Storage Level":{
"Use Disk":false,
"Use Memory":false,
"Deserialized":false,
"Replication":1
},
"Barrier":false,
"Number of Partitions":2,
"Number of Cached Partitions":0,
"Memory Size":0,
"Disk Size":0
},
{
"RDD ID":17,
"Name":"MapPartitionsRDD",
"Scope":"{\"id\":\"54\",\"name\":\"WholeStageCodegen (1)\"}",
"Callsite":"json at NativeMethodAccessorImpl.java:0",
"Parent IDs":[
16
],
"Storage Level":{
"Use Disk":false,
"Use Memory":false,
"Deserialized":false,
"Replication":1
},
"Barrier":false,
"Number of Partitions":2,
"Number of Cached Partitions":0,
"Memory Size":0,
"Disk Size":0
},
{
"RDD ID":19,
"Name":"MapPartitionsRDD",
"Scope":"{\"id\":\"53\",\"name\":\"ObjectHashAggregate\"}",
"Callsite":"json at NativeMethodAccessorImpl.java:0",
"Parent IDs":[
18
],
"Storage Level":{
"Use Disk":false,
"Use Memory":false,
"Deserialized":false,
"Replication":1
},
"Barrier":false,
"Number of Partitions":2,
"Number of Cached Partitions":0,
"Memory Size":0,
"Disk Size":0
},
{
"RDD ID":18,
"Name":"MapPartitionsRDD",
"Scope":"{\"id\":\"54\",\"name\":\"WholeStageCodegen (1)\"}",
"Callsite":"json at NativeMethodAccessorImpl.java:0",
"Parent IDs":[
17
],
"Storage Level":{
"Use Disk":false,
"Use Memory":false,
"Deserialized":false,
"Replication":1
},
"Barrier":false,
"Number of Partitions":2,
"Number of Cached Partitions":0,
"Memory Size":0,
"Disk Size":0
},
{
"RDD ID":16,
"Name":"ParallelCollectionRDD",
"Scope":"{\"id\":\"54\",\"name\":\"WholeStageCodegen (1)\"}",
"Callsite":"json at NativeMethodAccessorImpl.java:0",
"Parent IDs":[
],
"Storage Level":{
"Use Disk":false,
"Use Memory":false,
"Deserialized":false,
"Replication":1
},
"Barrier":false,
"Number of Partitions":2,
"Number of Cached Partitions":0,
"Memory Size":0,
"Disk Size":0
}
],
"Parent IDs":[
],
"Details":"org.apache.spark.sql.DataFrameWriter.json(DataFrameWriter.scala:808)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:238)\njava.lang.Thread.run(Thread.java:748)",
"Submission Time":1598251923336,
"Completion Time":1598251923538,
"Accumulables":[
{
"ID":182,
"Name":"time in aggregation build",
"Value":"1226",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":158,
"Name":"shuffle write time",
"Value":"143832599",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":307,
"Name":"internal.metrics.shuffle.write.recordsWritten",
"Value":5000,
"Internal":true,
"Count Failed Values":true
},
{
"ID":289,
"Name":"internal.metrics.executorDeserializeCpuTime",
"Value":17906715,
"Internal":true,
"Count Failed Values":true
},
{
"ID":184,
"Name":"number of output rows",
"Value":"200000",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":292,
"Name":"internal.metrics.resultSize",
"Value":4290,
"Internal":true,
"Count Failed Values":true
},
{
"ID":310,
"Name":"internal.metrics.input.recordsRead",
"Value":80000,
"Internal":true,
"Count Failed Values":true
},
{
"ID":181,
"Name":"number of output rows",
"Value":"12500",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":157,
"Name":"shuffle records written",
"Value":"12500",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":148,
"Name":"data size",
"Value":"1500000",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":306,
"Name":"internal.metrics.shuffle.write.bytesWritten",
"Value":204329,
"Internal":true,
"Count Failed Values":true
},
{
"ID":288,
"Name":"internal.metrics.executorDeserializeTime",
"Value":35,
"Internal":true,
"Count Failed Values":true
},
{
"ID":156,
"Name":"shuffle bytes written",
"Value":"510767",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":291,
"Name":"internal.metrics.executorCpuTime",
"Value":234585736,
"Internal":true,
"Count Failed Values":true
},
{
"ID":183,
"Name":"duration",
"Value":"1204",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":308,
"Name":"internal.metrics.shuffle.write.writeTime",
"Value":48377168,
"Internal":true,
"Count Failed Values":true
},
{
"ID":290,
"Name":"internal.metrics.executorRunTime",
"Value":248,
"Internal":true,
"Count Failed Values":true
}
]
}
}
Raw
task_info.json
{
"Event":"SparkListenerTaskEnd",
"Stage ID":3,
"Stage Attempt ID":2,
"Task Type":"ShuffleMapTask",
"Task End Reason":{
"Reason":"Success"
},
"Task Info":{
"Task ID":88,
"Index":45,
"Attempt":0,
"Launch Time":1598251924688,
"Executor ID":"2",
"Host":"test-32-2500-n1-standard-16-0-20200823234848898312-m.c.amiyaguchi-dev.internal",
"Locality":"NODE_LOCAL",
"Speculative":false,
"Getting Result Time":0,
"Finish Time":1598251925027,
"Failed":false,
"Killed":false,
"Accumulables":[
{
"ID":169,
"Name":"shuffle write time",
"Update":"2390577",
"Value":"89303679",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":168,
"Name":"shuffle records written",
"Update":"11",
"Value":"747",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":167,
"Name":"shuffle bytes written",
"Update":"16674",
"Value":"1131208",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":159,
"Name":"data size",
"Update":"17688",
"Value":"1201176",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":175,
"Name":"duration",
"Update":"1",
"Value":"9200",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":176,
"Name":"duration",
"Update":"412",
"Value":"21768",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":177,
"Name":"number of output rows",
"Update":"11",
"Value":"747",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":178,
"Name":"duration",
"Update":"207",
"Value":"11568",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":180,
"Name":"time in aggregation build",
"Update":"2",
"Value":"1372",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":179,
"Name":"number of output rows",
"Update":"11",
"Value":"747",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":150,
"Name":"local blocks read",
"Update":"1",
"Value":"72",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":154,
"Name":"fetch wait time",
"Update":"0",
"Value":"20",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":153,
"Name":"local bytes read",
"Update":"469",
"Value":"37212",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":155,
"Name":"records read",
"Update":"22",
"Value":"1494",
"Internal":true,
"Count Failed Values":true,
"Metadata":"sql"
},
{
"ID":333,
"Name":"internal.metrics.shuffle.write.writeTime",
"Update":2390577,
"Value":52414370,
"Internal":true,
"Count Failed Values":true
},
{
"ID":332,
"Name":"internal.metrics.shuffle.write.recordsWritten",
"Update":11,
"Value":562,
"Internal":true,
"Count Failed Values":true
},
{
"ID":331,
"Name":"internal.metrics.shuffle.write.bytesWritten",
"Update":16674,
"Value":851146,
"Internal":true,
"Count Failed Values":true
},
{
"ID":330,
"Name":"internal.metrics.shuffle.read.recordsRead",
"Update":22,
"Value":1124,
"Internal":true,
"Count Failed Values":true
},
{
"ID":329,
"Name":"internal.metrics.shuffle.read.fetchWaitTime",
"Update":0,
"Value":0,
"Internal":true,
"Count Failed Values":true
},
{
"ID":328,
"Name":"internal.metrics.shuffle.read.localBytesRead",
"Update":469,
"Value":23083,
"Internal":true,
"Count Failed Values":true
},
{
"ID":327,
"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk",
"Update":0,
"Value":0,
"Internal":true,
"Count Failed Values":true
},
{
"ID":326,
"Name":"internal.metrics.shuffle.read.remoteBytesRead",
"Update":0,
"Value":0,
"Internal":true,
"Count Failed Values":true
},
{
"ID":325,
"Name":"internal.metrics.shuffle.read.localBlocksFetched",
"Update":1,
"Value":46,
"Internal":true,
"Count Failed Values":true
},
{
"ID":324,
"Name":"internal.metrics.shuffle.read.remoteBlocksFetched",
"Update":0,
"Value":0,
"Internal":true,
"Count Failed Values":true
},
{
"ID":321,
"Name":"internal.metrics.diskBytesSpilled",
"Update":0,
"Value":0,
"Internal":true,
"Count Failed Values":true
},
{
"ID":320,
"Name":"internal.metrics.memoryBytesSpilled",
"Update":0,
"Value":0,
"Internal":true,
"Count Failed Values":true
},
{
"ID":317,
"Name":"internal.metrics.resultSize",
"Update":3971,
"Value":182709,
"Internal":true,
"Count Failed Values":true
},
{
"ID":316,
"Name":"internal.metrics.executorCpuTime",
"Update":12942280,
"Value":735481034,
"Internal":true,
"Count Failed Values":true
},
{
"ID":315,
"Name":"internal.metrics.executorRunTime",
"Update":322,
"Value":9451,
"Internal":true,
"Count Failed Values":true
},
{
"ID":314,
"Name":"internal.metrics.executorDeserializeCpuTime",
"Update":4352031,
"Value":227381420,
"Internal":true,
"Count Failed Values":true
},
{
"ID":313,
"Name":"internal.metrics.executorDeserializeTime",
"Update":4,
"Value":615,
"Internal":true,
"Count Failed Values":true
}
]
},
"Task Executor Metrics":{
"JVMHeapMemory":0,
"JVMOffHeapMemory":0,
"OnHeapExecutionMemory":0,
"OffHeapExecutionMemory":0,
"OnHeapStorageMemory":0,
"OffHeapStorageMemory":0,
"OnHeapUnifiedMemory":0,
"OffHeapUnifiedMemory":0,
"DirectPoolMemory":0,
"MappedPoolMemory":0,
"ProcessTreeJVMVMemory":0,
"ProcessTreeJVMRSSMemory":0,
"ProcessTreePythonVMemory":0,
"ProcessTreePythonRSSMemory":0,
"ProcessTreeOtherVMemory":0,
"ProcessTreeOtherRSSMemory":0,
"MinorGCCount":0,
"MinorGCTime":0,
"MajorGCCount":0,
"MajorGCTime":0
},
"Task Metrics":{
"Executor Deserialize Time":4,
"Executor Deserialize CPU Time":4352031,
"Executor Run Time":322,
"Executor CPU Time":12942280,
"Peak Execution Memory":0,
"Result Size":3971,
"JVM GC Time":0,
"Result Serialization Time":0,
"Memory Bytes Spilled":0,
"Disk Bytes Spilled":0,
"Shuffle Read Metrics":{
"Remote Blocks Fetched":0,
"Local Blocks Fetched":1,
"Fetch Wait Time":0,
"Remote Bytes Read":0,
"Remote Bytes Read To Disk":0,
"Local Bytes Read":469,
"Total Records Read":22
},
"Shuffle Write Metrics":{
"Shuffle Bytes Written":16674,
"Shuffle Write Time":2390577,
"Shuffle Records Written":11
},
"Input Metrics":{
"Bytes Read":0,
"Records Read":0
},
"Output Metrics":{
"Bytes Written":0,
"Records Written":0
},
"Updated Blocks":[
]
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment