JoshRosen

import java.lang.reflect.{Modifier, Field}

import com.google.common.collect.Sets

import scala.collection.mutable
import scala.collection.JavaConversions._

/**
 * Generates GraphViz DOT files for visualizing Java object graphs.
 */

JoshRosen

JoshRosen

  #!/usr/bin/env python
  # -*- coding: utf-8 -*-
  
  #
  # Licensed to the Apache Software Foundation (ASF) under one
  # or more contributor license agreements.  See the NOTICE file
  # distributed with this work for additional information
  # regarding copyright ownership.  The ASF licenses this file
  # to you under the Apache License, Version 2.0 (the
  # "License"); you may not use this file except in compliance

JoshRosen

Method<InternalName(org/apache/spark/sql/api/java/JavaSQLContext)/baseRelationToSchemaRDD: MethodDescriptor((Lorg/apache/spark/sql/sources/BaseRelation;)Lorg/apache/spark/sql/api/java/JavaSchemaRDD;)>
Method<InternalName(org/apache/spark/sql/catalyst/types/StructField$)/apply$default$4: MethodDescriptor(()Lorg/apache/spark/sql/catalyst/util/Metadata;)>
Method<InternalName(org/apache/spark/sql/SQLContext)/baseRelationToSchemaRDD: MethodDescriptor((Lorg/apache/spark/sql/sources/BaseRelation;)Lorg/apache/spark/sql/SchemaRDD;)>
Method<InternalName(org/apache/spark/sql/catalyst/types/StructField$)/apply: MethodDescriptor((Ljava/lang/String;Lorg/apache/spark/sql/catalyst/types/DataType;ZLorg/apache/spark/sql/catalyst/util/Metadata;)Lorg/apache/spark/sql/catalyst/types/StructField;)>
Method<InternalName(org/apache/spark/sql/sources/TableScan)/<init>: MethodDescriptor(()V)>

JoshRosen

import java.io._

import org.xerial.snappy.SnappyOutputStream


object Main {
  def main(args: Array[String]) {
    val blockSize = sys.env.getOrElse("BLOCK_SIZE", "32768").toInt
    val resetFrequency = sys.env.getOrElse("RESET_FREQUENCY", "100").toInt
    val byteArrayOutputstream = new ByteArrayOutputStream()

JoshRosen

package org.apache.spark.sql

import org.apache.spark.unsafe.memory.{MemoryAllocator, TaskMemoryManager, ExecutorMemoryManager}

import scala.util.Random

import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.types._

/**

JoshRosen

package org.apache.spark.sql


import org.apache.spark.sql.catalyst.expressions.{UnsafeFixedWidthAggregationMap, SpecificMutableRow, MutableRow, GenericRow}
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.memory.MemoryAllocator
import org.openjdk.jmh.annotations._

import scala.util.Random

JoshRosen

import java.io.{PrintWriter, FileOutputStream, File}

import org.apache.spark._
import org.apache.spark.executor.TaskMetrics
import org.apache.spark.rdd.{RDD, ShuffledRDD}
import org.apache.spark.scheduler.{SparkListenerStageSubmitted, SparkListenerTaskEnd, SparkListener}

/**
 * Extends [[ShuffledRDD]] to skip the shuffle fetching.
 */

JoshRosen

package org.apache.spark.sql

import org.apache.spark.unsafe.PlatformDependent
import org.apache.spark.unsafe.map.BytesToBytesMap
import org.apache.spark.unsafe.memory.{MemoryAllocator, TaskMemoryManager, ExecutorMemoryManager}

import scala.util.Random

/**
 * This benchmark measures the time to iterate over a BytesToBytesMap.

JoshRosen

package org.apache.spark.shuffle.unsafe;

import org.apache.spark.JavaAPISuite;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;

@RunWith(Suite.class)
@Suite.SuiteClasses({
  UnsafeShuffleWriterSuite.class,
  JavaAPISuite.class