Skip to content

Instantly share code, notes, and snippets.

@RussellSpitzer
Last active August 29, 2015 14:13
Show Gist options
  • Save RussellSpitzer/e36220a3d73e64e3ec80 to your computer and use it in GitHub Desktop.
Save RussellSpitzer/e36220a3d73e64e3ec80 to your computer and use it in GitHub Desktop.
Loading a table with java cassandra context and registering it in the hive context
package test;
/**
* Created by russellspitzer on 12/4/14.
*/
import java.io.Serializable;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.api.java.Row;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.api.java.JavaSchemaRDD;
import org.apache.spark.sql.hive.api.java.JavaHiveContext;
import static com.datastax.spark.connector.japi.CassandraJavaUtil.*;
public class sparksql {
public static class AsciiRow implements Serializable
{
private String pkey;
private String ckey1;
private String data1;
public String getPkey()
{
return pkey;
}
public void setPkey(String pkey)
{
this.pkey = pkey;
}
public String getCkey1()
{
return ckey1;
}
public void setCkey1(String ckey1)
{
this.ckey1 = ckey1;
}
public String getData1()
{
return data1;
}
public void setData1(String data1)
{
this.data1 = data1;
}
}
public static void main(String[] args) {
// create a new configuration
SparkConf conf = new SparkConf().setAppName("TestForJavaSparkSql");
// create a Spark context
JavaSparkContext jsc = new JavaSparkContext(conf);
JavaRDD<AsciiRow> rdd = javaFunctions(jsc).cassandraTable("ascii_ks", "ascii_cs", mapRowTo(AsciiRow.class)).where("pkey = 'One'");
// create a Hive Spark SQL context
JavaHiveContext sqlContext = new JavaHiveContext(jsc);
JavaSchemaRDD schemaAscii = sqlContext.applySchema(rdd, AsciiRow.class);
schemaAscii.registerTempTable("ascii");
JavaSchemaRDD result = sqlContext.sql("SELECT pkey FROM ascii");
List<String> pkeyNames = result.map(new Function<Row, String>() {
public String call(Row row) {
return "Name: " + row.getString(0);
}
}).collect();
System.out.println("Pkeys");
for( String name : pkeyNames){
System.out.println(name);
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment