Last active
August 29, 2015 14:13
-
-
Save RussellSpitzer/e36220a3d73e64e3ec80 to your computer and use it in GitHub Desktop.
Loading a table with java cassandra context and registering it in the hive context
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package test; | |
/** | |
* Created by russellspitzer on 12/4/14. | |
*/ | |
import java.io.Serializable; | |
import java.util.List; | |
import org.apache.spark.SparkConf; | |
import org.apache.spark.api.java.JavaSparkContext; | |
import org.apache.spark.api.java.function.Function; | |
import org.apache.spark.sql.api.java.Row; | |
import org.apache.spark.api.java.JavaRDD; | |
import org.apache.spark.sql.api.java.JavaSchemaRDD; | |
import org.apache.spark.sql.hive.api.java.JavaHiveContext; | |
import static com.datastax.spark.connector.japi.CassandraJavaUtil.*; | |
public class sparksql { | |
public static class AsciiRow implements Serializable | |
{ | |
private String pkey; | |
private String ckey1; | |
private String data1; | |
public String getPkey() | |
{ | |
return pkey; | |
} | |
public void setPkey(String pkey) | |
{ | |
this.pkey = pkey; | |
} | |
public String getCkey1() | |
{ | |
return ckey1; | |
} | |
public void setCkey1(String ckey1) | |
{ | |
this.ckey1 = ckey1; | |
} | |
public String getData1() | |
{ | |
return data1; | |
} | |
public void setData1(String data1) | |
{ | |
this.data1 = data1; | |
} | |
} | |
public static void main(String[] args) { | |
// create a new configuration | |
SparkConf conf = new SparkConf().setAppName("TestForJavaSparkSql"); | |
// create a Spark context | |
JavaSparkContext jsc = new JavaSparkContext(conf); | |
JavaRDD<AsciiRow> rdd = javaFunctions(jsc).cassandraTable("ascii_ks", "ascii_cs", mapRowTo(AsciiRow.class)).where("pkey = 'One'"); | |
// create a Hive Spark SQL context | |
JavaHiveContext sqlContext = new JavaHiveContext(jsc); | |
JavaSchemaRDD schemaAscii = sqlContext.applySchema(rdd, AsciiRow.class); | |
schemaAscii.registerTempTable("ascii"); | |
JavaSchemaRDD result = sqlContext.sql("SELECT pkey FROM ascii"); | |
List<String> pkeyNames = result.map(new Function<Row, String>() { | |
public String call(Row row) { | |
return "Name: " + row.getString(0); | |
} | |
}).collect(); | |
System.out.println("Pkeys"); | |
for( String name : pkeyNames){ | |
System.out.println(name); | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment