Created
April 1, 2020 22:37
-
-
Save janheinrichmerker/28b1bcfe7b151bb7ebf56b4bf0821f0d to your computer and use it in GitHub Desktop.
Elasticsearch Spark extension functions for Kotlin.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package de.webis.webisstud.thesis.reimer.clueweb.graph | |
import org.apache.spark.SparkConf | |
import org.apache.spark.api.java.JavaPairRDD | |
import org.apache.spark.api.java.JavaRDD | |
import org.apache.spark.api.java.JavaSparkContext | |
import org.elasticsearch.hadoop.cfg.ConfigurationOptions.* | |
import org.elasticsearch.spark.rdd.api.java.JavaEsSpark | |
fun JavaSparkContext.esRDD(): JavaPairRDD<String, Map<String, Any>> = | |
JavaEsSpark.esRDD(this) | |
fun JavaSparkContext.esRDD(resource: String): JavaPairRDD<String, Map<String, Any>> = | |
JavaEsSpark.esRDD(this, resource) | |
fun JavaSparkContext.esRDD(resource: String, query: String): JavaPairRDD<String, Map<String, Any>> = | |
JavaEsSpark.esRDD(this, resource, query) | |
fun JavaSparkContext.esRDD(cfg: Map<String, String>) = | |
JavaEsSpark.esRDD(this, cfg) | |
fun JavaSparkContext.esJsonRDD(): JavaPairRDD<String, String> = | |
JavaEsSpark.esJsonRDD(this) | |
fun JavaSparkContext.esJsonRDD(resource: String): JavaPairRDD<String, String> = | |
JavaEsSpark.esJsonRDD(this, resource) | |
fun JavaSparkContext.esJsonRDD(resource: String, query: String): JavaPairRDD<String, String> = | |
JavaEsSpark.esJsonRDD(this, resource, query) | |
fun JavaSparkContext.esJsonRDD(cfg: Map<String, String>): JavaPairRDD<String, String> = | |
JavaEsSpark.esJsonRDD(this, cfg) | |
fun JavaRDD<*>.saveToEs(resource: String) = | |
JavaEsSpark.saveToEs(this, resource) | |
fun JavaRDD<*>.saveToEs(resource: String, cfg: Map<String, String>) = | |
JavaEsSpark.saveToEs(this, resource, cfg) | |
fun JavaRDD<*>.saveToEs(cfg: Map<String, String>) = | |
JavaEsSpark.saveToEs(this, cfg) | |
fun <K, V> JavaPairRDD<K, V>.saveToEsWithMeta(resource: String) = | |
JavaEsSpark.saveToEsWithMeta(this, resource) | |
fun <K, V> JavaPairRDD<K, V>.saveToEsWithMeta(resource: String, cfg: Map<String, String>) = | |
JavaEsSpark.saveToEsWithMeta(this, resource, cfg) | |
fun <K, V> JavaPairRDD<K, V>.saveToEsWithMeta(cfg: Map<String, String>) = | |
JavaEsSpark.saveToEsWithMeta(this, cfg) | |
fun JavaRDD<String>.saveJsonToEs(resource: String) = | |
JavaEsSpark.saveJsonToEs(this, resource) | |
fun JavaRDD<String>.saveJsonToEs(resource: String, cfg: Map<String, String>) = | |
JavaEsSpark.saveJsonToEs(this, resource, cfg) | |
fun JavaRDD<String>.saveJsonToEs(cfg: Map<String, String>) = | |
JavaEsSpark.saveJsonToEs(this, cfg) | |
fun JavaRDD<ByteArray>.saveJsonByteArrayToEs(resource: String) = | |
JavaEsSpark.saveJsonByteArrayToEs(this, resource) | |
fun JavaRDD<ByteArray>.saveJsonByteArrayToEs(resource: String, cfg: Map<String, String>) = | |
JavaEsSpark.saveJsonByteArrayToEs(this, resource, cfg) | |
fun JavaRDD<ByteArray>.saveJsonByteArrayToEs(cfg: Map<String, String>) = | |
JavaEsSpark.saveJsonByteArrayToEs(this, cfg) | |
var SparkConf.esResource: String | |
get() = get(ES_RESOURCE) | |
set(value) { | |
set(ES_RESOURCE, value) | |
} | |
var SparkConf.esResourceRead: String | |
get() = get(ES_RESOURCE_READ) | |
set(value) { | |
set(ES_RESOURCE_READ, value) | |
} | |
var SparkConf.esResourceWrite: String | |
get() = get(ES_RESOURCE_WRITE) | |
set(value) { | |
set(ES_RESOURCE_WRITE, value) | |
} | |
var SparkConf.esNodes: String | |
get() = get(ES_NODES) | |
set(value) { | |
set(ES_NODES, value) | |
} | |
var SparkConf.esPort: Int | |
get() = getInt(ES_PORT, ES_PORT_DEFAULT.toInt()) | |
set(value) { | |
set(ES_PORT, value.toString()) | |
} | |
var SparkConf.esNodesWanOnly: Boolean | |
get() = getBoolean(ES_NODES_WAN_ONLY, ES_NODES_WAN_ONLY_DEFAULT.toBoolean()) | |
set(value) { | |
set(ES_NODES_WAN_ONLY, value.toString()) | |
} | |
var SparkConf.esNodesResolveHostname: Boolean | |
get() = getBoolean(ES_NODES_RESOLVE_HOST_NAME, !esNodesWanOnly) | |
set(value) { | |
set(ES_NODES_RESOLVE_HOST_NAME, value.toString()) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment