Last active
October 12, 2022 13:59
-
-
Save timrobertson100/bd1079c5d6d4f69a8c0826dbbb5b08fd to your computer and use it in GitHub Desktop.
Read the actual taxon match cache
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.gbif.pipelines.ingest.java; | |
import org.apache.hadoop.conf.Configuration; | |
import org.apache.hadoop.hbase.HBaseConfiguration; | |
import org.apache.hadoop.hbase.TableName; | |
import org.apache.hadoop.hbase.client.Connection; | |
import org.apache.hadoop.hbase.client.ConnectionFactory; | |
import org.apache.hadoop.hbase.client.Get; | |
import org.apache.hadoop.hbase.client.HTable; | |
import org.apache.hadoop.hbase.client.Result; | |
import org.apache.hadoop.hbase.client.Table; | |
import org.apache.hadoop.hbase.util.Bytes; | |
import org.gbif.kvs.KeyValueStore; | |
import org.gbif.kvs.SaltedKeyGenerator; | |
import org.gbif.kvs.conf.CachedHBaseKVStoreConfiguration; | |
import org.gbif.kvs.hbase.HBaseKVStoreConfiguration; | |
import org.gbif.kvs.species.NameUsageMatchKVStoreFactory; | |
import org.gbif.kvs.species.SpeciesMatchRequest; | |
import org.gbif.rest.client.configuration.ChecklistbankClientsConfiguration; | |
import org.gbif.rest.client.configuration.ClientConfiguration; | |
import org.gbif.rest.client.species.NameUsageMatch; | |
import java.io.IOException; | |
public class NameLookup { | |
public static void main(String[] args) throws IOException { | |
ChecklistbankClientsConfiguration clientConfiguration = | |
ChecklistbankClientsConfiguration.builder() | |
.nameUSageClientConfiguration( | |
ClientConfiguration.builder() | |
.withBaseApiUrl("https://api.gbif.org/v1/") | |
.build()) | |
.checklistbankClientConfiguration( | |
ClientConfiguration.builder() | |
.withBaseApiUrl("https://api.gbif.org/v1/") | |
.build()) | |
.build(); | |
KeyValueStore<SpeciesMatchRequest, NameUsageMatch> store = NameUsageMatchKVStoreFactory.nameUsageMatchKVStore(CachedHBaseKVStoreConfiguration.builder() | |
.withValueColumnQualifier("j") //stores JSON data | |
.withHBaseKVStoreConfiguration(HBaseKVStoreConfiguration.builder() | |
.withTableName("name_usage_kv") //Geocode KV HBase table | |
.withColumnFamily("v") //Column in which qualifiers are stored | |
.withNumOfKeyBuckets(10) //Buckets for salted key generations | |
.withHBaseZk("c5zk1.gbif.org,c5zk2.gbif.org,c5zk3.gbif.org") //HBase Zookeeper ensemble | |
.build()).build(), | |
clientConfiguration); | |
SpeciesMatchRequest r = SpeciesMatchRequest.builder().withKingdom("Fungi").withScientificName("Fungi").build(); | |
NameUsageMatch match = store.get(r); | |
System.out.println(match); | |
Configuration config = HBaseConfiguration.create(); | |
config.set("hbase.zookeeper.quorum", "c5zk1.gbif.org,c5zk2.gbif.org,c5zk3.gbif.org"); | |
Connection connection = ConnectionFactory.createConnection(config); | |
SaltedKeyGenerator saltedKeyGenerator = new SaltedKeyGenerator(10); | |
try (Table table = connection.getTable(TableName.valueOf("name_usage_kv"))) { | |
byte[] saltedKey = saltedKeyGenerator.computeKey(Bytes.toBytes(r.getLogicalKey())); | |
System.out.println(Bytes.toString(saltedKey)); | |
Get get = new Get(saltedKey); | |
Result result = table.get(get); | |
System.out.println(Bytes.toString(result.getValue(Bytes.toBytes("v"), Bytes.toBytes("j")))); | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment