Skip to content

Instantly share code, notes, and snippets.

@hukl
Created May 24, 2011 16:24
Show Gist options
  • Save hukl/989055 to your computer and use it in GitHub Desktop.
Save hukl/989055 to your computer and use it in GitHub Desktop.
int cursor = 0;
SearchResponse searchResponse = new SearchRequestBuilder(client)
.setIndices(alias)
.setSearchType(SearchType.SCAN)
.setQuery(matchAllQuery())
.setSize(10)
.setFrom(cursor)
.setScroll(TimeValue.timeValueMinutes(10))
.execute()
.actionGet();
// start scrolling, until we get no results
do {
searchResponse = client.prepareSearchScroll(searchResponse.scrollId()).setScroll(TimeValue.timeValueMinutes(10)).execute().actionGet();
BulkRequestBuilder bulkRequest = new BulkRequestBuilder(client);
for (SearchHit hit : searchResponse.hits()) {
// batch up 10k+1
if(cursor % 10000 == 0) {
bulkRequest.add(client.prepareIndex(toIndex, hit.getType(), hit.getId()).setSource(hit.getSource()));
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
// process failures by iterating through each bulk response item
System.out.println(bulkResponse.buildFailureMessage());
}
bulkRequest = new BulkRequestBuilder(client);
} else {
bulkRequest.add(client.prepareIndex(toIndex, hit.getType(), hit.getId()).setSource(hit.getSource()));
}
cursor++;
}
} while(searchResponse.hits().hits().length != 0);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment