Skip to content

Instantly share code, notes, and snippets.

@pascaldimassimo
Created July 9, 2015 02:21
Show Gist options
  • Save pascaldimassimo/17097ef7a3dc391e18f0 to your computer and use it in GitHub Desktop.
Save pascaldimassimo/17097ef7a3dc391e18f0 to your computer and use it in GitHub Desktop.
Simple code to page through Solr results with httpclient and jackson
import java.net.URI;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
public class SolrPager {
private final String solrEndpoint;
public SolrPager(String solrEndpoint) {
this.solrEndpoint = solrEndpoint;
}
void page() throws Exception {
int rows = 1000;
CloseableHttpClient httpclient = null;
try {
httpclient = HttpClients.createDefault();
int start = 0;
while (true) {
HttpGet httpget = new HttpGet(buildParams(start, rows));
CloseableHttpResponse response = httpclient.execute(httpget);
try {
HttpEntity entity = response.getEntity();
if (entity == null) {
break;
}
ObjectMapper mapper = new ObjectMapper();
JsonNode tree = mapper.readTree(entity.getContent());
JsonNode responseNode = tree.get("response");
int nbdocs = 0;
for (JsonNode doc : responseNode.get("docs")) {
nbdocs++;
// ... do something with doc...
}
if (nbdocs < rows) {
break;
}
start += rows;
} finally {
response.close();
}
}
} finally {
if (httpclient != null) {
httpclient.close();
}
}
}
private URI buildParams(int start, int rows) throws Exception {
return new URIBuilder(new URI(solrEndpoint))
.setPath("/solr/select")
.setParameter("q", "*:*")
.setParameter("fl", "id,title")
.setParameter("wt", "json")
.setParameter("start", String.valueOf(start))
.setParameter("rows", String.valueOf(rows))
.build();
}
public static void main(String[] args) throws Exception {
new SolrPager("http://localhost:8888").page();
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment