Created
July 9, 2015 02:21
-
-
Save pascaldimassimo/17097ef7a3dc391e18f0 to your computer and use it in GitHub Desktop.
Simple code to page through Solr results with httpclient and jackson
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.net.URI; | |
import org.apache.http.HttpEntity; | |
import org.apache.http.client.methods.CloseableHttpResponse; | |
import org.apache.http.client.methods.HttpGet; | |
import org.apache.http.client.utils.URIBuilder; | |
import org.apache.http.impl.client.CloseableHttpClient; | |
import org.apache.http.impl.client.HttpClients; | |
import com.fasterxml.jackson.databind.JsonNode; | |
import com.fasterxml.jackson.databind.ObjectMapper; | |
public class SolrPager { | |
private final String solrEndpoint; | |
public SolrPager(String solrEndpoint) { | |
this.solrEndpoint = solrEndpoint; | |
} | |
void page() throws Exception { | |
int rows = 1000; | |
CloseableHttpClient httpclient = null; | |
try { | |
httpclient = HttpClients.createDefault(); | |
int start = 0; | |
while (true) { | |
HttpGet httpget = new HttpGet(buildParams(start, rows)); | |
CloseableHttpResponse response = httpclient.execute(httpget); | |
try { | |
HttpEntity entity = response.getEntity(); | |
if (entity == null) { | |
break; | |
} | |
ObjectMapper mapper = new ObjectMapper(); | |
JsonNode tree = mapper.readTree(entity.getContent()); | |
JsonNode responseNode = tree.get("response"); | |
int nbdocs = 0; | |
for (JsonNode doc : responseNode.get("docs")) { | |
nbdocs++; | |
// ... do something with doc... | |
} | |
if (nbdocs < rows) { | |
break; | |
} | |
start += rows; | |
} finally { | |
response.close(); | |
} | |
} | |
} finally { | |
if (httpclient != null) { | |
httpclient.close(); | |
} | |
} | |
} | |
private URI buildParams(int start, int rows) throws Exception { | |
return new URIBuilder(new URI(solrEndpoint)) | |
.setPath("/solr/select") | |
.setParameter("q", "*:*") | |
.setParameter("fl", "id,title") | |
.setParameter("wt", "json") | |
.setParameter("start", String.valueOf(start)) | |
.setParameter("rows", String.valueOf(rows)) | |
.build(); | |
} | |
public static void main(String[] args) throws Exception { | |
new SolrPager("http://localhost:8888").page(); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment