To generate a large report I was required to retrieve a significant set of results from elasticsearch (60,000+ documents). For this I created a node.js script that paged through the results. I have modified that script to allow the paging algorithm to be used in the browser.
Last active
January 2, 2016 16:18
-
-
Save grahamjenson/8328812 to your computer and use it in GitHub Desktop.
Paging Elasticsearch Algorithm
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <html> | |
| <meta charset="utf-8"> | |
| <body> | |
| <link rel="stylesheet" href="https://raw.github.com/olton/Metro-UI-CSS/master/css/metro-bootstrap.css"> | |
| <style> | |
| /* tile width 130 260 390 520 650 780 910 1040*/ | |
| @media screen and (min-width: 260) { | |
| #list { | |
| width: 260px; | |
| } | |
| #list :nth-child(4n+1), #list :nth-child(4n+2){ | |
| float: left; | |
| } | |
| #list :nth-child(4n+3), #list :nth-child(4n+4) { | |
| float: right; | |
| } | |
| } | |
| @media screen and (min-width: 390px) { | |
| #list { | |
| width: 390px; | |
| } | |
| #list :nth-child(6n+1), #list :nth-child(6n+2), #list :nth-child(6n+3){ | |
| float: left; | |
| } | |
| #list :nth-child(6n+4), #list :nth-child(6n+5), #list :nth-child(6n+6) { | |
| float: right; | |
| } | |
| } | |
| @media screen and (min-width: 520px) { | |
| #list { | |
| width: 520px; | |
| } | |
| #list :nth-child(8n+1), #list :nth-child(8n+2), #list :nth-child(8n+3), #list :nth-child(8n+4){ | |
| float: left; | |
| } | |
| #list :nth-child(8n+5), #list :nth-child(8n+6), #list :nth-child(8n+7), #list :nth-child(8n+8) { | |
| float: right; | |
| } | |
| } | |
| @media screen and (min-width: 650px) { | |
| #list { | |
| width: 650px; | |
| } | |
| #list :nth-child(10n+1), #list :nth-child(10n+2), #list :nth-child(10n+3), #list :nth-child(10n+4), #list :nth-child(10n+5){ | |
| float: left; | |
| } | |
| #list :nth-child(10n+6), #list :nth-child(10n+7), #list :nth-child(10n+8), #list :nth-child(10n+9), #list :nth-child(10n+10) { | |
| float: right; | |
| } | |
| } | |
| </style> | |
| <script src='http://code.jquery.com/jquery-2.0.3.js'></script> | |
| <script src='http://code.jquery.com/ui/1.10.3/jquery-ui.js'></script> | |
| <script src="http://coffeescript.org/extras/coffee-script.js"></script> | |
| <div class='metro' id='list'> | |
| <div class='tile' style='opacity: 0;'></div> | |
| </div> | |
| <script type="text/coffeescript"> | |
| $ = require 'jquery' | |
| process.step = 0 | |
| process.size = 100 | |
| get_query = -> | |
| process.step += 1 | |
| { | |
| "size": process.size | |
| "from": process.size * process.step | |
| "sort": [{"datetime": "asc"}] | |
| "query": | |
| "filtered": | |
| "query": | |
| "term": | |
| "category": "api" | |
| "filter": | |
| "range": | |
| "datetime": | |
| "from": "2013-10-21" | |
| "to": "2014-11-10" | |
| } | |
| get_next_step = -> | |
| $.when($.post("http://localhost:19201/#{es_type}/_search", JSON.stringify(get_query()))) | |
| .done((x) -> | |
| return x | |
| ) | |
| .fail((x) -> console.log x) | |
| get_all = -> | |
| $.when(get_next_step()).done( | |
| (x) -> | |
| if x.hits.hits.length > 0 | |
| elastic_to_csv x | |
| setTimeout((-> get_all()),100) | |
| ) | |
| get_all() | |
| </script> | |
| </body> | |
| </html> | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment