Link to the Elasticsearch Blog.
Last active
March 21, 2022 13:45
-
-
Save stormpython/8796112 to your computer and use it in GitHub Desktop.
Data Visualization with Elasticsearch Aggregations and D3 (Tutorial)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <title>Elastic Aggregations</title> | |
| <script src="scripts/require.js"></script> | |
| <script>require(["scripts/main"], function () {})</script> | |
| <style> | |
| body { | |
| font: 14px sans-serif; | |
| } | |
| .arc path { | |
| stroke: #fff; | |
| stroke-width: 3px; | |
| } | |
| .node circle { | |
| fill: #fff; | |
| stroke: steelblue; | |
| stroke-width: 1.5px; | |
| } | |
| .node { | |
| font: 10px sans-serif; | |
| } | |
| .link { | |
| fill: none; | |
| stroke: #ccc; | |
| stroke-width: 1.5px; | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <div id="donut-chart"></div> | |
| <div id="dendrogram"></div> | |
| </body> | |
| </html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| define(['scripts/d3.v3', 'scripts/elasticsearch'], function (d3, elasticsearch) { | |
| "use strict"; | |
| var client = new elasticsearch.Client(); | |
| client.search({ | |
| index: 'nfl', | |
| size: 5, | |
| body: { | |
| // Begin query. | |
| query: { | |
| // Boolean query for matching and excluding items. | |
| bool: { | |
| must: { match: { "description": "TOUCHDOWN" }}, | |
| must_not: { match: { "qtr": 5 }} | |
| } | |
| }, | |
| // Aggregate on the results | |
| aggs: { | |
| touchdowns: { | |
| terms: { | |
| field: "qtr", | |
| order: { "_term" : "asc" } | |
| } | |
| } | |
| } | |
| // End query. | |
| } | |
| }).then(function (resp) { | |
| console.log(resp); | |
| // D3 code goes here. | |
| var touchdowns = resp.aggregations.touchdowns.buckets; | |
| // d3 donut chart | |
| var width = 600, | |
| height = 300, | |
| radius = Math.min(width, height) / 2; | |
| var color = ['#ff7f0e', '#d62728', '#2ca02c', '#1f77b4']; | |
| var arc = d3.svg.arc() | |
| .outerRadius(radius - 60) | |
| .innerRadius(120); | |
| var pie = d3.layout.pie() | |
| .sort(null) | |
| .value(function (d) { return d.doc_count; }); | |
| var svg = d3.select("#donut-chart").append("svg") | |
| .attr("width", width) | |
| .attr("height", height) | |
| .append("g") | |
| .attr("transform", "translate(" + width/1.4 + "," + height/2 + ")"); | |
| var g = svg.selectAll(".arc") | |
| .data(pie(touchdowns)) | |
| .enter() | |
| .append("g") | |
| .attr("class", "arc"); | |
| g.append("path") | |
| .attr("d", arc) | |
| .style("fill", function (d, i) { | |
| return color[i]; | |
| }); | |
| g.append("text") | |
| .attr("transform", function (d) { return "translate(" + arc.centroid(d) + ")"; }) | |
| .attr("dy", ".35em") | |
| .style("text-anchor", "middle") | |
| .style("fill", "white") | |
| .text(function (d) { return d.data.key; }); | |
| }); | |
| client.search({ | |
| index: 'nfl', | |
| size: 5, | |
| body: { | |
| query: { | |
| bool: { | |
| must: { match: { "description": "TOUCHDOWN"}}, | |
| must_not: [ | |
| { match: { "description": "intercepted"}}, | |
| { match: { "description": "incomplete"}}, | |
| { match: { "description": "FUMBLES"}}, | |
| { match: { "description": "NULLIFIED"}} | |
| ] | |
| } | |
| }, | |
| aggs: { | |
| teams: { | |
| terms: { | |
| field: "off", | |
| exclude: "", | |
| size: 5 | |
| }, | |
| aggs: { | |
| players: { | |
| terms: { | |
| field: "description", | |
| include: "([a-z]?[.][a-z]+)", | |
| size: 200 | |
| }, | |
| aggs: { | |
| qtrs: { | |
| terms: { | |
| field: "qtr", | |
| order: { "_term": "asc" } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| }).then(function (resp) { | |
| console.log(resp); | |
| // D3 code goes here. | |
| var root = createChildNodes(resp); | |
| // d3 dendrogram | |
| var width = 600, | |
| height = 2000; | |
| var color = ['#ff7f0e', '#d62728', '#2ca02c', '#1f77b4']; | |
| var cluster = d3.layout.cluster() | |
| .size([height, width - 200]); | |
| var diagonal = d3.svg.diagonal() | |
| .projection(function(d) { return [d.y, d.x]; }); | |
| var svg = d3.select("#dendrogram").append("svg") | |
| .attr("width", width) | |
| .attr("height", height) | |
| .append("g") | |
| .attr("transform", "translate(120,0)"); | |
| var nodes = cluster.nodes(root), | |
| links = cluster.links(nodes); | |
| var link = svg.selectAll(".link") | |
| .data(links) | |
| .enter().append("path") | |
| .attr("class", "link") | |
| .attr("d", diagonal); | |
| var node = svg.selectAll(".node") | |
| .data(nodes) | |
| .enter().append("g") | |
| .attr("class", "node") | |
| .attr("transform", function(d) { return "translate(" + d.y + "," + d.x + ")"; }); | |
| node.append("circle") | |
| .attr("r", 4.5) | |
| .style("fill", function (d) { | |
| return d.children ? "#ffffff" : color[d.key - 1]; | |
| }) | |
| .style("stroke", function (d) { | |
| return d.children ? "#4682B4" : color[d.key - 1]; | |
| }); | |
| node.append("text") | |
| .attr("dx", function(d) { return d.children ? -8 : 8; }) | |
| .attr("dy", 3) | |
| .style("text-anchor", function(d) { return d.children ? "end" : "start"; }) | |
| .text(function(d) { return d.children? d.key : d.key + ": " + d.doc_count; }); | |
| d3.select(self.frameElement).style("height", height + "px"); | |
| function createChildNodes(dataObj) { | |
| var root = {}; | |
| root.key = "NFL"; | |
| root.children = dataObj.aggregations.teams.buckets; | |
| root.children.forEach(function (d) { d.children = d.players.buckets; }); | |
| root.children.forEach(function (d) { | |
| d.children.forEach(function (d) { | |
| d.children = d.qtrs.buckets; | |
| }); | |
| }); | |
| return root; | |
| } | |
| }); | |
| }); |
when I tried to use curl -XPUT localhost:9200/nfl/2013/_mapping?pretty -d @nfl_mapping.json
it keeps getting this error:
curl -XPUT localhost:9200/nfl/2013/_mapping?pretty -d @/Users/roccia/Downloads/nfl_mapping.json
{
"error" : {
"root_cause" : [ {
"type" : "mapper_parsing_exception",
"reason" : "Mapping definition for [_index] has unsupported parameters: [enabled : true]"
} ],
"type" : "mapper_parsing_exception",
"reason" : "Mapping definition for [_index] has unsupported parameters: [enabled : true]"
},
"status" : 400
}
it seems the issue has something related to this : elastic/elasticsearch#12356
in the nfl_mapping.json, the _index filed was set to ["enabled", true] which is no longer allowed to modified now.
not sure if I am right about this, if so would you mind fix the tutorial? Thanks
OK I rewrite the mapping.json, it looks like this:
{
"mappings" : {
"my_type": {
"properties" : {
"gameid" : {
"type" : "string",
"index" : "not_analyzed",
"store" : "true"
},
"qtr" : {
"type" : "short",
"store" : "true"
},
"min" : {
"type" : "short",
"store" : "true",
"ignore_malformed": "true"
},
"sec" : {
"type" : "short",
"store" : "true"
},
"off" : {
"type" : "string"
},
"def" : {
"type" : "string"
},
"down" : {
"type" : "short",
"store" : "true",
"ignore_malformed": "true"
},
"togo" : {
"type" : "short",
"store" : "true",
"ignore_malformed": "true"
},
"ydline" : {
"type" : "short",
"store" : "true",
"ignore_malformed": "true"
},
"scorediff" : {
"type" : "short",
"store" : "true"
},
"series1stdn" : {
"type" : "short",
"store" : "true"
},
"description" : {
"type" : "string",
"store" : "true"
},
"scorechange" : {
"type" : "short",
"store" : "true"
},
"nextscore" : {
"type" : "short",
"store" : "true"
},
"teamwin" : {
"type" : "short",
"store" : "true"
},
"offscore" : {
"type" : "short",
"store" : "true"
},
"defscore" : {
"type" : "short",
"store" : "true"
},
"season" : {
"type" : "date",
"format" : "YYYY",
"ignore_malformed": "true"
}
}
}
}
}
but I'm still getting "reason" : "Root mapping definition has unsupported parameters: error , any idea why? need your help! thank you
@roccia try this..
{
"2013" : {
"properties" : {
"gameid" : {
"type" : "string",
"index" : "not_analyzed",
"store" : "yes"
},
"qtr" : {
"type" : "short",
"store" : "yes"
},
"min" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"sec" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"off" : {
"type" : "string",
"index" : "not_analyzed"
},
"def" : {
"type" : "string",
"index" : "not_analyzed"
},
"down" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"togo" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"ydline" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"scorediff" : {
"type" : "short",
"store" : "yes"
},
"series1stdn" : {
"type" : "short",
"store" : "yes"
},
"description" : {
"type" : "string",
"store" : "yes"
},
"scorechange" : {
"type" : "short",
"store" : "yes"
},
"nextscore" : {
"type" : "short",
"store" : "yes"
},
"teamwin" : {
"type" : "short",
"store" : "yes"
},
"offscore" : {
"type" : "short",
"store" : "yes"
},
"defscore" : {
"type" : "short",
"store" : "yes"
},
"season" : {
"type" : "date",
"format" : "YYYY",
"ignore_malformed" : "true"
}
}
}
}
Hi,while I am trying to view the chart on the http-server, I created using node js and with the following port no. as http://127.0.0.1:8080, I am getting this kind of error in my console.
XMLHttpRequest cannot load http://localhost:9200/nfl/2013/_search?size=5. No 'Access-Control-Allow-Origin' header is present on the requested resource. Origin 'http://127.0.0.1:8080' is therefore not allowed access.
Anybody knowing how to remove this
@tiwarisahil91 open run and enter chrome.exe --user-data-dir="C:/Chrome dev session" --disable-web-security . A new chrome browser will open run your url in that window
Hi,
I have a problem with this example. Please have a look at the screeen shots enclosed.
Thanks!
hi
i am getting the following error while launching the server:
elasticsearch.js:19450 ERROR: 2018-04-19T10:24:55Z
Error: Request error, retrying -- Request failed to complete.
at Log.error (http://localhost:8001/scripts/elasticsearch.js:19151:56)
at checkRespForFailure (http://localhost:8001/scripts/elasticsearch.js:19817:18)
at XMLHttpRequest.xhr.onreadystatechange (http://localhost:8001/scripts/elasticsearch.js:18655:7)
the following helped to get the issue cleared...
var elasticsearch = require('elasticsearch');
var AgentKeepAlive = require('agentkeepalive');
var client = new elasticsearch.Client({
host: 'localhost:9200',
requestTimeout: Infinity, // Tested
keepAlive: true ,// Tested
maxRetries: 10,
keepAlive: true,
maxSockets: 10,
minSockets: 10,
createNodeAgent: function (connection, config) {
return new AgentKeepAlive(connection.makeAgentConfig(config));
}
});
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
@agttga, my apologies for the late reply. I was unaware of the comment you made. I updated the link to the blog post which these files are supplemental material for. How to use these files is outlined there. Just click on the Blog link in the README.