Skip to content

Instantly share code, notes, and snippets.

@6d61726b760a

6d61726b760a/.env

Created January 31, 2023 23:54
Show Gist options
  • Save 6d61726b760a/1f7e84d73deaddba092e0ad61beddcce to your computer and use it in GitHub Desktop.
Save 6d61726b760a/1f7e84d73deaddba092e0ad61beddcce to your computer and use it in GitHub Desktop.
Download ZIP
splunkcloud scripted dashboard generation
Raw
.env
TZ="Australia/Brisbane"
SPLUNK_ENDPOINT="https://myendpoint.splunkcloud.com:8089"
SPLUNK_USERNAME="splunkusername"
SPLUNK_PASSWORD="splunkpassword"
SPLUNK_APP="splunkapp"
Raw
dashboard.xml
<dashboard version="1.1">
<label>Client Monthly Dashboard - --CLIENT--NAME--</label>
<description></description>
<row>
<panel>
<title>Customer Details</title>
<table>
<search>
<query>| makeresults | eval client_id="--CLIENT--ID--" | lookup my_clients client_id as client_id OUTPUT client_name, client_company_name, client_country | fillnull value="client id not found!" client_country client_name client_company_name | table client_name client_company_name client_country client_id | rename client_id as "client id", client_name as "client name", client_company_name as "client company name", client_country as country</query>
<earliest>-24h@h</earliest>
<latest>now</latest>
</search>
<option name="drilldown">none</option>
<option name="refresh.display">progressbar</option>
</table>
</panel>
</row>
<row>
<panel>
<html>
<br/>
</html>
</panel>
</row>
<row>
<panel>
<title>TOTAL Monthly Usage - last 6 months</title>
<chart>
<search>
<query>index=summary source=summary_client-usage app_env=prod client_id="--CLIENT--ID--" (status<![CDATA[&gt;]]>199 AND status<![CDATA[&lt;]]>300) AND NOT (service_access_id=undefined) | timechart count by client_id span=1mon</query>
<earliest>-6mon@mon</earliest>
<latest>@mon</latest>
</search>
<option name="charting.chart">column</option>
<option name="charting.chart.showDataLabels">all</option>
<option name="charting.drilldown">all</option>
<option name="charting.legend.placement">none</option>
<option name="refresh.display">progressbar</option>
</chart>
</panel>
<panel>
<title>Top 10 Calls - last month</title>
<chart>
<search>
<query>index=summary source=summary_client-usage app_env=prod client_id="--CLIENT--ID--" AND status<![CDATA[&gt;]]>199 AND status<![CDATA[&lt;]]>300 AND NOT (service_access_id=undefined) | stats count by service_access_id | sort -count | head 10</query>
<earliest>-1mon@mon</earliest>
<latest>@mon</latest>
</search>
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
<option name="charting.axisTitleX.text">API group</option>
<option name="charting.axisTitleX.visibility">visible</option>
<option name="charting.axisTitleY.text">Number of calls</option>
<option name="charting.axisTitleY.visibility">visible</option>
<option name="charting.axisTitleY2.visibility">visible</option>
<option name="charting.axisX.scale">linear</option>
<option name="charting.axisY.scale">linear</option>
<option name="charting.axisY2.enabled">0</option>
<option name="charting.axisY2.scale">inherit</option>
<option name="charting.chart">bar</option>
<option name="charting.chart.bubbleMaximumSize">50</option>
<option name="charting.chart.bubbleMinimumSize">10</option>
<option name="charting.chart.bubbleSizeBy">area</option>
<option name="charting.chart.nullValueMode">gaps</option>
<option name="charting.chart.showDataLabels">all</option>
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
<option name="charting.chart.stackMode">default</option>
<option name="charting.chart.style">shiny</option>
<option name="charting.drilldown">all</option>
<option name="charting.layout.splitSeries">0</option>
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
<option name="charting.legend.placement">none</option>
<option name="height">304</option>
<option name="refresh.display">progressbar</option>
</chart>
</panel>
</row>
<row>
<panel>
<title>TOTAL Monthly Usage - last 6 months - stacked by calls</title>
<chart>
<search>
<query>index=summary source=summary_client-usage app_env=prod client_id="--CLIENT--ID--" status<![CDATA[&gt;]]>199 AND status<![CDATA[&lt;]]>300 AND NOT (service_access_id=undefined) | timechart count by service_access_id span=1mon</query>
<earliest>-6mon@mon</earliest>
<latest>@mon</latest>
</search>
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
<option name="charting.axisTitleX.visibility">collapsed</option>
<option name="charting.axisTitleY.text">Number of calls</option>
<option name="charting.axisTitleY.visibility">visible</option>
<option name="charting.axisTitleY2.text">Response time (ms)</option>
<option name="charting.axisTitleY2.visibility">visible</option>
<option name="charting.axisX.scale">linear</option>
<option name="charting.axisY.scale">linear</option>
<option name="charting.axisY2.abbreviation">none</option>
<option name="charting.axisY2.enabled">1</option>
<option name="charting.axisY2.scale">inherit</option>
<option name="charting.chart">column</option>
<option name="charting.chart.bubbleMaximumSize">50</option>
<option name="charting.chart.bubbleMinimumSize">10</option>
<option name="charting.chart.bubbleSizeBy">area</option>
<option name="charting.chart.nullValueMode">gaps</option>
<option name="charting.chart.overlayFields">"90th Percentile"</option>
<option name="charting.chart.showDataLabels">all</option>
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
<option name="charting.chart.stackMode">stacked</option>
<option name="charting.chart.style">shiny</option>
<option name="charting.drilldown">all</option>
<option name="charting.layout.splitSeries">0</option>
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
<option name="charting.legend.placement">top</option>
<option name="height">538</option>
<option name="refresh.display">progressbar</option>
</chart>
</panel>
</row>
<row>
<panel>
<title>Call Status - last month</title>
<chart>
<search>
<query>index=summary source=summary_client-usage app_env=prod client_id="--CLIENT--ID--" | eval Status=case(status<![CDATA[&lt;]]>200,"Informational",status<![CDATA[&lt;]]>300,"Success",status<![CDATA[&lt;]]>400,"Redirectional",status<![CDATA[&lt;]]>500,"Client Error",status<![CDATA[&lt;]]>600,"Server Error", status==status,"Unknown")| top 100 Status</query>
<earliest>-1mon@mon</earliest>
<latest>@mon</latest>
</search>
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
<option name="charting.axisTitleX.visibility">visible</option>
<option name="charting.axisTitleY.visibility">visible</option>
<option name="charting.axisTitleY2.visibility">visible</option>
<option name="charting.axisX.scale">linear</option>
<option name="charting.axisY.scale">linear</option>
<option name="charting.axisY2.enabled">0</option>
<option name="charting.axisY2.scale">inherit</option>
<option name="charting.chart">pie</option>
<option name="charting.chart.bubbleMaximumSize">50</option>
<option name="charting.chart.bubbleMinimumSize">10</option>
<option name="charting.chart.bubbleSizeBy">area</option>
<option name="charting.chart.nullValueMode">gaps</option>
<option name="charting.chart.showDataLabels">none</option>
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
<option name="charting.chart.stackMode">default</option>
<option name="charting.chart.style">shiny</option>
<option name="charting.drilldown">all</option>
<option name="charting.layout.splitSeries">0</option>
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
<option name="charting.legend.placement">right</option>
<option name="height">347</option>
<option name="refresh.display">progressbar</option>
</chart>
</panel>
</row>
</dashboard>
Raw
requirements-dev.txt
# install our "usual" requirements
-r requirements.txt
# then install the following modules for development
pre-commit
bandit
black
flake8
pylint
Raw
requirements.txt
python-dotenv
logzero
requests
Raw
splunkcloud_dashboard_gen.py
"""
generate pdf of a dashboard
https://community.splunk.com/t5/Getting-Data-In/Generate-PDF-from-View-in-REST-API/m-p/274783
"""
import json
import os
from datetime import datetime
import requests
from requests.packages import urllib3
from dotenv import load_dotenv
from logzero import setup_logger
from email.message import EmailMessage
import smtplib
# suppress ssl insecure request warnings
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# typically we never want to do this (^), however the api for
# splunkcloud is secured with a self signed certificate, so we dont
# have any choice here.
#
# Note: the splunk api uses a self signed certificate but there are ACL's
# that restrict access to certain IP's (its not a public free for all)
# set default logger and log level
# CRITICAL,50|ERROR,40|WARNING,30|INFO,20|DEBUG,10|NOTSET,0
logger = setup_logger(name="logger", level=20)
# read config from .env file
load_dotenv()
# get our envvars
try:
# splunk vars
splunk_endpoint = os.environ["SPLUNK_ENDPOINT"]
splunk_username = os.environ["SPLUNK_USERNAME"]
splunk_password = os.environ["SPLUNK_PASSWORD"]
splunk_auth = (splunk_username, splunk_password)
splunk_app = "cl_api"
splunk_kvstore = "cl_api_reporting"
except Exception as e:
logger.error("couldnt find environment variable")
logger.error(traceback.format_exc())
raise SystemExit(1) from e
def get_client_list():
"""
retrieve data from kvstore into a dict
"""
app_uri = f"{splunk_endpoint}/servicesNS/nobody/{splunk_app}/storage/collections/data/{splunk_kvstore}?output_mode=json"
headers = {"Content-Type": "application/json"}
response = requests.get(app_uri, auth=splunk_auth, verify=False, headers=headers)
if not response.ok:
logger.info(f"[{response.status_code}] {response.text}")
# transform result into list
client_id_list = [result for result in response.json()]
return client_id_list
def main():
"""main"""
# get client id's from lookup
client_list = get_client_list()
with open("dashboard.xml", encoding="utf-8") as xml_file:
xml_dashboard = xml_file.read()
# with open("client_list.json", encoding="utf-8") as json_file:
# client_list = json.loads(json_file.read())
for client in client_list:
logger.info(f"generating dashboard pdf for {client['name']} [{client['id']}]")
# replace tokens in dashboard
this_xml_dashboard = xml_dashboard
this_xml_dashboard = this_xml_dashboard.replace(
"--CLIENT--NAME--", client["name"]
)
this_xml_dashboard = this_xml_dashboard.replace("--CLIENT--ID--", client["id"])
# send xml dashboard to render endpoint
render_url = f"{splunk_endpoint}/services/pdfgen/render"
render_response = requests.post(
render_url,
auth=(splunk_username, splunk_password),
params={
"input-dashboard-xml": this_xml_dashboard.encode(),
"paper-size": "a4-landscape",
},
verify=False,
)
logger.info(f"render_response: {render_response.status_code}")
# render endpoint returns a pdf
if render_response.status_code == 200:
_client = client["name"].replace(" ", "_").lower()
_timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
_filesize = render_response.headers["Content-Length"]
outfile = f"./output/dashboard-{_client}-{_timestamp}.pdf"
with open(outfile, "wb") as pdffile:
pdffile.write(render_response.content)
logger.info(f"wrote {outfile} [{_filesize} bytes]")
logger.info(f"emailing report to: {client['email']}")
EML_SUBJECT = f"monthy report: {client['name']}"
EML_FROM = "[email protected]"
EML_TO = f"{client['email']}"
msg = EmailMessage()
msg["Subject"] = EML_SUBJECT
msg["From"] = EML_FROM
msg["To"] = EML_TO
# definitely don't mess with the .preamble
msg.set_content("report attached")
with open(outfile, "rb") as fp:
msg.add_attachment(
fp.read(),
maintype="application",
subtype="pdf",
filename=f"{_client}-{_timestamp}.pdf"
)
# Notice how smtplib now includes a send_message() method
with smtplib.SMTP("mail.mydomain.com") as s:
s.send_message(msg)
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment