Skip to content

Instantly share code, notes, and snippets.

@jasonLaster
Created December 2, 2020 22:39
Show Gist options
  • Select an option

  • Save jasonLaster/90955885d1d42f2961a6e8aea9a4b8de to your computer and use it in GitHub Desktop.

Select an option

Save jasonLaster/90955885d1d42f2961a6e8aea9a4b8de to your computer and use it in GitHub Desktop.
diff --git a/dashboard.js b/dashboard.js
index 74ff00b..97b5742 100644
--- a/dashboard.js
+++ b/dashboard.js
@@ -9,11 +9,9 @@ const {
getLogAge,
isTestConnection,
} = require("./logs");
-const {
- getDescription,
-} = require("./filters");
-const { log, S3Bucket } = require("./utils");
-const { spawnSync } = require("child_process");
+const {getDescription} = require("./filters");
+const {log, S3Bucket} = require("./utils");
+const {spawnSync} = require("child_process");
// Age categories for error and request metrics, in hours.
const MetricAges = [1, 4, 24, 72];
@@ -28,7 +26,13 @@ function computeErrorMetrics() {
let numUserFixed = 0;
let numTestLogs = 0;
let numTestErrors = 0;
- for (let { timestamp, forTest, hasErrors, fatalError, isFixed } of getLogs()) {
+ for (let {
+ timestamp,
+ forTest,
+ hasErrors,
+ fatalError,
+ isFixed,
+ } of getLogs()) {
// For now we only care about logs which had fatal errors. We didn't use to keep
// track of this information, hence this oddness.
if (fatalError === false) {
@@ -69,7 +73,7 @@ function computeErrorMetrics() {
function getPercent(num, denom) {
if (denom) {
- return (Math.round(1000 * num / denom) / 10) + "%";
+ return Math.round((1000 * num) / denom) / 10 + "%";
} else {
return "n/a";
}
@@ -82,7 +86,7 @@ function buildCommonMetricRows(title) {
text += ` ${age} hour${age > 1 ? "s" : ""} |`;
}
text += "\n";
- text += "| --- |";
+ text += "| --- |";
for (let i = 0; i < MetricAges.length; i++) {
text += " --- |";
}
@@ -92,21 +96,36 @@ function buildCommonMetricRows(title) {
// Contents of each row in the error metrics table.
const ErrorMetricRows = [
- { title: "User Logs", callback: ({ numUserLogs }) => numUserLogs },
- { title: "% Errors",
- callback: ({ numUserLogs, numUserErrors }) => getPercent(numUserErrors, numUserLogs) },
- { title: "% Stale errors",
- callback: ({ numUserLogs, numUserFixed }) => getPercent(numUserFixed, numUserLogs) },
- { title: "Test Logs", callback: ({ numTestLogs }) => numTestLogs },
- { title: "% Test errors",
- callback: ({ numTestLogs, numTestErrors }) => getPercent(numTestErrors, numTestLogs) },
+ {title: "User Logs", callback: ({numUserLogs}) => numUserLogs},
+ {
+ title: "% Errors",
+ callback: ({numUserLogs, numUserErrors}) =>
+ getPercent(numUserErrors, numUserLogs),
+ },
+ {
+ title: "% Stale errors",
+ callback: ({numUserLogs, numUserFixed}) =>
+ getPercent(numUserFixed, numUserLogs),
+ },
+ {title: "Test Logs", callback: ({numTestLogs}) => numTestLogs},
+ {
+ title: "% Test errors",
+ callback: ({numTestLogs, numTestErrors}) =>
+ getPercent(numTestErrors, numTestLogs),
+ },
];
function computeRequestMetrics(wantTest) {
const now = Date.now();
const metrics = new Map();
- for (const { name, duration, timestamp, connectionId, forTest } of getRequests()) {
+ for (const {
+ name,
+ duration,
+ timestamp,
+ connectionId,
+ forTest,
+ } of getRequests()) {
if (wantTest != !!(forTest || isTestConnection(connectionId))) {
continue;
}
@@ -127,9 +146,9 @@ function computeRequestMetrics(wantTest) {
const entries = [...metrics.entries()].map(([title, durations]) => {
const metrics = durations.map(getTimingSummary);
- return { title, metrics };
+ return {title, metrics};
});
- entries.sort((a, b) => a.title < b.title ? -1 : 1);
+ entries.sort((a, b) => (a.title < b.title ? -1 : 1));
return entries;
}
@@ -148,7 +167,7 @@ function getTimingSummary(durations) {
return "0: n/a";
}
- durations.sort((a, b) => a < b ? -1 : 1);
+ durations.sort((a, b) => (a < b ? -1 : 1));
const median = durationToString(durations[Math.floor(durations.length / 2)]);
const _90th = durationToString(durations[Math.floor(durations.length * 0.9)]);
const max = durationToString(durations[durations.length - 1]);
@@ -158,7 +177,7 @@ function getTimingSummary(durations) {
function buildRequestMetricsTable(wantTest) {
const requestMetrics = computeRequestMetrics(wantTest);
let text = buildCommonMetricRows("Request Age");
- for (const { title, metrics } of requestMetrics) {
+ for (const {title, metrics} of requestMetrics) {
text += `| ${title} |`;
for (const metric of metrics) {
text += ` ${metric} |`;
@@ -207,26 +226,30 @@ function computeErrorBuckets() {
const now = Date.now();
const buckets = [];
- const lines = spawnSync("aws", [
- "s3",
- "ls",
- `s3://${S3Bucket}/triage/`
- ]).stdout.toString().split("\n");
+ const lines = spawnSync("aws", ["s3", "ls", `s3://${S3Bucket}/triage/`])
+ .stdout.toString()
+ .split("\n");
for (const line of lines) {
const match = /PRE (.*?)\//.exec(line);
if (match) {
const bucket = match[1];
const counts = newErrorCounts();
+ const urls = [];
const lines = spawnSync("aws", [
"s3",
"ls",
- `s3://${S3Bucket}/triage/${bucket}/`
- ]).stdout.toString().split("\n");
+ `s3://${S3Bucket}/triage/${bucket}/`,
+ ])
+ .stdout.toString()
+ .split("\n");
for (const line of lines) {
let match = /replay-.*?\.log/.exec(line);
if (match) {
- const logAge = getLogAge(now, match[0]);
+ const file = match[0];
+ const logAge = getLogAge(now, file);
+ const {url} = getLogInfo(file);
+ urls.push(url);
if (!logAge) {
// If we don't know how old this log is then it probably was
// forgotten for being too old, in which case we ignore it during
@@ -249,8 +272,8 @@ function computeErrorBuckets() {
}
}
}
- if (counts.some(n => n > 0)) {
- buckets.push({ counts, bucket });
+ if (counts.some((n) => n > 0)) {
+ buckets.push({counts, bucket, urls});
}
}
}
@@ -272,7 +295,7 @@ function buildDashboard() {
const errorMetrics = computeErrorMetrics();
text += buildCommonMetricRows("Log Age");
- for (const { title, callback } of ErrorMetricRows) {
+ for (const {title, callback} of ErrorMetricRows) {
text += `| ${title} |`;
for (const metric of errorMetrics) {
text += ` ${callback(metric)} |`;
@@ -291,8 +314,10 @@ function buildDashboard() {
const fatalCounts = newErrorCounts();
const normalCounts = newErrorCounts();
- for (const { counts, bucket } of buckets) {
- const aggregateCounts = bucket.startsWith("fatal_") ? fatalCounts : normalCounts;
+ for (const {counts, bucket} of buckets) {
+ const aggregateCounts = bucket.startsWith("fatal_")
+ ? fatalCounts
+ : normalCounts;
for (let i = 0; i < counts.length; i++) {
aggregateCounts[i] += counts[i];
}
@@ -300,24 +325,28 @@ function buildDashboard() {
const fatalString = errorCountsToString(fatalCounts, /* verbose */ true);
text += `## Fatal Log Errors: ${fatalString}\n`;
- for (const { counts, bucket } of buckets) {
+ for (const {counts, bucket} of buckets) {
if (bucket.startsWith("fatal_")) {
const nbucket = bucket.substring(6);
const desc = buildDescription(nbucket);
- text += `\n\n**${errorCountsToString(counts)}**: **${nbucket}** *${desc}*\n`;
+ text += `\n\n**${errorCountsToString(
+ counts
+ )}**: **${nbucket}** *${desc}*\n`;
}
}
const normalString = errorCountsToString(normalCounts, /* verbose */ true);
text += `## Non-Fatal Log Errors: ${normalString}\n`;
- for (const { counts, bucket } of buckets) {
+ for (const {counts, bucket} of buckets) {
if (!bucket.startsWith("fatal_")) {
const desc = buildDescription(bucket);
- text += `\n\n**${errorCountsToString(counts)}**: **${bucket}** *${desc}*\n`;
+ text += `\n\n**${errorCountsToString(
+ counts
+ )}**: **${bucket}** *${desc}*\n`;
}
}
return text;
}
-module.exports = { buildDashboard };
+module.exports = {buildDashboard};
diff --git a/logs.js b/logs.js
index 8c366e0..9e9fddd 100644
--- a/logs.js
+++ b/logs.js
@@ -4,7 +4,7 @@
// logs recent enough that they will be considered in the dashboard.
const fs = require("fs");
-const { log } = require("./utils");
+const {log} = require("./utils");
// For logs that we've encountered recently and want to consider when updating
// the dashboard, map log file names to information about the log.
@@ -25,11 +25,11 @@ function loadLogs() {
const logs = json.logs ? json.logs : json;
const requests = json.requests || [];
const connections = json.connections || [];
- for (const { log, info} of logs) {
+ for (const {log, info} of logs) {
gLogs.set(log, info);
}
requests.forEach(addRequest);
- for (const { id, info } of connections) {
+ for (const {id, info} of connections) {
gConnections.set(id, info);
}
} catch (e) {
@@ -50,7 +50,7 @@ function storeLogs() {
// Ignore logs that won't be included in the dashboard metrics.
const age = timestampAge(now, info.timestamp);
if (age < MaxLogAge) {
- logs.push({ log: name, info });
+ logs.push({log: name, info});
} else {
log(`Discarding log ${name} age ${age}`);
}
@@ -73,11 +73,11 @@ function storeLogs() {
const connections = [];
for (const [id, info] of gConnections.entries()) {
if (requestConnectionIds.has(id)) {
- connections.push({ id, info });
+ connections.push({id, info});
}
}
- fs.writeFileSync("logs.json", JSON.stringify({ logs, requests, connections }));
+ fs.writeFileSync("logs.json", JSON.stringify({logs, requests, connections}));
log("Finished updating stored logs.");
}
@@ -88,13 +88,15 @@ function timestampAge(now, timestamp) {
}
function isTestConnection(connectionId) {
- return connectionId &&
- gConnections.has(connectionId) &&
- gConnections.get(connectionId).forTest;
+ return (
+ connectionId &&
+ gConnections.has(connectionId) &&
+ gConnections.get(connectionId).forTest
+ );
}
function addTestConnection(connectionId) {
- gConnections.set(connectionId, { forTest: true });
+ gConnections.set(connectionId, {forTest: true});
}
function getLogs() {
@@ -115,6 +117,10 @@ function getLogAge(now, file) {
return logInfo ? timestampAge(now, logInfo.timestamp) : 0;
}
+function getLog(file) {
+ return gLogs.get(file);
+}
+
function getRequests() {
return gRequests.values();
}
@@ -122,7 +128,7 @@ function getRequests() {
function addRequest(request) {
// Extract any uniquely identifying information for this request, making
// sure we don't double count it.
- const { name, controlId, connectionId, messageId, sessionId } = request;
+ const {name, controlId, connectionId, messageId, sessionId} = request;
const key = `${name}:${controlId}:${connectionId}:${messageId}:${sessionId}`;
if (!gRequests.has(key)) {
gRequests.set(key, request);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment