Skip to content

Instantly share code, notes, and snippets.

@willblaschko

willblaschko/benchmark.js

Created May 17, 2019 21:18
Show Gist options
  • Save willblaschko/a0e361e8e5e3afb3799a31165c362729 to your computer and use it in GitHub Desktop.
Save willblaschko/a0e361e8e5e3afb3799a31165c362729 to your computer and use it in GitHub Desktop.
Download ZIP
Alexa APL Benchmarking Script
Raw
benchmark.js
const puppeteer = require('puppeteer');
const moment = require("moment");
let baseLayout = require('./layout_base.json');
const exec = require('child_process').execSync;
let sleep = function (ms) {
return new Promise(resolve => setTimeout(resolve, ms));
};
let runTests = async function (baseLayout, customLayout, data, count) {
return new Promise(
async (resolve, reject) => {
let sleep = function (ms) {
return new Promise(resolve => setTimeout(resolve, ms));
};
let browserTimes = [];
let deviceTimes = [];
let checkTimesLength = async function () {
console.log("checkTimesLength called");
if (browserTimes.length >= count) {
resolve({
"browser": browserTimes,
"device": deviceTimes
});
return;
}
getLayoutLoadTime();
};
let sparky = null;
let startTime = Date.now();
$("body").off("DOMSubtreeModified");
$("body").on('DOMSubtreeModified', "#apml-renderer", function () {
if (sparky) {
clearTimeout(sparky);
}
sparky = setTimeout(async function () {
//do browser work
let start = startTime;
let end = Date.now();
browserTimes.push(end - start);
//start device work
$(".askaa-device-push__button").click();
console.log("Device button pressed");
await sleep(1000);
//start the cycle again
await checkTimesLength();
}, 300);
});
let getLayoutLoadTime = function () {
editor.setValue(JSON.stringify(baseLayout)); //reset
setTimeout(function () {
startTime = Date.now();
editor.setValue(JSON.stringify(customLayout)); //custom layout
}, 200); //give time for our APL to trigger and start rendering
};
//set-up
console.log("Clicking from scratch");
$('[data-qa-hook="apl-authoring-templates-item-start-from-scratch"]').click();
await sleep(1000);
console.log("Clicking one panel");
$('#astro-toggle-2').click();
await sleep(1000);
console.log("Clicking data tab");
$('#astro-radio-2-label').click();
console.log("Setting data");
let editor = ace.edit("brace-editor");
editor.setValue(JSON.stringify(data));
await sleep(1000);
console.log("Clicking code tab");
$('#astro-radio-1-label').click();
await sleep(3000);
//run tests
checkTimesLength(customLayout);
});
};
async function runRendering(baseLayout, customLayout, data, count, userDataDir, device = false) {
let lastFinishedRender = moment();
let deviceTimes = [];
let checkLogs = function () {
//do device work
let command = "adb logcat -t \"" + lastFinishedRender.format("M-D H:m:s.SSS") + "\" -v time";
let times = [];
let start = null;
let end = null;
let logs = exec(command).toString();
let lines = logs.split("\n");
for (let i = 0; i < lines.length; i++) {
let line = lines[i];
if (line.includes("onActivityResume for class com.amazon.aria.AriaActivity")) {
start = moment(line.split(" ")[1], "HH:mm:ss.SSS");
//console.log("Render start found: " + start);
lastFinishedRender = start;
}
if (line.includes("received on inflation event")) {
end = moment(line.split(" ")[1], "HH:mm:ss.SSS");
//console.log("Render end found: " + end);
if (start) {
deviceTimes.push(end.valueOf() - start.valueOf());
start = null;
lastFinishedRender = end;
}
}
}
deviceTimes = deviceTimes.concat(times);
};
const browser = await puppeteer.launch({
headless: false,
userDataDir: userDataDir
});
let pages = await browser.pages();
let page = null;
if (pages.length > 0) {
for (let i = pages.length - 1; i > 0; i--) {
pages[i].close();
}
page = pages[0];
} else {
page = await browser.newPage();
}
await page.goto('https://developer.amazon.com/alexa/console/ask/displays', {
waitUntil: 'networkidle2'
});
await page.addScriptTag({ url: 'https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.24.0/moment.min.js' });
let sparky = null;
if(device){
sparky = setInterval(checkLogs, 2000);
}
let times = await page.evaluate(runTests, baseLayout, customLayout, data, count);
await page.screenshot({ path: 'example.png' });
await browser.close();
await sleep(10000);
if(device){
clearInterval(sparky);
checkLogs();
}
times.device = deviceTimes;
return times;
}
//if this function freezes, make sure ADB is actually connected to your FireTV device
async function getBenchmarkTimes(layout, data, count, userDataDir, device = false) {
return new Promise(async (resolve, reject) => {
let times = await runRendering(baseLayout, layout, data, count, userDataDir, device);
//console.log(times);
resolve(times);
});
}
/////////////////////////////////////////////////////////////////////
module.exports = { getBenchmarkTimes : getBenchmarkTimes };
Raw
data_example.json
{
"grid": {
"values": [
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3],
[2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, 1, 0, 1, 3]
]
},
"tiles": {
"height": "${100 / payload.grid.values.length}%",
"width": "${100 / payload.grid.values[0].length}%"
},
"game": {
"colors": [
"#ff0000",
"#ffdb58",
"#aabbcc",
"#ccbbaa"
]
}
}
Raw
index.js
const Benchmark = require("./benchmark");
//a helper function to output information about our benchmarks
function parseTimes(name, times) {
let min = Number.MAX_SAFE_INTEGER;
let max = Number.MIN_SAFE_INTEGER;
let total = 0;
for (let i = 0; i < times.length; i++) {
let time = times[i];
min = Math.min(min, time);
max = Math.max(max, time);
total += time;
}
return ("\nRunning benchmark for " + name +
"\nBenchmark ran " + times.length + " times." +
"\nAverage render: " + parseInt(total / times.length) + "ms" +
"\nMax render: " + max + "ms" +
"\nMin render: " + min + "ms");
}
//in this case we're using a single data object for all our benchmarks
let data = require('./data_example.json');
//the array of layouts we want to loop through to compare
layouts = [ require('./layout_example_0.json'), require('./layout_example_1.json'), require('./layout_example_2.json')];
(async function () {
for (let i = 0; i < layouts.length; i++) {
let layout = layouts[i];
//getBenchmarkTimes(layout file, layout data, number of iterations, user data folder, if we want to use ADB)
let times = await Benchmark.getBenchmarkTimes(layout, data, 100, 'C:/tmp/User Data');
console.log("\n");
console.log(parseTimes("Layout " + i + " (Browser)", times.browser));
console.log(JSON.stringify(times.browser));
console.log(parseTimes("Layout " + i + " (Device)", times.device));
console.log(JSON.stringify(times.device));
}
})();
Raw
layout_base.json
{
"type": "APL",
"version": "1.0",
"theme": "dark",
"import": [],
"resources": [],
"styles": {},
"layouts": {},
"mainTemplate": {
"parameters": [
"payload"
],
"items": []
}
}
Raw
layout_example_0.json
{
"type": "APL",
"version": "1.0",
"theme": "dark",
"import": [
{
"name": "alexa-viewport-profiles",
"version": "1.0.0"
}
],
"layouts": {
"Tile": {
"parameters": [
{
"name": "name",
"type": "string"
}
],
"items": [
{
"type": "Frame",
"width": "${payload.tiles.width}",
"height": "100%",
"id": "${name}",
"style": "colorGrid"
}
]
}
},
"resources": [],
"styles": {
"grid": {
"values": [
{
}
]
},
"colorGrid": {
"extends": "grid",
"values": [
{
"fontFamily": "Amazon Ember",
"backgroundColor": "#abc"
},
{
"when": "${state.disabled}",
"backgroundColor": "red"
},
{
"when": "${state.checked}",
"backgroundColor": "green"
}
]
}
},
"mainTemplate": {
"parameters": [
"payload"
],
"items": [
{
"type": "Container",
"style": "gridBase",
"width": "100vw",
"height": "100vh",
"items": [
{
"type": "Container",
"width": "100vw",
"height": "100vh",
"data": "${payload.grid.values}",
"items": [
{
"type": "Container",
"direction": "row",
"width": "100%",
"height": "${payload.tiles.height}",
"data": "${data}",
"bind": [
{
"name": "parentIndex",
"value": "${index}"
}
],
"items": [
{
"type": "Tile",
"name": "${parentIndex+'.'+index}"
}
]
}
]
}
]
}
]
}
}
Raw
layout_example_1.json
{
"type": "APL",
"version": "1.0",
"theme": "dark",
"import": [
{
"name": "alexa-viewport-profiles",
"version": "1.0.0"
}
],
"layouts": {
"Tile": {
"parameters": [
{
"name": "name",
"type": "string"
}
],
"items": [
{
"type": "Frame",
"width": "${payload.tiles.width}",
"height": "100%",
"id": "${name}",
"style": "colorGrid"
}
]
}
},
"resources": [],
"styles": {
"grid": {
"values": [
{
}
]
},
"colorGrid": {
"extends": "grid",
"values": [
{
"fontFamily": "Amazon Ember",
"backgroundColor": "#abc"
},
{
"when": "${state.disabled}",
"backgroundColor": "red"
},
{
"when": "${state.checked}",
"backgroundColor": "green"
}
]
}
},
"mainTemplate": {
"parameters": [
"payload"
],
"items": [
{
"type": "Container",
"style": "gridBase",
"alignItems": "start",
"direction": "column",
"width": "100vw",
"height": "100vh",
"items": [
{
"type": "Container",
"width": "100vw",
"height": "100vh",
"data": "${payload.grid.values}",
"items": [
{
"type": "Container",
"direction": "row",
"width": "100%",
"height": "${payload.tiles.height}",
"data": "${data}",
"bind": [
{
"name": "parentIndex",
"value": "${index}"
}
],
"items": [
{
"type": "Tile",
"name": "${parentIndex+'.'+index}"
}
]
}
]
}
]
}
]
}
}
Raw
layout_example_2.json
{
"type": "APL",
"version": "1.0",
"theme": "dark",
"import": [
{
"name": "alexa-viewport-profiles",
"version": "1.0.0"
}
],
"layouts": {
"Tile": {
"parameters": [
{
"name": "name",
"type": "string"
}
],
"items": [
{
"type": "Pager",
"width": "${payload.tiles.width}",
"height": "100%",
"data":"${payload.game.colors}",
"items":
[
{
"type": "Frame",
"width": "100%",
"height": "100%",
"id": "${name}",
"backgroundColor":"${data}"
}
]
}
]
}
},
"resources": [],
"styles": {
"grid": {
"values": [
{
}
]
},
"colorGrid": {
"extends": "grid",
"values": [
{
"fontFamily": "Amazon Ember",
"backgroundColor": "#abc"
},
{
"when": "${state.disabled}",
"backgroundColor": "red"
},
{
"when": "${state.checked}",
"backgroundColor": "green"
}
]
}
},
"mainTemplate": {
"parameters": [
"payload"
],
"items": [
{
"type": "Container",
"style": "gridBase",
"width": "100vw",
"height": "100vh",
"items": [
{
"type": "Container",
"width": "100vw",
"height": "100vh",
"data": "${payload.grid.values}",
"items": [
{
"type": "Container",
"direction": "row",
"width": "100%",
"height": "${payload.tiles.height}",
"data": "${data}",
"bind": [
{
"name": "parentIndex",
"value": "${index}"
}
],
"items": [
{
"type": "Tile",
"name": "${parentIndex+'.'+index}"
}
]
}
]
}
]
}
]
}
}
Raw
package.json
{
"name": "APL Benchmark",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"moment": "^2.24.0",
"puppeteer": "^1.16.0"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment