Benchmarking compile times for CDATA vs codbehind for Roku Projects

bench.js

#!/usr/bin/env node
import fs from 'node:fs';
import path from 'node:path';
import net from 'node:net';
import { rokuDeploy } from 'roku-deploy';
import { undent } from 'undent';

const SCENARIOS = [
  { folder: 'cdata', title: 'CDATA Bench', inline: true },
  { folder: 'codebehind', title: 'Codebehind Bench', inline: false },
];

const DEFAULT_COMPONENT_COUNT = 100;
const TELNET_PORT = 8085;
const SETTLE_MS = 500;
const SETTLE_MAX_MS = 30_000;
const RUN_TIMEOUT_MS = 120_000;

function parseArgs() {
  const args = process.argv.slice(2);
  const opts = {
    host: null,
    password: null,
    runs: 5,
    components: DEFAULT_COMPONENT_COUNT,
  };
  for (let i = 0; i < args.length; i++) {
    const a = args[i];
    if (a === '--host') opts.host = args[++i];
    else if (a === '--password') opts.password = args[++i];
    else if (a === '--runs') opts.runs = parseInt(args[++i], 10);
    else if (a === '--components') opts.components = parseInt(args[++i], 10);
    else if (a === '-h' || a === '--help') {
      printHelp();
      process.exit(0);
    } else {
      console.error(`Unknown arg: ${a}`);
      printHelp();
      process.exit(1);
    }
  }
  if (!opts.host) {
    console.error('Missing required --host');
    printHelp();
    process.exit(1);
  }
  if (!opts.password) {
    console.error('Missing required --password');
    printHelp();
    process.exit(1);
  }
  if (!Number.isFinite(opts.runs) || opts.runs < 1) {
    console.error('--runs must be a positive integer');
    process.exit(1);
  }
  if (!Number.isFinite(opts.components) || opts.components < 1) {
    console.error('--components must be a positive integer');
    process.exit(1);
  }
  return opts;
}

function printHelp() {
  console.log(undent`
    Usage: node bench.js --host <ip> --password <pwd> [--runs <n>] [--components <n>]

      --host <ip>         Roku device IP (required)
      --password <pwd>    Developer password (required)
      --runs <n>          Runs per scenario (default: 5)
      --components <n>    Components to generate per scenario (default: ${DEFAULT_COMPONENT_COUNT})
  `);
}

function regenerateScenarios(componentCount) {
  for (const s of SCENARIOS) {
    fs.rmSync(path.resolve(s.folder), { recursive: true, force: true });
    generateScenario(s, componentCount);
  }
}

function generateScenario({ folder, title, inline }, componentCount) {
  const root = path.resolve(folder);
  fs.mkdirSync(path.join(root, 'source'), { recursive: true });
  fs.mkdirSync(path.join(root, 'components', 'scene'), { recursive: true });
  fs.mkdirSync(path.join(root, 'components', 'generated'), { recursive: true });

  fs.writeFileSync(path.join(root, 'manifest'), undent`
    title=${title}
    major_version=1
    minor_version=0
    build_version=0
    splash_color=#808080
    splash_min_time=0
    ui_resolutions=fhd
  `);

  fs.writeFileSync(path.join(root, 'source', 'main.brs'), undent`
    sub Main(_inputArguments as object)
      dt = createObject("roDateTime")
      print "BENCH_MAIN_START_MS="; (dt.asSecondsLong() * 1000) + dt.getMilliseconds()
      screen = createObject("roSGScreen")
      m.port = createObject("roMessagePort")
      screen.setMessagePort(m.port)
      scene = screen.CreateScene("MainScene")
      screen.show()
      scene.observeField("appExit", m.port)
      scene.setFocus(true)

      while true
        msg = wait(0, m.port)
        msgType = type(msg)
        if msgType = "roSGScreenEvent" then
          if msg.isScreenClosed() then return
        else if msgType = "roSGNodeEvent" then
          if msg.getField() = "appExit" then return
        end if
      end while
    end sub
  `);

  fs.writeFileSync(path.join(root, 'components', 'scene', 'MainScene.brs'), undent`
    sub init()
    end sub
  `);

  const childTags = [];
  for (let i = 1; i <= componentCount; i++) {
    childTags.push(`<Component${String(i).padStart(3, '0')}/>`);
  }
  childTags.push('<FinalLogger/>');
  const childrenBlock = childTags.map((t) => `      ${t}`).join('\n');

  fs.writeFileSync(path.join(root, 'components', 'scene', 'MainScene.xml'), undent`
    <?xml version="1.0" encoding="utf-8" ?>
    <component name="MainScene" extends="Scene">
      <script type="text/brightscript" uri="MainScene.brs" />

      <interface>
        <field id="appExit" type="bool" alwaysnotify="true" value="false"/>
      </interface>

      <children>
${childrenBlock}
      </children>
    </component>
  `);

  for (let i = 1; i <= componentCount; i++) {
    const name = `Component${String(i).padStart(3, '0')}`;
    writeComponent(root, name, dummyBrs(name), inline);
  }
  writeComponent(root, 'FinalLogger', finalLoggerBrs(), inline);
}

function dummyBrs(name) {
  return undent`
    sub init()
    end sub

    function ${name}_funcA() as integer
      return 0
    end function

    function ${name}_funcB() as integer
      return 0
    end function

    function ${name}_funcC() as integer
      return 0
    end function

    function ${name}_funcD() as integer
      return 0
    end function
  `;
}

function finalLoggerBrs() {
  return undent`
    sub init()
      dt = createObject("roDateTime")
      print "BENCH_FINAL_LOGGER_INIT_MS="; (dt.asSecondsLong() * 1000) + dt.getMilliseconds()
    end sub
  `;
}

function writeComponent(root, name, brs, inline) {
  const dir = path.join(root, 'components', 'generated');
  if (inline) {
    fs.writeFileSync(path.join(dir, `${name}.xml`), undent`
      <?xml version="1.0" encoding="utf-8" ?>
      <component name="${name}" extends="Group">
        <script type="text/brightscript"><![CDATA[
${brs}
        ]]></script>
      </component>
    `);
  } else {
    fs.writeFileSync(path.join(dir, `${name}.brs`), brs);
    fs.writeFileSync(path.join(dir, `${name}.xml`), undent`
      <?xml version="1.0" encoding="utf-8" ?>
      <component name="${name}" extends="Group">
        <script type="text/brightscript" uri="${name}.brs" />
      </component>
    `);
  }
}

class Telnet {
  constructor(host) {
    this.host = host;
    this.socket = null;
    this.buffer = '';
    this.lastDataAt = Date.now();
    this.listeners = new Set();
  }

  connect() {
    return new Promise((resolve, reject) => {
      const sock = net.createConnection({ host: this.host, port: TELNET_PORT });
      sock.once('error', reject);
      sock.once('connect', () => {
        sock.off('error', reject);
        sock.on('error', (err) => console.error('\n[telnet error]', err.message));
        sock.on('close', () => console.error('\n[telnet closed]'));
        sock.on('data', (chunk) => {
          const s = chunk.toString('utf8');
          this.buffer += s;
          this.lastDataAt = Date.now();
          process.stdout.write(s);
          if (/Console connection is already in use/i.test(s)) {
            console.error(
              '\n[fatal] Roku reports port 8085 is already in use by another telnet client.\n' +
                '        Close any active VS Code BrightScript debug session, other telnet clients, or stale terminals connected to this device, then re-run.',
            );
            process.exit(1);
          }
          for (const cb of this.listeners) cb();
        });
        this.socket = sock;
        resolve();
      });
    });
  }

  resetBuffer() {
    this.buffer = '';
  }

  async waitForLine(regex, timeoutMs = RUN_TIMEOUT_MS) {
    if (regex.test(this.buffer)) return;
    return new Promise((resolve, reject) => {
      const timer = setTimeout(() => {
        this.listeners.delete(cb);
        reject(new Error(`Timeout waiting for ${regex}`));
      }, timeoutMs);
      const cb = () => {
        if (regex.test(this.buffer)) {
          clearTimeout(timer);
          this.listeners.delete(cb);
          resolve();
        }
      };
      this.listeners.add(cb);
    });
  }

  async waitForSettle(idleMs = SETTLE_MS, maxMs = SETTLE_MAX_MS) {
    const deadline = Date.now() + maxMs;
    while (true) {
      const idle = Date.now() - this.lastDataAt;
      if (idle >= idleMs) return;
      if (Date.now() >= deadline) return;
      await sleep(Math.min(idleMs - idle + 10, 200));
    }
  }

  close() {
    if (this.socket) this.socket.destroy();
  }
}

function sleep(ms) {
  return new Promise((r) => setTimeout(r, ms));
}

function extractMetrics(buffer) {
  const compiles = [...buffer.matchAll(/AppCompileComplete[^)]*\((\d+)\s*ms\)/g)]
    .map((m) => parseInt(m[1], 10));
  const launch = buffer.match(/AppLaunchChainComplete[^)]*\((\d+)\s*ms\)/);
  const mainMs = buffer.match(/BENCH_MAIN_START_MS=\s*(\d+)/);
  const finalMs = buffer.match(/BENCH_FINAL_LOGGER_INIT_MS=\s*(\d+)/);
  const runMs =
    mainMs && finalMs ? parseInt(finalMs[1], 10) - parseInt(mainMs[1], 10) : null;
  return {
    compile1: compiles[0] ?? null,
    compile2: compiles[1] ?? null,
    launchChain: launch ? parseInt(launch[1], 10) : null,
    runMs,
  };
}

async function runScenario(scenario, opts, telnet) {
  console.log(`\n========== Scenario: ${scenario.title} (${opts.runs} runs) ==========`);
  const results = [];
  for (let i = 1; i <= opts.runs; i++) {
    console.log(`\n----- ${scenario.folder} run ${i}/${opts.runs} -----`);
    telnet.resetBuffer();
    try {
      const deployOpts = {
        host: opts.host,
        password: opts.password,
        rootDir: path.resolve(scenario.folder),
        stagingDir: path.resolve('out', `${scenario.folder}-staging`),
        outDir: path.resolve('out'),
        outFile: `${scenario.folder}.zip`,
      };
      await rokuDeploy.stage(deployOpts);
      await rokuDeploy.zip(deployOpts);
      await rokuDeploy.sideload(deployOpts);
      await telnet.waitForLine(/BENCH_FINAL_LOGGER_INIT_MS=\s*\d+/);
      await telnet.waitForSettle();
      const metrics = extractMetrics(telnet.buffer);
      results.push({ run: i, ...metrics });
    } catch (err) {
      console.error(`\n[run ${i} failed] ${err.message}`);
      results.push({
        run: i,
        compile1: null,
        compile2: null,
        launchChain: null,
        runMs: null,
        error: err.message,
      });
    }
    try {
      await rokuDeploy.keyPress({ host: opts.host, key: 'Home' });
    } catch (err) {
      console.error(`\n[home press failed] ${err.message}`);
    }
    await telnet.waitForSettle();
  }
  return results;
}

function avg(results, key) {
  const vals = results.map((r) => r[key]).filter((v) => typeof v === 'number');
  if (!vals.length) return null;
  return Math.round(vals.reduce((a, b) => a + b, 0) / vals.length);
}

function printTable(title, results) {
  console.log(`\n=== ${title} ===`);
  console.log('| Run | Compile 1 | Compile 2 | Run (Main→FinalLogger) | LaunchChain |');
  console.log('|-----|-----------|-----------|------------------------|-------------|');
  for (const r of results) {
    const cell = (v) => (v == null ? '   ?  ' : `${v} ms`);
    console.log(
      `| ${String(r.run).padStart(3)} | ${cell(r.compile1).padStart(9)} | ${cell(r.compile2).padStart(9)} | ${cell(r.runMs).padStart(22)} | ${cell(r.launchChain).padStart(11)} |`,
    );
  }
  const avgCell = (v) => (v == null ? '   ?  ' : `${v} ms`);
  console.log(
    `| avg | ${avgCell(avg(results, 'compile1')).padStart(9)} | ${avgCell(avg(results, 'compile2')).padStart(9)} | ${avgCell(avg(results, 'runMs')).padStart(22)} | ${avgCell(avg(results, 'launchChain')).padStart(11)} |`,
  );
}

async function main() {
  const opts = parseArgs();
  console.log(`Host: ${opts.host}    Runs per scenario: ${opts.runs}    Components: ${opts.components}`);

  console.log('Regenerating cdata/ and codebehind/...');
  regenerateScenarios(opts.components);

  console.log(`Connecting to telnet at ${opts.host}:${TELNET_PORT}...`);
  const telnet = new Telnet(opts.host);
  await telnet.connect();
  console.log('Connected. Pressing Home for clean baseline...');
  try {
    await rokuDeploy.keyPress({ host: opts.host, key: 'Home' });
  } catch (err) {
    console.error(`[initial home press failed] ${err.message}`);
  }
  await telnet.waitForSettle();

  const allResults = {};
  for (const scenario of SCENARIOS) {
    allResults[scenario.title] = await runScenario(scenario, opts, telnet);
  }

  console.log('\n\n############ SUMMARY ############');
  for (const [title, results] of Object.entries(allResults)) {
    printTable(title, results);
  }

  telnet.close();
}

try {
  await main();
} catch (err) {
  console.error('\nFatal:', err);
  process.exit(1);
}

package.json

{
  "name": "roku-compile-bench",
  "private": true,
  "version": "0.0.0",
  "type": "module",
  "description": "Benchmarks Roku compile/launch times for inline CDATA vs codebehind components.",
  "scripts": {
    "bench": "node bench.js"
  },
  "dependencies": {
    "roku-deploy": "^4.0.0-alpha.2",
    "undent": "^1.0.0"
  }
}

Roku Compile-Time Benchmark: Inline CDATA vs. Codebehind

Setup

Two synthetic Roku channels generated by the same script, identical in every way except how each child component's BrightScript is attached to its XML:

• CDATA: BrightScript embedded inline via <script type="text/brightscript"><![CDATA[ ... ]]></script>
• Codebehind: BrightScript in a sibling .brs file, referenced via <script type="text/brightscript" uri="ComponentNNN.brs" />

Each project contains:

• A slim Main that prints BENCH_MAIN_START_MS=<epoch_ms> on its first line
• A MainScene with 100 child components (Component001–Component100) plus a FinalLogger
• Each component declares an empty init() plus 4 dummy functions
• FinalLogger.init() prints BENCH_FINAL_LOGGER_INIT_MS=<epoch_ms>

Metrics captured per run from the Roku device's beacon log on telnet port 8085:

Metric	Source
Compile 1	First AppCompileComplete duration
Compile 2	Second AppCompileComplete duration (Roku double-compiles every sideload)
Run	BENCH_FINAL_LOGGER_INIT_MS − BENCH_MAIN_START_MS
LaunchChain	AppLaunchChainComplete duration

Each scenario was run 10 times: sideload, capture beacons, press ECP Home, wait for telnet to go quiet, repeat.

Raw Results (10 runs each)

CDATA

Run	Compile 1	Compile 2	Run (Main→FinalLogger)	LaunchChain
1	363 ms	303 ms	90 ms	899 ms
2	361 ms	299 ms	88 ms	893 ms
3	358 ms	304 ms	88 ms	917 ms
4	368 ms	294 ms	88 ms	860 ms
5	361 ms	297 ms	89 ms	878 ms
6	529 ms	296 ms	93 ms	930 ms
7	389 ms	303 ms	88 ms	911 ms
8	375 ms	293 ms	90 ms	908 ms
9	480 ms	280 ms	85 ms	792 ms
10	359 ms	300 ms	88 ms	880 ms
avg	394 ms	297 ms	89 ms	887 ms

Codebehind

Run	Compile 1	Compile 2	Run (Main→FinalLogger)	LaunchChain
1	570 ms	326 ms	96 ms	953 ms
2	601 ms	321 ms	91 ms	925 ms
3	384 ms	331 ms	86 ms	891 ms
4	486 ms	302 ms	87 ms	779 ms
5	364 ms	302 ms	88 ms	772 ms
6	548 ms	335 ms	87 ms	1004 ms
7	368 ms	324 ms	88 ms	883 ms
8	401 ms	324 ms	88 ms	905 ms
9	364 ms	335 ms	93 ms	968 ms
10	579 ms	335 ms	88 ms	997 ms
avg	467 ms	324 ms	89 ms	908 ms

Comparison

Medians (resistant to outliers)

Metric	CDATA	Codebehind	Δ
Compile 1	365 ms	444 ms	+78 ms
Compile 2	298 ms	325 ms	+27 ms
Total compile	663 ms	769 ms	+106 ms (~16%)

Averages

Metric	CDATA	Codebehind	Δ
Compile 1	394 ms	467 ms	+73 ms
Compile 2	297 ms	324 ms	+27 ms
Total compile	691 ms	791 ms	+100 ms (~14%)
Run	89 ms	89 ms	0 ms
LaunchChain	887 ms	908 ms	+21 ms

Variance

• CDATA Compile 1: tight cluster of 358–389 ms across 8 of 10 runs, with two outliers at 480 and 529 ms.
• Codebehind Compile 1: half the runs (5/10) spike into 486–601 ms; no comparable tight cluster.
• Compile 2: codebehind is consistently 20–30 ms slower (302–335 ms vs. 280–304 ms).

Takeaways

1. Inline CDATA compiles ~14–16% faster than codebehind at 100 components on this device. The gap shows up in both compile passes but is largest on Compile 1.
2. Codebehind has noticeably higher variance on Compile 1 — half the runs spike. Likely cost of opening 100 extra .brs files vs. parsing the BrightScript inline as part of the XML the device is already reading.
3. Run-time after compile is identical (89 ms in both scenarios). This is purely a load/compile-phase cost.
4. AppLaunchChainComplete is within noise (~2% slower for codebehind), so end-user-perceived launch time is dominated by the compile delta, not anything that happens after Main starts.

Reproduce

node bench.js --host <roku-ip> --password <dev-password> --runs 10 --components 100