Komodo Stack Cloning

#stack-factory.md

Disclaimer

stack-factory was written for Komodo 1.18. Hopefully future releases will render it unnecessary.

Backstory

I'm using Komodo to manage Docker and found myself wanting to deploy a "core" set of stacks running to every server. This turned out to be a bit of a challenge, because each stack in Komodo is tied to a single server. Stacks can be copied, but it’s point-in-time and they can diverge after that. Actions to the rescue!

Setup and usage

The idea behind stack-factory is to find all stacks with a specific tag and duplicate them for each tagged server.

Setup is pretty straightforward:

1. Create the "original" for each stack you want to clone. Get it fully set up and deployed to one server.
2. Add a tag to each of these stacks (stack-factory's default is core, but it's configurable).
3. Add a tag to each server that should receive the stacks. It's safe to tag the "originating" server from step #1. (And again, the default is core).
4. Add a new Action in Komodo named stack-factory and paste the contents of the script.
5. Make any necessary configuration changes at the top of the script.
6. Save and run!

Notes

• Because Komodo doesn't allow duplicate stack names, the cloned stacks are called <stack name>_<server hash>.
• Multiple sets of stack and server tags can be configured.
• Re-running the action will validate and update the clones from their originals.
• Cloned stacks receive their own (also-configurable) tag.
• I'm sure there are things that could be improved or optimized, but this is good enough for my use case. 🙃

stack-factory.ts

// This action will clone any stacks tagged to all targeted servers.

// Array of tags to use for stack duplication.
const config = [
  {
    source: 'core',  // Stacks with this tag will be duplicated.
    deployed: 'core-deployed',  // This tag will be added to stacks after they're duplicated.
    server: 'core',  // Only servers with this tag will be targeted.
  },
  {
    source: 'agent-core',  // Stacks with this tag will be duplicated.
    deployed: 'agent-core-deployed',  // This tag will be added to stacks after they're duplicated.
    server: 'agent-core',  // Only servers with this tag will be targeted.
  },
  {
    source: 'pihole',  // Stacks with this tag will be duplicated.
    deployed: 'pihole-deployed',  // This tag will be added to stacks after they're duplicated.
    server: 'pihole',  // Only servers with this tag will be targeted.
  },
  {
    source: 'searxng',  // Stacks with this tag will be duplicated.
    deployed: 'searxng-deployed',  // This tag will be added to stacks after they're duplicated.
    server: 'searxng',  // Only servers with this tag will be targeted.
  }
];

const debug = false;

// ------------------------------------------

// Per-server configs are stored in the source stack's Environment in TOML format:
// # !!!stack-factory-server-config!!!
// # [server_name]
// # VAR1=Value1
// # VAR2=Value2
// # !!!stack-factory-server-config!!!
const stackFactoryConfigDelimiter = '#\\s*!!!stack-factory-server-config!!!';
const stackFactoryConfigRegex = new RegExp(`${stackFactoryConfigDelimiter}(?<config>[\\s\\S]+?)${stackFactoryConfigDelimiter}`);

// When updating a previously-duplicated stack config from the source,
// these config keys will be omitted.
const ignoreConfigKeysInUpdate = {
  server_id: true,
};



/**
 * Type of data produced by `parseIniFile`
 */
type INI_Data<T> = { [name: string]: T | { [name: string]: T } };
/**
 * Callback type for optional value converter used by `parseIniFile`.
 *
 * Note that `section` is `undefined` when it is global.
 */
type INI_ConvertCB<T> = (cb: { key: string, value: string, section?: string }) => T;

/**
 * Parses INI-formatted data, with optional value-type converter.
 * https://stackoverflow.com/a/79667584
 *
 * - section `[name]` namespaces are supported:
 *   - When a section appears multiple times, its inner values are extended.
 *   - Sections called `global` (case-insensitive) expose global variables.
 * - each variable must be in the form of `name = value`
 * - spaces surrounding `=` or `value` are ignored
 * - the `value` is taken until the end of line
 * - lines that start with `;` or `#` are skipped
 */
function parseIniFile<T = string>(iniFile: string, cb?: INI_ConvertCB<T>): INI_Data<T> {
    const lines = iniFile
        .replace(/\r/g, '')
        .split('\n')
        .map(a => a.trim())
        .filter(f => f.length > 1 && f[0] !== ';' && f[0] !== '#');

    const result: INI_Data<T> = {};
    let root: any = result, section: string | undefined;
    for (const a of lines) {
        const m = a.match(/^\s*([\w$][\w.$]*)\s*=\s*(.*)/);
        if (m) {
            const key = m[1], value = m[2];
            root[key] = typeof cb === 'function' ? cb({key, value, section}) : value;
        } else {
            const s = a.match(/\[\s*([\w$.-]+)\s*]/);
            if (s) {
                section = s[1];
                if (section.toLowerCase() === 'global') {
                    root = result;
                    section = undefined;
                } else {
                    root = result[section] ??= {};
                }
            }
        }
    }
    return result;
}



// Replace per-server environment variables.
function updateEnvironmentForServer(serverName: string, environment: string, perServerConfig: object) {
  if (perServerConfig[serverName]) {
    for (const envVar in perServerConfig[serverName]) {
      environment = environment.replace(new RegExp(`^\\s*${envVar}\\s*=.*`, 'gm'), `${envVar}=${perServerConfig[serverName][envVar]}`)
    }
  }
  return environment
}

// Need a way to compare stack configs.
function deepEqual(x: object, y: object) {
  const ok = Object.keys, tx = typeof x, ty = typeof y;
  return x && y && tx === 'object' && tx === ty ? (
    ok(x).length === ok(y).length &&
      ok(x).every(key => deepEqual(x[key], y[key]))
  ) : (x === y);
}

// RegExp.escape() isn't available yet.
function escapeRegex(str: string) {
    return str.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&');
}

// Hash function to get unique-enough suffixes for cloned stacks.
async function sha256(message: string, abbrev?: number) {
    // encode as UTF-8
    const msgBuffer = new TextEncoder().encode(message);                    

    // hash the message
    const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer);

    // convert ArrayBuffer to Array
    const hashArray = Array.from(new Uint8Array(hashBuffer));

    // convert bytes to hex string                  
    const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
    
    return hashHex.substring(0, abbrev || null);
}


// Error tracking.
let errors = [];

// Loop through all tags that need to be duplicated.
for (const conf of config) {

  console.log('🧬 Config:', conf);

  // Figure out the ID of the core-deployed tag. Needed for cloning.
  let deployedTagId;
  const tags = await komodo.read('ListTags', {});
  const deployedTag = tags.filter(tag => tag.name == conf.deployed);
  if (deployedTag.length > 0) {
    deployedTagId = deployedTag[0]._id.$oid;
  } else {
    try {
      if (debug) {
        console.log(`🔶 DEBUG MODE -- would have created ${conf.deployed} tag`)
      } else {
        const deployedTagInfo = await komodo.write('CreateTag', { name: conf.deployed });
        deployedTagId = deployedTagInfo._id.$oid;
      }
    } catch(e) {
      console.log(`🛑 Tag for deployed stacks (${conf.deployed}) does not exist and could not be created:`);
      console.log(e);
      errors.push(e);
      continue;
    }
  }

  // Find all objects we need to do the work.
  const sourceStacks = await komodo.read('ListStacks', {query: {tags: [conf.source]}});
  const deployedStacks = await komodo.read('ListStacks', {query: {tags: [conf.deployed]}});
  const servers = await komodo.read('ListServers', {query: {tags: [conf.server]}});

  
  for (const sourceStack of sourceStacks) {

    let sourceStackName = sourceStack.name;

    if (sourceStack.info.server_id != '') {
      const deployedServer = await komodo.read('GetServer', {server: sourceStack.info.server_id});
      const deployedServerRegexName = escapeRegex(deployedServer.name)
      // Remove the source server name from the stack name.
      sourceStackName = sourceStackName.replace((new RegExp(`(^${deployedServerRegexName}_|_${deployedServerRegexName}|$)`)), '')
    }

    console.log('');
    console.log(`🥞 Source Stack: ${sourceStackName}`);
    const stackInfo = await komodo.read('GetStack', {stack: sourceStack.id});

    // Parse out the per-server config.
    let perServerConfig = {};
    const configParse = stackFactoryConfigRegex.exec(stackInfo.config.environment);
    if (configParse && configParse.groups && configParse.groups.config) {
      try {
        perServerConfig = parseIniFile(configParse.groups.config.replace(/^#\s*/gm, ''));
        //perServerConfig = TOML.parse(configParse.groups.config.replace(/^#\s*/gm, ''));
      } catch(e) {
        console.log(`🔸 Error parsing per-server config: ${e}`);
        errors.push(e);
      }
    }

    const originalEnvironment = stackInfo.config.environment;

    for (const server of servers) {

      let deploy = false;
      // Used for deployment if needed.
      let stackId = '';

      console.log(`🖥️ Server: ${server.name}`);

      // Since we're cloning stacks from one server to others, we need to avoid doing anything to the source stack.
      if (sourceStack.info.server_id == server.id) {
        console.log(`  ✅ ${server.name} is the source for this stack; no action needed.`);
        continue;
      }

      // Update the environment if there are per-server settings.
      stackInfo.config.environment = updateEnvironmentForServer(server.name, originalEnvironment, perServerConfig)

      // Have we already deployed the source stack to this server?
      const alreadyDeployed = deployedStacks.filter(function (deployedStack) {
        return (
          deployedStack.info.server_id == server.id
          // Naming convention for deployed stacks is <source-stack-name>_<server-name>.
          && deployedStack.name.startsWith(`${sourceStackName}_`)
        );
      });

      if (alreadyDeployed.length > 0) {
        console.log(`  Already deployed to ${server.name}; checking settings...`);
        stackId = alreadyDeployed[0].id;

        const existingStack = await komodo.read('GetStack', {stack: stackId})
        // console.log(stackInfo.config);
        // console.log(existingStack);
        
        // Give the project the same name as the source stack for easy `docker compose ls` viewing.
        stackInfo.config.project_name = sourceStackName

        let mismatch = false;
        const updateParams = {
          'id': stackId,
          'config': {},
        }
        for (const prop in stackInfo.config) {
          if (ignoreConfigKeysInUpdate[prop]) {
            continue;
          }
          if (!deepEqual(existingStack.config[prop], stackInfo.config[prop])) {
            mismatch = true
            // console.log(existingStack.config[prop])
            // console.log(stackInfo.config[prop])
            console.log(` > ${prop}: ${JSON.stringify(existingStack.config[prop])} -> ${JSON.stringify(stackInfo.config[prop])}`)
            updateParams.config[prop] = stackInfo.config[prop]
          }
        }

        if (mismatch) {
          console.log('✏️ Updating mismatched properties...');
          try {
            if (debug) {
              console.log('🔶 DEBUG MODE')
              console.log(updateParams)
            } else {
              await komodo.write('UpdateStack', updateParams);
              console.log('  ✅ Updated');
              if (alreadyDeployed[0].info.state == Types.StackState.Running) {
                deploy = true;
              } else {
                console.log('  ℹ️ Stack is not running and will not be redeployed')
              }
            }
          } catch(e) {
            console.log('  ⚠️ Failed: ');
            console.log(e);
            errors.push(e);
          } 
        } else {
          console.log('  ✅ All good');
        }

      } else {
        console.log(`  ✴️ Needs to be deployed to ${server.name}`);

        const newStackName = `${sourceStackName}_${await sha256(server.name, 6)}`;

        const newStackParams = {
          'name': newStackName,
          'config': Object.assign({},
            stackInfo.config,
            { 'server_id': server.id }
          ),
        }

        // console.log(newStackParams);
        console.log(`  🐑 Cloning and tagging ${sourceStackName}...`);
        try {
          if (debug) {
            console.log('🔶 DEBUG MODE')
            console.log(newStackParams)
          } else {
            const newStack = await komodo.write('CreateStack', newStackParams);
            stackId = newStack._id.$oid
            // Set the tag on the new stack.
            await komodo.write('UpdateResourceMeta', { target: { type: 'Stack', id: stackId }, tags: [ deployedTagId ] })
            console.log('  ✅ Created');
            deploy = true;
          }
        } catch(e) {
          console.log('  ⚠️ Failed: ');
          console.log(e);
          errors.push(e);
        }
      }

      if (deploy) {
        console.log('  🚀 Deploying');
        try {

          if (debug) {
            console.log('  🔶 DEBUG MODE - not deploying')
          } else {
            await komodo.execute('DeployStack', { stack: stackId })
            console.log('  ✅ Deployed');
          }
        } catch(e) {
          console.log('  ⚠️ Failed: ');
          console.log(e);
          errors.push(e);
        }
      }
      
    }  // 🖥️ Servers.

  }  // 🥞 Stacks.

  console.log(''); console.log('');
}  // 🧬 Configs.


if (errors.length > 0) {
  throw `⚠️ ${errors.length} errors encountered.`
}