secrets data layout proposal...

data.js

let base = require('taskcluster-base');
let taskcluster = require('taskcluster-client');

/** Entities for secrets */
let Secret = base.Entity.configure({
  version:              1,
  signEntities:         true,
  partitionKey:         base.Entity.keys.HashKey('parent'),
  rowKey:               base.Entity.keys.StringKey('name'),
  properties: {
    parent:             base.Entity.types.String,
    name:               base.Entity.types.String,
    secret:             base.Entity.types.EncryptedJSON,
    expires:            base.Entity.types.Date,
    lastModified:       base.Entity.types.Date,
    lastDateSecretUsed: base.Entity.types.Date,
    lastDateChildUsed:  base.Entity.types.Date,
  },
});

Secret.prototype.json = function() {
  let name = this.name;
  if (this.parent.length > 0) {
    name = this.parent + '.' + this.name;
  }
  return {
    name:         name,
    secret:       this.secret,
    expires:      this.expires.toJSON(),
    lastModified: this.lastModified.toJSON(),
    lastDateUsed: this.lastDateUsed.toJSON(),
  };
};

/**
 * Ensure that secrets on the path exists and have:
 *  A) secret.expires >= expires (as given by argument),
 *  B) secret.lastDateChildUsed >= now - 6 hours
 *
 * This way we ensure that parents expire after their children, and we maintain
 * a lastDateChildUsed value that is no more than 6 hours out-dated. Hence,
 * avoiding updating entities every time we read a secret.
 */
Secret.ensureSecret = async function(path, expires) {
  let Secret = this;
  
  // Stop recurssion at the root
  if (path.length === 0) {
    return null;
  }
  
  // Round to date to avoid updating all the time
  expires = new Date(
    expires.getFullYear(),
    expires.getMonth(),
    expires.getDate(),
    0, 0, 0, 0
  );
  
  // Parse path
  if (!(path instanceof Array)) {
    path = path.split('.');
  }
  
  // Find parent and name
  let name    = path.pop() || '';
  let parent  = path.join('.');
  
  // Load secret if it exists
  let secret = await Secret.load({parent, name}, true);
  
  // If secret didn't exist or we need to update it, then we must update the
  // parents on the path too...
  if (!secret || secret.expires < expires ||
      secret.lastDateChildUsed < taskcluster.fromNow('- 6 hours')) {
    await Secret.ensureNode(path, expires);
  }
  
  // If secret doesn't exist, we create it
  if (!secret) {
    return await Secret.create({
      parent,
      name,
      expires,
      secret:             null,
      lastModified:       new Date(),
      lastDateSecretUsed: new Date(),
      lastDateChildUsed:  new Date(),
    });
  }
  
  // Update secret if necessary
  await secret.modify(secret => {
    if (secret.expires < expires) {
      secret.expires = expires;
    }
    if (secret.lastDateChildUsed < taskcluster.fromNow('- 6 hours')) {
      secret.lastDateChildUsed = new Date();
    }
  });
};

/**
 * Update lastDateUsed of this secret and lastDateChildUsed of all parents to
 * now(), if out-dated by more than 6 hours. This way we won't be constantly
 * updating, but still have a solid idea about which secrets are in use.
 */
Secret.prototype.updateLastDateUsed = async function() {
  let Secret = this.prototype;
  this.modify(async (secret) => {
    if (secret.lastDateUsed < taskcluster.fromNow('- 6 hours')) {
      // Update parents first to ensure they have a lastChildUsed that is
      // no more than 6 hours out-dated
      await Secret.ensureSecret(secret.parent, secret.expires);
      // Then set lastDateUsed to now, as we updated the parents
      secret.lastDateUsed = new Date();
    }
  });
};

/** Expire secrets that are past their expiration. */
Secret.expire = async function(now = new Date()) {
  assert(now instanceof Date, "now must be given as option");
  var count = 0;
  await this.scan({
    expires:  base.Entity.op.lessThan(now)
  }, {
    limit:    250, // max number of concurrent delete operations
    handler:  secret => { count++; return secret.remove(true); }
  });
  return count;
};



///////////////////////// USAGE


// Create a secret
async function(req, res) {
  //... scopes, etc...
  
  let path   = req.params.secret.split('.');
  let name   = path.pop() || '';
  let parent = path.join('.');
  await this.Secret.ensureSecret(parent, new Date(req.body.expires));
  let secret = await this.Secret.create({
    parent,
    name,
    expires:            new Date(req.body.expires),
    secret:             req.body.secret,
    lastModified:       new Date(),
    lastDateSecretUsed: new Date(),
    lastDateChildUsed:  new Date(),
  });
  
  res.reply(secret.json());
};


// Load a secret
async function(req, res) {
  //... scopes, etc...
  let path   = req.params.secret.split('.');
  let name   = path.pop() || '';
  let parent = path.join('.');
  let secret = await this.Secret.load({parent, name}, true);
  
  if (!secret) {
    return res.status(404).json({message: "Secret not found"});
  }
  
  // Update lastUsed (if necessary)
  await secret.updateLastDateUsed();
  
  // Reply with secret
  res.reply(secret.json());
};