johndstein

#!/usr/bin/env bash

# minimum size to get benefit from AWS S3 intelligent tiering.
size=128000

function help() {
  echo ""
  echo "Usage: s3-stats.sh (options)"
  echo ""
  echo "  Get counts and percent of files over a given size."

johndstein

#!/usr/bin/env bash
# list empty buckets
buckets=$(aws s3api list-buckets| jq --raw-output '.Buckets[].Name')
for bucket in $buckets;do
  files=$(aws s3 ls $bucket)
  if [[ -z "$myVar" ]]; then
    echo "$bucket"
  fi
done

johndstein

#!/usr/bin/env bash

help () {
    echo "
Usage:

test/s3select.sh -v \\
  --bucket some-bucket \\
  --prefix foo/2020/10/20/20/57 \\
  --expression \"select * from s3object o where o.rawmsghostname = 'ynthtest-Oct20T20-filler-5b45a8d7'\"

johndstein

mongo --ssl \
  --host $(cat ~/infrastructure/keys/varsdb-creds | cut -d ':' -f2 | cut -d '@' -f2):27017 \
  --sslCAFile /opt/helix-tools/rds-combined-ca-bundle.pem \
  --username $(cat ~/infrastructure/keys/varsdb-creds | cut -d ':' -f1) \
  --password $(cat ~/infrastructure/keys/varsdb-creds | cut -d ':' -f2 | cut -d '@' -f1)

johndstein

#!/usr/bin/env node
'use strict';
class Log {
  constructor(options) {
    options = options || {};
    this.level = 'off';
    this.levels = {
      debug: 0,
      info: 1,
      warn: 2,

johndstein

#!/usr/bin/env node
'use strict';
const AWS = require('aws-sdk');
if (process.env.AWS_PROFILE) { // eslint-disable-line
  AWS.config.credentials = new AWS.SharedIniFileCredentials({
    profile: process.env.AWS_PROFILE // eslint-disable-line
  });
  AWS.config.logger = console;
}
const s3opts = {};

johndstein

#!/usr/bin/env node
'use strict';
const AWS = require('aws-sdk');
if (process.env.AWS_PROFILE) { // eslint-disable-line
  AWS.config.credentials = new AWS.SharedIniFileCredentials({
    profile: process.env.AWS_PROFILE // eslint-disable-line
  });
  AWS.config.logger = console;
}
const s3opts = { apiVersion: '2006-03-01' };

johndstein

#!/usr/bin/env node
'use strict';
const AWS = require('aws-sdk');
if (process.env.AWS_PROFILE) { // eslint-disable-line
  AWS.config.credentials = new AWS.SharedIniFileCredentials({
    profile: process.env.AWS_PROFILE // eslint-disable-line
  });
  AWS.config.logger = console;
}
const s3opts = { apiVersion: '2006-03-01' };

johndstein

#!/usr/bin/env node
'use strict';
const AWS = require('aws-sdk');
if (process.env.AWS_PROFILE) { // eslint-disable-line
  AWS.config.credentials = new AWS.SharedIniFileCredentials({
    profile: process.env.AWS_PROFILE // eslint-disable-line
  });
}
const s3opts = { apiVersion: '2006-03-01' };
if (process.env.AWS_S3_ENDPOINT) { // eslint-disable-line

johndstein

const AWS = require('aws-sdk')
if (process.env.AWS_PROFILE) { // eslint-disable-line
  AWS.config.credentials = new AWS.SharedIniFileCredentials({
    profile: process.env.AWS_PROFILE // eslint-disable-line
  })
  AWS.config.logger = console
}
const s3 = new AWS.S3({ apiVersion: '2006-03-01' })
exports.handler = async(event, context) => {
    const objects = await s3.listObjects(event).promise();