s3-diff script

!README.md

s3-diff

https://gist.github.com/mryhryki/3cb4024056b5ce2bf437c55bb750df34

.gitignore

package-lock.json
node_modules
**/*.js
.temp

cache.ts

import * as fs from "fs/promises";
import * as path from "path";
import { S3Object } from "./s3";

const getCacheDir = async (): Promise<string> => {
  const cacheDir = path.resolve(__dirname, '.temp', 'cache');
  await fs.mkdir(cacheDir, { recursive: true });
  return cacheDir;
};

export const getCache = async (key: string): Promise<S3Object[] | null> => {
  const cacheDir = await getCacheDir();
  const cacheFile = path.resolve(cacheDir, `${key}.json`)
  const exists: boolean = await fs.stat(cacheFile).then(() => true).catch(() => false)
  if (!exists) {
    return null
  }
  const cacheData = await fs.readFile(cacheFile)
  return JSON.parse(cacheData.toString('utf-8'))
};

export const putCache = async (key: string, objects: S3Object[]): Promise<void> => {
  const cacheDir = await getCacheDir();
  const cacheFile = path.resolve(cacheDir, `${key}.json`)
  await fs.writeFile(cacheFile, JSON.stringify(objects, null, 2))
}

index.ts

import { getS3Client, S3Object, S3Service } from "./s3";
// import { getCache, putCache } from "./cache";

const filter = (bucket: string, objects: S3Object[]): S3Object[] => {
  return objects.filter((object) => {
    if (`${object.Key}`.startsWith("archive/")) {
      return false;
    } else if (`${object.Key}`.startsWith("public/")) {
        return false;
    } else if (`${object.Key}`.includes(".DS_Store")) {
      return false;
    }
    return true;
  });
};

const listAllObjects = async (service: S3Service, bucket: string): Promise<S3Object[]> => {
  // const cacheData = await getCache(bucket);
  // if (cacheData != null) {
  //   return cacheData;
  // }

  const s3 = getS3Client(service);
  let nextToken: string | undefined = undefined;
  const objects: S3Object[] = [];

  do {
    const result = await s3.listObjectsV2({ Bucket: bucket, ContinuationToken: nextToken }).promise();
    result.Contents.forEach((object) => {
      if (!object.Key.endsWith("/")) {
        objects.push(object);
      }
    });
    nextToken = result.NextContinuationToken;
  } while (nextToken != null);

  // await putCache(bucket, objects);
  return objects;
};

type S3ObjectMap = { [s3Key: string]: S3Object }
const indexBy = (objects: S3Object[]): S3ObjectMap => {
  const objectMap: S3ObjectMap = {};
  objects.forEach((object) => {
    objectMap[object.Key.normalize()] = object;
  });
  return objectMap;
};

const diff = (source: S3ObjectMap, target: S3ObjectMap): S3Object[] => {
  return Object.entries(source).reduce((arr, [key, value]) => {
    if (target[key.normalize()] == null) {
      arr.push(value);
    }
    return arr;
  }, [] as S3Object[]);
};

const main = async (): Promise<void> => {
  const [appObjects, dataObjects] = await Promise.all([
    filter("mryhryki-app", await listAllObjects("aws", "mryhryki-app")),
    filter("mryhryki-data", await listAllObjects("wasabi", "mryhryki-data")),
  ]);

  const appObjectMap = indexBy(appObjects);
  const dataObjectMap = indexBy(dataObjects);

  const onlyExistsInApp = diff(appObjectMap, dataObjectMap);
  const onlyExistsInData = diff(dataObjectMap, appObjectMap);

  console.log("[onlyExistsInApp]");
  onlyExistsInApp.forEach((object) => {
    const { Key } = object
    console.log(`${Key}`);
  });
  console.log("");
  console.log("[onlyExistsInData]");
  onlyExistsInData.forEach(({ Key }) => {
    console.log(`- ${Key.normalize()}`);
  });
};
main();

package.json

{
  "name": "s3-diff",
  "version": "0.1.0",
  "author": "mryhryki",
  "scripts": {
    "build": "tsc ./index.ts",
    "exec": "node ./index.js",
    "start": "run-s build exec",
    "watch": "nodemon --watch . --exec 'npm start' --ext ts"
  },
  "license": "MIT",
  "dependencies": {
    "@types/aws-sdk": "^2.7.0",
    "@types/node": "^16.3.1",
    "aws-sdk": "^2.945.0",
    "nodemon": "^2.0.12",
    "npm-run-all": "^4.1.5",
    "typescript": "^4.3.5"
  }
}

s3.ts

import * as AWS from "aws-sdk";
import { Object } from "aws-sdk/clients/s3";

export type S3Object = Object

const AwsCredentials = new AWS.Credentials({
  accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? "",
  secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? "",
  sessionToken: process.env.AWS_SESSION_TOKEN ?? undefined,
});
const awsS3 = new AWS.S3({ apiVersion: "2006-03-01", credentials: AwsCredentials });

const WasabiCredentials = new AWS.Credentials({
  accessKeyId: process.env.WASABI_AWS_ACCESS_KEY_ID ?? "",
  secretAccessKey: process.env.WASABI_AWS_SECRET_ACCESS_KEY ?? "",
});
const endpoint = new AWS.Endpoint("s3.wasabisys.com");
const wasabi = new AWS.S3({ apiVersion: "2006-03-01", endpoint, credentials: WasabiCredentials });

export type S3Service = "aws" | "wasabi"
export const getS3Client = (service: S3Service): AWS.S3 => {
  if (service === "wasabi") {
    return wasabi;
  }
  return awsS3;
};