0-Cameron-Code-and-Test.md

MyCRT Code and Test: Cameron Taylor

Description

The following code provides an interface to a file storage backend for MyCRT. There is an abstract class StorageBackend which is implemented in subclasses LocalBackend and S3Backend. When MyCRT is running locally for development, the LocalBackend is used, but when MyCRT is running remotely, the S3Backend is used.

Technology

All of the code is written in TypeScript and tested with mocha and chai. To mock or spy certain objects, ts-mockito is used.

1-backend.ts

export abstract class StorageBackend {

   public abstract async exists(key: string): Promise<boolean>;

   public abstract async allMatching(dirPrefix: string, pattern: RegExp): Promise<string[]>;

   public abstract async readJson<T>(key: string): Promise<T>;

   public abstract async writeJson<T>(key: string, value: T): Promise<void>;

   public abstract async deleteJson(key: string): Promise<void>;

   public abstract async deletePrefix(dirPrefix: string): Promise<void>;

   public abstract rootDirectory(): string;

}

2-local-backend.ts

import fs = require('fs-extra');
import path = require('path');

import Logging = require('../logging');

import { StorageBackend } from './backend';

const logger = Logging.defaultLogger(__dirname);

export class LocalBackend extends StorageBackend {
   constructor(private rootDir: string) {
      super();

      if (!fs.existsSync(this.rootDir)) {
         fs.mkdirsSync(this.rootDir);
      }
   }

   public rootDirectory(): string { return this.rootDir; }

   public async exists(key: string): Promise<boolean> {
      const file = path.join(this.rootDir, key);
      return await fs.pathExists(file);
   }

   public async allMatching(dirPrefix: string, pattern: RegExp): Promise<string[]> {
      const fullDirPrefix = path.join(this.rootDir, dirPrefix);
      const result: string[] = [];

      if (fs.existsSync(fullDirPrefix)) {
         fs.readdirSync(fullDirPrefix).forEach((file) => {
            if (file.match(pattern)) {
               result.push(path.join(dirPrefix, file));
            }
         });
      }

      return result;
   }

   public async readJson<T>(key: string): Promise<T> {
      const file = path.join(this.rootDir, key);
      return fs.readJsonSync(file);
   }

   public async writeJson<T>(key: string, value: T): Promise<void> {
      const file = path.join(this.rootDir, key);
      const dir = path.dirname(file);
      if (!fs.existsSync(dir)) {
         fs.mkdirsSync(dir);
      }
      await fs.writeJSON(file, value);
   }

   public async deleteJson(key: string): Promise<void> {
      const file = path.join(this.rootDir, key);
      return await fs.unlink(file);
   }

   public async deletePrefix(dirPrefix: string): Promise<void> {
      dirPrefix = path.join(this.rootDir, dirPrefix);
      if (await fs.pathExists(dirPrefix)) {
         await fs.remove(dirPrefix);
      }
   }

}

3-s3-backend.ts

import { AWSError, S3 } from 'aws-sdk';

import Logging = require('../logging');

import { StorageBackend } from './backend';

const logger = Logging.defaultLogger(__dirname);

export class S3Backend extends StorageBackend {
   constructor(private s3: S3, private bucket: string) {
      super();
   }

   public rootDirectory(): string { return this.bucket; }

   public exists(key: string): Promise<boolean> {

      const params: S3.HeadObjectRequest = {
         Bucket: this.bucket,
         Key: key,
      };

      return new Promise<boolean>((resolve, reject) => {
         this.s3.headObject(params, (err: AWSError, data: S3.HeadObjectOutput) => {
            if (err && err.code === 'NotFound') {
               resolve(false);
            } else if (!err) {
               resolve(true);
            } else {
               reject(err);
            }
         });
      });
   }

   public async allMatching(dirPrefix: string, pattern: RegExp): Promise<string[]> {
      const keys = await this.listObjects(dirPrefix);
      const result: string[] = [];
      keys.forEach((key) => {
         const check = key.substring(dirPrefix.length);
         if (check.match(pattern)) {
            result.push(key);
         }
      });
      return result;
   }

   public async readJson<T>(key: string): Promise<T> {

      const params: S3.GetObjectRequest = {
         Bucket: this.bucket,
         Key: key,
      };

      return new Promise<T>((resolve, reject) => {
         this.s3.getObject(params, (err: AWSError, data: S3.GetObjectOutput) => {

            if (err) {
               logger.error(err.message);
               reject(err.code);
            } else {
               const body: string = data.Body!.toString();
               const result: T = JSON.parse(body) as T;
               resolve(result);
            }

         });
      });
   }

   public async writeJson<T>(key: string, value: T): Promise<void> {

      const params: S3.PutObjectRequest = {
         Body: Buffer.from(JSON.stringify(value)),
         Bucket: this.bucket,
         Key: key,
      };

      return new Promise<void>((resolve, reject) => {
         this.s3.putObject(params, (err: AWSError, data: S3.PutObjectOutput) => {

            if (err) {
               logger.error(err.message);
               reject(err.code);
            } else {
               resolve();
            }

         });
      });
   }

   public async deleteJson(key: string): Promise<void> {

      const params: S3.DeleteObjectRequest = {
         Bucket: this.bucket,
         Key: key,
      };

      return new Promise<void>((resolve, reject) => {
         this.s3.deleteObject(params, (err: AWSError, data: S3.DeleteObjectOutput) => {

            if (err) {
               logger.error(err.message);
               reject(err.code);
            } else {
               resolve();
            }

         });
      });
   }

   public async deletePrefix(dirPrefix: string): Promise<void> {
      const keys = await this.listObjects(dirPrefix);
      keys.forEach(async (key) => await this.deleteJson(key));
   }

   private async listObjects(prefix?: string): Promise<string[]> {
      const params: S3.ListObjectsRequest = {
         Bucket: this.bucket,
         Prefix: prefix,
      };
      return new Promise<string[]>((resolve, reject) => this.s3.listObjects(params, (err, data) => {
         if (err) {
            logger.error(err.message);
            reject(err);
         } else if (!data.Contents) {
            resolve([]);
         } else {
            resolve(data.Contents.map((s3Obj) => s3Obj.Key || ''));
         }
      }));
   }

}

4-local-backend.test.ts

import { expect } from 'chai';
import fs = require('fs-extra');
import 'mocha';
import path = require('path');
import uuid = require('uuid/v1');

import { LocalBackend } from '../../storage/local-backend';

import { dummyData, key } from './data';

describe("LocalBackend", () => {

   let backend: LocalBackend;
   let rootDir: string;

   before(() => {
      rootDir = path.join(__dirname, uuid());
      fs.mkdirSync(rootDir);
      backend = new LocalBackend(rootDir);
   });

   after(() => {
      fs.remove(rootDir);
   });

   it("should know if a file exists", async () => {
      expect(await backend.exists(key)).to.be.false;
      await backend.writeJson(key, dummyData);
      expect(await backend.exists(key)).to.be.true;
   });

   it("should write data, then read it back", async () => {

      await backend.writeJson(key, dummyData);
      const result = await backend.readJson(key) as any;

      expect(result.name).to.equal(dummyData.name);
      expect(result.age).to.equal(dummyData.age);

   });

   it("should delete files, and fail to read missing files", async () => {

      await backend.writeJson(key, dummyData);
      await backend.deleteJson(key);
      const result = await backend.readJson(key)
         .catch((reason) => {
            expect(reason).to.not.be.null;
         });

   });

});

5-s3-backend.test.ts

import { S3 } from 'aws-sdk';
import { expect } from 'chai';
import 'mocha';
import mockito from 'ts-mockito';

import { S3Backend } from '../../storage/s3-backend';

import { dummyData, key } from './data';

describe("S3Backend", () => {

   let s3: S3;
   let spiedS3: S3;
   let backend: S3Backend;

   before(() => {
      s3 = new S3();
      spiedS3 = mockito.spy(s3);

      mockito.when(spiedS3.getObject(mockito.anything(), mockito.anyFunction()))
         .thenCall((params, callback) => {
            callback(null, {
               Body: JSON.stringify(dummyData),
            });
         });

      mockito.when(spiedS3.putObject(mockito.anything(), mockito.anyFunction()))
         .thenCall((params, callback) => {
            callback();
         });

      mockito.when(spiedS3.deleteObject(mockito.anything(), mockito.anyFunction()))
         .thenCall((params, callback) => {
            callback();
         });

      backend = new S3Backend(s3, 'lil-test-environment');
   });

   it("should know if a file exists", async () => {

      mockito.when(spiedS3.headObject(mockito.anything(), mockito.anyFunction()))
         .thenCall((params, callback) => {
            callback({code: 'NotFound'});
         });
      expect(await backend.exists(key)).to.be.false;

      await backend.writeJson(key, dummyData);
      mockito.when(spiedS3.headObject(mockito.anything(), mockito.anyFunction()))
         .thenCall((params, callback) => {
            callback();
         });
      expect(await backend.exists(key)).to.be.true;
   });

   it("should write data, then read it back", async () => {

      await backend.writeJson(key, dummyData);
      const result = await backend.readJson(key) as any;

      expect(result.name).to.equal(dummyData.name);
      expect(result.age).to.equal(dummyData.age);

   });

   it("should delete files, and fail to read missing files", async () => {

      await backend.writeJson(key, dummyData);
      await backend.deleteJson(key);

      mockito.when(spiedS3.getObject(mockito.anything(), mockito.anyFunction()))
         .thenCall((params, callback) => {
            callback("file does not exist", null);
         });

      const result = await backend.readJson(key)
         .catch((reason) => {
            expect(reason).to.not.be.null;
         });

   });

});