AWS DMS and DB Monitor sls project

serverless.yml

service: appeals
frameworkVersion: ">=1.0.0 <2.0.0"
package:
  individually: true
  exclude:
    - .git/**
    - lib/**
    - .venv/**
    - .venvdev/**
    - .vscode/**
    - __pycache__/**
    - node_modules/**

provider:
  name: aws
  runtime: python3.6
  stage: dev
  region: us-gov-west-1
  memorySize: 512
  logRetentionInDays: 14
  role: arn:aws-us-gov:iam::008577686731:role/dsva-appeals-lambda-monitor
  stackTags:
    team: "dsva-appeals"
    dev: "devops"
  environment:
    CREDSTASH_TABLE: "appeals-credstash"
    HOSTALIASES: "/tmp/HOSTALIASES"

custom:
  pythonRequirements:
    dockerizePip: true

#############
# FUNCTIONS #
#############
functions:

  createRDSSnapshot:
    handler: utils/rds.takeSnapshot
    description: Creates a RDS Snapshot based of 1 param [ source_db_id ] with the provided tags
  
  createRDFromSnapshot:
    handler: utils/rds.createRDSFromSnapshot
    description: Creates an RDS from Snapshot based on 2 params [source_db_id, db_snapshot_id]
  
  snapshotStatus:
    handler: utils/rds.isSnapshotReady
    description: Asks AWS if the Snapshot is available or not
  
  setVACOLSBackupCreds:
    handler: vacols/credentials.setVACOLSBackupCreds
    description: Sets VACOLS Backup DB Credentials
  
  unsetVACOLSBackupCreds:
    handler: vacols/credentials.unsetVACOLSBackupCreds
    description: Unset VACOLS Backup DB Credentials
  
  vacolsDDL:
    handler: vacols/backup.getSchemaDDL
    description: Gets the entire DDL from VACOLS VA Oracle DB and saves the file to S3
    timeout: 300
    package:
      # include oracle dependencies
      include:
        - lib/**
    vpc:
      securityGroupIds:
        - sg-1a2fbf7e
      subnetIds:
        - subnet-7f220808
        - subnet-5ab9833f
  
  importOneDDL:
    handler: vacols/backup.importOneDDL
    description: Imports one DDL at the time based of a list that exists in ssm parameters.
    timeout: 900
    package:
      # include oracle dependencies
      include:
        - lib/**
    vpc:
      securityGroupIds:
        - sg-1a2fbf7e
      subnetIds:
        - subnet-7f220808
        - subnet-5ab9833f
      
  vacolsBackupRDSStatus:
    handler: utils/rds.isRDSReady
    description: Gets the Status of the RDS that we are creating to perform the backup
  
  terminateRDS:
    handler: utils/rds.terminateRDS
    description: Terminates the RDS that we used for the backup

  setSlackVars:
    handler: utils/slackCredentials.setCreds
    description: Set Slack Webhook
  
  unsetSlackVars:
    handler: utils/slackCredentials.unsetCreds
    description: Unsets Slack Webhook

  vacolsSetVars:
    handler: vacols/credentials.setCreds
    description: Get and Set temporary vacols credentials to insert the SNOTES.

  insertSnotes:
    handler: vacols/insert_snotes.insert_snotes_vacols
    description: Insert SNOTES in VACOLS Prod DB, use as monitor for DMS Task
    timeout: 30
    package:
      # include oracle dependencies
      include:
        - lib/**
    vpc:
      securityGroupIds:
        - sg-1a2fbf7e
      subnetIds:
        - subnet-7f220808
        - subnet-5ab9833f

  vacolsUnsetVars:
    handler: vacols/credentials.unsetCreds
    description: Unset all temporary vacols credentials

  setDMSCreds:
    handler: utils/dmsCredentials.setCreds
    description: Set Temporary Credentials for DMS Task
  
  queryDMS:
    handler: dms/query.queryOne
    description: Query the DMS RDS to monitor if its up to date (only one record)
    timeout: 30
    package:
        # include oracle dependencies
        include:
          - lib/**
    vpc:
      securityGroupIds: # spoke security groups
        - sg-1a2fbf7e
      subnetIds: # spoke vpcs
        - subnet-7f220808
        - subnet-5ab9833f
  
  validateDMS:
    handler: utils/date.lessThanOneHour
    description: Compare two dates and validate if its less than 1 hour
  
  slackNotification:
    handler: utils/slack.notify
    description: Sends a slack notification
  
  stopDMSTask:
    handler: utils/dms.stopDMS
    description: Stops the DMS Task 
  
  startDMSTask:
    handler: utils/dms.startDMS
    description: Starts the DMS task
  
  unsetDMSCreds:
    handler: utils/dmsCredentials.unsetCreds
    description: Unsets temporary credentials for DMS Task

########################
#     STEP FUNCTIONS   #
########################
  
stepFunctions:
  stateMachines:
    # VACOLS Backup Step Function
    # This set of functions, creates a snapshot from the RDS used in the DMS Task
    # Then connects to VACOLS VA DB to get the DDL (triggers, database, etc)
    # downloads them and save them to S3 for historical purposes
    # then imports them into a new RDS created from the snapshot that was created in the first step
    # Finally we take a snapshot of everything, since it now includes triggers, indexes, tablespaces, and data. essentially everything. a full backup.
    VACOLSBackup:
      name: VACOLSBackup
      role: arn:aws-us-gov:iam::008577686731:role/StepFunctionsLambdaRole
      events:
        - schedule:
            rate: cron(0 22 * * ? *)
            enabled: true
            input: 
              env: prod
              source_db_id: dsva-appeals-spoke-prod-vacols-dr
              dms:
                task_arn: dsva-appeals-vacols-spoke-prod-dms-task

      definition:
        StartAt: StopDMS
        States:
          StopDMS:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-stopDMSTask
            Next: TakeRDSSnapshot
          TakeRDSSnapshot:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-createRDSSnapshot
            Next: StartDMS
          StartDMS:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-startDMSTask
            Next: SetVACOLSCredsForDDL
          SetVACOLSCredsForDDL:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-setVACOLSBackupCreds
            Next: SnapshotStatus
          SnapshotStatus:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-snapshotStatus
            Next: IsSnapshotReady
          IsSnapshotReady:
            Type: Choice
            Choices:
              - Variable: "$.snapshot_ready"
                StringEquals: "true"
                Next: CreateRDSFromSnapshot
              - Variable: "$.snapshot_ready"
                StringEquals: "false"
                Next: WaitUntilSnapshotReady
          WaitUntilSnapshotReady:
            Type: Wait
            Seconds: 120
            Next: SnapshotStatus
          CreateRDSFromSnapshot:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-createRDFromSnapshot
            Next: GetVACOLSDDLs
          GetVACOLSDDLs:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-vacolsDDL
            Next: SetDMSCreds
          SetDMSCreds:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-setDMSCreds
            Next: GetSnapshotRDSStatus
          GetSnapshotRDSStatus:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-vacolsBackupRDSStatus
            Next: IsRDSFromSnapshotReady
          IsRDSFromSnapshotReady:
            Type: Choice
            Choices:
              - Variable: "$.rds_ready"
                StringEquals: "true"
                Next: LoadDDLOneByOne
              - Variable: "$.rds_ready"
                StringEquals: "false"
                Next: WaitUntilRDSFromSnapshotReady
          WaitUntilRDSFromSnapshotReady:
            Type: Wait
            Seconds: 60
            Next: GetSnapshotRDSStatus
          LoadDDLOneByOne:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-importOneDDL
            Next: LoadingDDLCompleted
          LoadingDDLCompleted:
            Type: Choice
            Choices:
              - Variable: "$.ddl_processing"
                StringEquals: "not_completed"
                Next: LoadDDLOneByOne
              - Variable: "$.ddl_processing"
                StringEquals: "completed"
                Next: CreateSnapshotForBackup
          CreateSnapshotForBackup:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-createRDSSnapshot
            Next: UnsetVACOLSCredsForDDL
          UnsetVACOLSCredsForDDL:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-unsetVACOLSBackupCreds
            Next: UnsetDMSCreds
          UnsetDMSCreds:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-unsetDMSCreds
            Next: BackupSnapshotStatus
          BackupSnapshotStatus:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-snapshotStatus
            Next: isBackupReady
          isBackupReady:
            Type: Choice
            Choices:
              - Variable: "$.snapshot_ready"
                StringEquals: "true"
                Next: TerminateRDS
              - Variable: "$.snapshot_ready"
                StringEquals: "false"
                Next: WaitforBackup
          WaitforBackup:
            Type: Wait
            Seconds: 120
            Next: BackupSnapshotStatus
          TerminateRDS:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-terminateRDS
            End: true


    # DMS Monitor Step Function
    # This set of functions queries the SNOTES in the DMS replica RDS
    # And validates if its less than 1 hour and sends an alert if not after
    # trying to resolve itself by stopping and starting the task
    DMSMonitor:
      name: DMSMonitor
      role: arn:aws-us-gov:iam::008577686731:role/StepFunctionsLambdaRole
      events:
        - schedule:
            rate: rate(4 hours)
            input: 
              env: prod
      definition:
        StartAt: SetVariables
        States:
          SetVariables:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-setDMSCreds
            Next: QueryDMS
          QueryDMS:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-queryDMS
            Next: Validate
          Validate:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-validateDMS
            Next: ValidOrNot
          ValidOrNot:
            Type: Choice
            Choices:
              - Variable: "$.alert"
                StringEquals: "true"
                Next: StopDMS
              - Variable: "$.started"
                StringEquals: "true"
                Next: SetSlack
          StopDMS:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-stopDMSTask
            Next: CheckDMSStop
          CheckDMSStop:
            Type: Choice
            Choices:
              - Variable: "$.inprogress"
                StringEquals: "false"
                Next: StartDMS
              - Variable: "$.inprogress"
                StringEquals: "true"
                Next: WaitStop
          WaitStop:
            Type: Wait
            Seconds: 30
            Next: StopDMS
          StartDMS:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-startDMSTask
            Next: CheckDMSStart
          CheckDMSStart:
            Type: Choice
            Choices:
              - Variable: "$.inprogress"
                StringEquals: "true"
                Next: WaitStart
              - Variable: "$.inprogress"
                StringEquals: "false"
                Next: SetVariables
          WaitStart:
            Type: Wait
            Seconds: 300
            Next: StartDMS
          SetSlack:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-setSlackVars
            Next: Notify
          UnsetSlack:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-unsetSlackVars
            Next: UnsetVariables
          Notify:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-slackNotification
            Next: UnsetSlack
          UnsetVariables:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-unsetDMSCreds
            End: true
    
    # VACOLS Insert Step function
    # This set of functions insert a snote in the VACOLS Prod DB
    # To later test it with the DMS Monitor using the DMS Service
    VACOLSMonitor:
      name: vacolsMonitor
      role: arn:aws-us-gov:iam::008577686731:role/StepFunctionsLambdaRole
      events:
        - schedule:
            rate: rate(2 minutes)
            input: 
              env: prod
      definition:
        StartAt: vacolsSetVars
        States:
          vacolsSetVars:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-vacolsSetVars
            Next: insertSnotes
          insertSnotes:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-insertSnotes
            Next: vacolsUnsetVars
          vacolsUnsetVars:
            Type: Task
            Resource: arn:aws-us-gov:lambda:#{AWS::Region}:#{AWS::AccountId}:function:${self:service}-${opt:stage}-vacolsUnsetVars
            End: true



plugins:
  - serverless-step-functions
  - serverless-pseudo-parameters
  - serverless-python-requirements