Dagster Cloud ECS Template

ecs-agent-vpc.yaml

AWSTemplateFormatVersion: 2010-09-09

Description: >
  Deploys a Dagster Cloud user agent to a new or existing ECS cluster in a new VPC.
  Version DAGSTER_VERSION.
  See https://docs.dagster.cloud/agents/ecs/setup for more details.

Outputs:
  TemplateVersion:
    Description: The revision of the ECS agent template.
    Value: DAGSTER_VERSION

Parameters:
  DagsterOrganization:
    Type: String
    Description: Enter your organization name as it appears in the dagster.cloud subdomain, e.g. `hooli` corresponding with https://hooli.dagster.cloud/.
    AllowedPattern: "^[a-zA-Z0-9-_]*$"
  DagsterDeployment:
    Type: String
    Description: Enter your deployment name, e.g. `prod` corresponding with https://hooli.dagster.cloud/prod/. Leave empty to only serve branch deployments.
    AllowedPattern: "^[a-zA-Z0-9-_]*$"
  EnableBranchDeployments:
    Type: String
    AllowedValues:
      - "true"
      - "false"
    Default: false
    Description: Whether this agent should serve branch deployments. For more information, see https://docs.dagster.io/dagster-cloud/developing-testing/branch-deployments.
  AgentToken:
    Type: String
    Description: A Dagster agent token, obtained on https://{organization}.dagster.cloud/{deployment}/cloud-settings/tokens/.
    NoEcho: true

Metadata:
  TemplateVersion:
    Description: DAGSTER_VERSION
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: "Configuration"
        Parameters:
          - DagsterOrganization
          - DagsterDeployment
          - EnableBranchDeployments
      - Label:
          default: "Secrets"
        Parameters:
          - AgentToken
    ParameterLabels:
      DagsterOrganization:
        default: "Dagster Cloud Organization"
      DagsterDeployment:
        default: "Dagster Cloud Deployment"
      EnableBranchDeployments:
        default: "Enable Branch Deployments"
      AgentToken:
        default: "Agent Token"

Conditions:
  IsDeploymentEmpty: !Equals [!Ref DagsterDeployment, ""]

Resources:
  AgentLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      RetentionInDays: 7

  AgentVPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: 10.0.0.0/16
      EnableDnsSupport: 'true'
      EnableDnsHostnames: 'true'
  InternetGateway:
    Type: AWS::EC2::InternetGateway
    DependsOn: AgentVPC
  AttachGateway:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      VpcId:
        Ref: AgentVPC
      InternetGatewayId:
        Ref: InternetGateway

  PublicRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId:
        Ref: AgentVPC
      Tags:
      - Key: Name
        Value: Public
  RouteToGateway:
    Type: AWS::EC2::Route
    DependsOn: AttachGateway
    Properties:
      RouteTableId:
        Ref: PublicRouteTable
      DestinationCidrBlock: 0.0.0.0/0
      GatewayId:
        Ref: InternetGateway

## AZ-A
  AgentSubnetA:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId:
        Ref: AgentVPC
      CidrBlock: 10.0.0.0/22
      AvailabilityZone:
        Fn::Select:
        - 0
        - Fn::GetAZs: ''
      MapPublicIpOnLaunch: true
  PublicSubnetARouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId:
        Ref: AgentSubnetA
      RouteTableId:
        Ref: PublicRouteTable

## AZ-B
  AgentSubnetB:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId:
        Ref: AgentVPC
      CidrBlock: 10.0.4.0/22
      AvailabilityZone:
        Fn::Select:
        - 1
        - Fn::GetAZs: ''
      MapPublicIpOnLaunch: true
  PublicSubnetBRouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId:
        Ref: AgentSubnetB
      RouteTableId:
        Ref: PublicRouteTable

## AZ-C
  AgentSubnetC:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId:
        Ref: AgentVPC
      CidrBlock: 10.0.8.0/22
      AvailabilityZone:
        Fn::Select:
        - 2
        - Fn::GetAZs: ''
      MapPublicIpOnLaunch: true
  PublicSubnetCRouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId:
        Ref: AgentSubnetC
      RouteTableId:
        Ref: PublicRouteTable

  AgentCluster:
    Type: AWS::ECS::Cluster
    Properties:
      ClusterName: !Sub "Dagster-Cloud-${DagsterOrganization}-${DagsterDeployment}-Cluster"
      CapacityProviders:
        - FARGATE
      DefaultCapacityProviderStrategy:
        - CapacityProvider: FARGATE
          Weight: 1

  AgentTaskDefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      Cpu: "256"
      Memory: "512"
      ContainerDefinitions:
        - Name: DagsterAgent
          Image: "docker.io/dagster/dagster-cloud-agent:DAGSTER_VERSION"
          Environment:
            - Name: DAGSTER_HOME
              Value: "/opt/dagster/dagster_home"
          EntryPoint:
            - "bash"
            - "-c"
          StopTimeout: 120
          Command:
            - !Sub
              - |-
                /bin/bash -c "
                mkdir -p $DAGSTER_HOME && echo 'instance_class:
                  module: dagster_cloud
                  class: DagsterCloudAgentInstance

                dagster_cloud_api:
                  url: \"https://${DagsterOrganization}.agent.dagster.cloud\"
                  agent_token: \"${AgentToken}\"
                  ${DeploymentConfig}
                  branch_deployments: ${EnableBranchDeployments}

                user_code_launcher:
                  module: dagster_cloud.workspace.ecs
                  class: EcsUserCodeLauncher
                  config:
                    cluster: ${ConfigCluster}
                    subnets: [${ConfigSubnet}]
                    service_discovery_namespace_id: ${ServiceDiscoveryNamespace}
                    execution_role_arn: ${TaskExecutionRole.Arn}
                    task_role_arn: ${AgentRole}
                    log_group: ${AgentLogGroup}
                ' > $DAGSTER_HOME/dagster.yaml && cat $DAGSTER_HOME/dagster.yaml && dagster-cloud agent run"
              - ConfigCluster: !Ref AgentCluster
                ConfigSubnet: !Join [ ',', [!Ref AgentSubnetA, !Ref AgentSubnetB, !Ref AgentSubnetC]]
                DeploymentConfig:
                  !If [IsDeploymentEmpty, "", !Sub "deployment: ${DagsterDeployment}"]
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group:
                Ref: AgentLogGroup
              awslogs-region: !Ref AWS::Region
              awslogs-stream-prefix: "agent"
      ExecutionRoleArn:
        Ref: TaskExecutionRole
      TaskRoleArn:
        Ref: AgentRole
      NetworkMode: awsvpc
      RequiresCompatibilities:
        - FARGATE

  AgentService:
    Type: AWS::ECS::Service
    DependsOn:
      - PublicSubnetARouteTableAssociation
      - PublicSubnetBRouteTableAssociation
      - PublicSubnetCRouteTableAssociation
      - RouteToGateway
    Properties:
      Cluster: !Ref AgentCluster
      DesiredCount: 1
      LaunchType: FARGATE
      # prevents two agent tasks running simultaniously in update scenarios
      DeploymentConfiguration:
        MaximumPercent: 100
        MinimumHealthyPercent: 0
        DeploymentCircuitBreaker:
          Enable: True
          Rollback: True
      TaskDefinition:
        Ref: AgentTaskDefinition
      NetworkConfiguration:
        AwsvpcConfiguration:
          Subnets:
            - Ref: AgentSubnetA
            - Ref: AgentSubnetB
            - Ref: AgentSubnetC
          AssignPublicIp: "ENABLED"

  ServiceDiscoveryNamespace:
    Type: AWS::ServiceDiscovery::PrivateDnsNamespace
    Properties:
      Vpc:
        Ref: AgentVPC
      Name: !Sub
        - "dagster-agent-${DagsterOrganization}-${DagsterDeployment}-${UUID}.local"
        - UUID:
            !Select [
              4,
              !Split ["-", !Select [2, !Split ["/", !Ref AWS::StackId]]],
            ]
      Properties:
        DnsProperties:
          SOA:
            TTL: 100

  TaskExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - "ecs-tasks.amazonaws.com"
            Action:
              - sts:AssumeRole
            Condition:
              ArnLike:
                "aws:SourceArn": !Sub "arn:aws:ecs:${AWS::Region}:${AWS::AccountId}:*"
              StringEquals:
                "aws:SourceAccount": !Ref AWS::AccountId
      Path: "/"
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"

  AgentRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - "ecs-tasks.amazonaws.com"
            Action:
              - sts:AssumeRole
            Condition:
              ArnLike:
                "aws:SourceArn": !Sub "arn:aws:ecs:${AWS::Region}:${AWS::AccountId}:*"
              StringEquals:
                "aws:SourceAccount": !Ref AWS::AccountId
      Path: "/"
      Policies:
        - PolicyName: root
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - ecs:ListTagsForResource
                  - secretsmanager:DescribeSecret
                  - secretsmanager:GetSecretValue
                  - secretsmanager:ListSecrets
                Resource: "*"
              - Effect: Allow
                Action:
                  - ec2:DescribeRouteTables
                  - ec2:DescribeNetworkInterfaces
                  - ecs:ListAccountSettings
                Resource: "*"
              - Effect: Allow
                Action:
                  - ecs:CreateService
                  - ecs:DeleteService
                  - ecs:DescribeServices
                  - ecs:DescribeTasks
                  - ecs:ListServices
                  - ecs:ListTasks
                  - ecs:RunTask
                  - ecs:StopTask
                  - ecs:UpdateService
                Resource: "*"
                Condition:
                  ArnLike:
                    "ecs:cluster": !GetAtt AgentCluster.Arn
              - Effect: Allow
                Action:
                  - ecs:TagResource
                Resource: !Sub "${AgentCluster.Arn}*"
              # No way to scope down these further :(
              - Effect: Allow
                Action:
                  - ecs:DescribeTaskDefinition
                  - ecs:RegisterTaskDefinition
                  - ecs:TagResource
                Resource: "*"
              - Effect: Allow
                Action:
                  - iam:PassRole
                Resource: "*"
              - Effect: "Allow"
                Action:
                  - logs:GetLogEvents
                Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:${AgentLogGroup}:log-stream:*"
              - Effect: Allow
                Action:
                  - servicediscovery:ListServices
                  - servicediscovery:ListTagsForResource
                  - servicediscovery:ListInstances
                  - servicediscovery:DeregisterInstance
                  - servicediscovery:GetOperation
                  - servicediscovery:DeleteService
                Resource: "*"
              - Effect: Allow
                Action:
                  - servicediscovery:CreateService
                  - servicediscovery:GetNamespace
                  - servicediscovery:TagResource
                Resource: !GetAtt ServiceDiscoveryNamespace.Arn