airflow_cft.yml

AWSTemplateFormatVersion: 2010-09-09

Description:  This template creates an IAM role and use this role to create an Amazon MWAA environment. For the VPC network setup, it deploys a VPC, with a pair of public and private subnets spread across two Availability Zones. It deploys an internet gateway, with a default route on the public subnets. It deploys a pair of NAT gateways (one in each AZ), and default routes for them in the private subnets. It also creates S3 gateway VPC endpoints and EMR interface VPC endpoints in the private subnets in two AZs. 


Parameters:
  AirflowBucketName:
    Description: Name of the S3 bucket to store DAGs and support files. The S3 bucket must be in the same Region where you create the environment. The name must start with 'airflow-'.
    Type: String
  EnvironmentName:
    Description: An environment name that is prefixed to resource names. All the resources created by this templated will be named after the value saved for this field.
    Type: String
    Default: mwaa-

  VpcCIDR:
    Description: Please enter the IP range (CIDR notation) for this VPC
    Type: String
    Default: 10.192.0.0/16

  PublicSubnet1CIDR:
    Description: Please enter the IP range (CIDR notation) for the public subnet in the first Availability Zone
    Type: String
    Default: 10.192.10.0/24

  PublicSubnet2CIDR:
    Description: Please enter the IP range (CIDR notation) for the public subnet in the second Availability Zone
    Type: String
    Default: 10.192.11.0/24

  PrivateSubnet1CIDR:
    Description: Please enter the IP range (CIDR notation) for the private subnet in the first Availability Zone
    Type: String
    Default: 10.192.20.0/24

  PrivateSubnet2CIDR:
    Description: Please enter the IP range (CIDR notation) for the private subnet in the second Availability Zone
    Type: String
    Default: 10.192.21.0/24

Resources:
  VPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: !Ref VpcCIDR
      EnableDnsSupport: true
      EnableDnsHostnames: true
      Tags:
        - Key: Name
          Value: !Ref EnvironmentName

  InternetGateway:
    Type: AWS::EC2::InternetGateway
    Properties:
      Tags:
        - Key: Name
          Value: !Ref EnvironmentName

  InternetGatewayAttachment:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      InternetGatewayId: !Ref InternetGateway
      VpcId: !Ref VPC

  PublicSubnet1:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref VPC
      AvailabilityZone: !Select [ 0, !GetAZs '' ]
      CidrBlock: !Ref PublicSubnet1CIDR
      MapPublicIpOnLaunch: true
      Tags:
        - Key: Name
          Value: !Sub ${EnvironmentName} Public Subnet (AZ1)

  PublicSubnet2:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref VPC
      AvailabilityZone: !Select [ 1, !GetAZs  '' ]
      CidrBlock: !Ref PublicSubnet2CIDR
      MapPublicIpOnLaunch: true
      Tags:
        - Key: Name
          Value: !Sub ${EnvironmentName} Public Subnet (AZ2)

  PrivateSubnet1:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref VPC
      AvailabilityZone: !Select [ 0, !GetAZs  '' ]
      CidrBlock: !Ref PrivateSubnet1CIDR
      MapPublicIpOnLaunch: false
      Tags:
        - Key: Name
          Value: !Sub ${EnvironmentName} Private Subnet (AZ1)

  PrivateSubnet2:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref VPC
      AvailabilityZone: !Select [ 1, !GetAZs  '' ]
      CidrBlock: !Ref PrivateSubnet2CIDR
      MapPublicIpOnLaunch: false
      Tags:
        - Key: Name
          Value: !Sub ${EnvironmentName} Private Subnet (AZ2)

  NatGateway1EIP:
    Type: AWS::EC2::EIP
    DependsOn: InternetGatewayAttachment
    Properties:
      Domain: vpc

  NatGateway2EIP:
    Type: AWS::EC2::EIP
    DependsOn: InternetGatewayAttachment
    Properties:
      Domain: vpc

  NatGateway1:
    Type: AWS::EC2::NatGateway
    Properties:
      AllocationId: !GetAtt NatGateway1EIP.AllocationId
      SubnetId: !Ref PublicSubnet1

  NatGateway2:
    Type: AWS::EC2::NatGateway
    Properties:
      AllocationId: !GetAtt NatGateway2EIP.AllocationId
      SubnetId: !Ref PublicSubnet2

  PublicRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub ${EnvironmentName} Public Routes

  DefaultPublicRoute:
    Type: AWS::EC2::Route
    DependsOn: InternetGatewayAttachment
    Properties:
      RouteTableId: !Ref PublicRouteTable
      DestinationCidrBlock: 0.0.0.0/0
      GatewayId: !Ref InternetGateway

  PublicSubnet1RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref PublicRouteTable
      SubnetId: !Ref PublicSubnet1

  PublicSubnet2RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref PublicRouteTable
      SubnetId: !Ref PublicSubnet2


  PrivateRouteTable1:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub ${EnvironmentName} Private Routes (AZ1)

  DefaultPrivateRoute1:
    Type: AWS::EC2::Route
    Properties:
      RouteTableId: !Ref PrivateRouteTable1
      DestinationCidrBlock: 0.0.0.0/0
      NatGatewayId: !Ref NatGateway1

  PrivateSubnet1RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref PrivateRouteTable1
      SubnetId: !Ref PrivateSubnet1

  PrivateRouteTable2:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub ${EnvironmentName} Private Routes (AZ2)

  DefaultPrivateRoute2:
    Type: AWS::EC2::Route
    Properties:
      RouteTableId: !Ref PrivateRouteTable2
      DestinationCidrBlock: 0.0.0.0/0
      NatGatewayId: !Ref NatGateway2

  PrivateSubnet2RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref PrivateRouteTable2
      SubnetId: !Ref PrivateSubnet2

  NoIngressSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupName: "no-ingress-sg"
      GroupDescription: "Security group with no ingress rule"
      VpcId: !Ref VPC

  NoIngressSecurityGroupIngresRule:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      SourceSecurityGroupId: {Ref: NoIngressSecurityGroup}
      GroupId: {Ref: NoIngressSecurityGroup}
      IpProtocol: -1
      ToPort: -1
      FromPort: -1

  S3VPCEndpoint:
    Type: "AWS::EC2::VPCEndpoint"
    Properties:
      RouteTableIds:
      - !Ref PrivateRouteTable1
      - !Ref PrivateRouteTable2
      ServiceName: !Sub 'com.amazonaws.${AWS::Region}.s3'
      VpcId: !Ref VPC

  EmrEndpoint:
    Type: "AWS::EC2::VPCEndpoint"
    Properties:
      PrivateDnsEnabled: "true"
      VpcEndpointType: "Interface"
      VpcId: !Ref VPC
      SecurityGroupIds:
      - !Ref NoIngressSecurityGroup
      ServiceName: !Sub 'com.amazonaws.${AWS::Region}.elasticmapreduce'
      SubnetIds:
      - !Ref PrivateSubnet1
      - !Ref PrivateSubnet2

  MwaaExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - airflow-env.amazonaws.com
            - airflow.amazonaws.com
          Action:
          - sts:AssumeRole

  # Refer to Doc: https://docs.aws.amazon.com/mwaa/latest/userguide/mwaa-create-role.html#role-aws-key
  MwaaExecutionRoleDefaultPolicy:
    Type: "AWS::IAM::ManagedPolicy"
    Properties:
      Roles:
      - !Ref "MwaaExecutionRole"
      ManagedPolicyName: MwaaExecutionRoleDefaultPolicy
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Action:
          - "airflow:PublishMetrics"
          Resource:
          - !Sub arn:aws:airflow:${AWS::Region}:${AWS::AccountId}:environment/*
        - Effect: Deny
          Action:
          - "s3:ListAllMyBuckets"
          Resource:
          - !Sub arn:aws:s3:::${AirflowBucketName}
          - !Sub arn:aws:s3:::${AirflowBucketName}/*
        - Effect: Allow
          Action:
          - "s3:GetObject*"
          - "s3:GetBucket*"
          - "s3:List*"
          Resource:
          - !Sub arn:aws:s3:::${AirflowBucketName}
          - !Sub arn:aws:s3:::${AirflowBucketName}/*
        - Effect: Allow
          Action:
          - "logs:CreateLogStream"
          - "logs:CreateLogGroup"
          - "logs:PutLogEvents"
          - "logs:GetLogEvents"
          - "logs:GetLogRecord"
          - "logs:GetLogGroupFields"
          - "logs:GetQueryResults"
          - "logs:DescribeLogGroups"
          Resource:
          - !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:airflow-*
        - Effect: Allow
          Action: "cloudwatch:PutMetricData"
          Resource: "*"
        - Effect: Allow
          Action:
          - "sqs:ChangeMessageVisibility"
          - "sqs:DeleteMessage"
          - "sqs:GetQueueAttributes"
          - "sqs:GetQueueUrl"
          - "sqs:ReceiveMessage"
          - "sqs:SendMessage"
          Resource:
          - !Sub arn:aws:sqs:${AWS::Region}:*:airflow-celery-*
        - Effect: Allow
          Action:
          - "kms:Decrypt"
          - "kms:DescribeKey"
          - "kms:GenerateDataKey*"
          - "kms:Encrypt"
          NotResource:
          - !Sub arn:aws:kms:*:${AWS::AccountId}:key/*
          Condition:
            StringLike:
              kms:ViaService:
              - !Sub sqs.${AWS::Region}.amazonaws.com


  EmrPermissionsForMwaaExecutionRole:
    Type: "AWS::IAM::ManagedPolicy"
    Properties:
      Roles:
      - !Ref "MwaaExecutionRole"
      ManagedPolicyName: EmrPermissionsForMwaaExecutionRole
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Action:
          - "elasticmapreduce:StartNotebookExecution"
          - "elasticmapreduce:DescribeStep"
          - "elasticmapreduce:DescribeNotebookExecution"
          - "elasticmapreduce:AddJobFlowSteps"
          - "elasticmapreduce:DescribeCluster"
          - "elasticmapreduce:RunJobFlow"
          - "elasticmapreduce:TerminateJobFlows"
          Resource:
          - !Sub "arn:aws:elasticmapreduce:${AWS::Region}:${AWS::AccountId}:cluster/*"
          - !Sub "arn:aws:elasticmapreduce:${AWS::Region}:${AWS::AccountId}:editor/*"
          - !Sub "arn:aws:elasticmapreduce:${AWS::Region}:${AWS::AccountId}:notebook-execution/*"
        - Effect: Allow
          Action:
          - "iam:PassRole"
          Resource:
          - !Sub arn:aws:iam::${AWS::AccountId}:role/EMR_EC2_DefaultRole
          - !Sub arn:aws:iam::${AWS::AccountId}:role/EMR_DefaultRole
          - !Sub arn:aws:iam::${AWS::AccountId}:role/EMR_Notebooks_DefaultRole

  MwaaEnvironment:
    Type: AWS::MWAA::Environment
    Properties:
      Name: !Ref EnvironmentName
      NetworkConfiguration:
        SubnetIds:
        - !Ref PrivateSubnet1
        - !Ref PrivateSubnet2
        SecurityGroupIds: 
        - !Ref NoIngressSecurityGroup
      WebserverAccessMode: PUBLIC_ONLY
      LoggingConfiguration:
        DagProcessingLogs:
          Enabled: false
          LogLevel: INFO
        SchedulerLogs:
          Enabled: false
          LogLevel: INFO
        TaskLogs:
          Enabled: true
          LogLevel: INFO
        WebserverLogs:
          Enabled: false
          LogLevel: INFO
        WorkerLogs:
          Enabled: false
          LogLevel: INFO
      SourceBucketArn: !Sub arn:aws:s3:::${AirflowBucketName}
      ExecutionRoleArn: !GetAtt MwaaExecutionRole.Arn
      MaxWorkers: 1
      DagS3Path: "dags"
      EnvironmentClass: mw1.small

Outputs:
  VPC:
    Description: A reference to the created VPC
    Value: !Ref VPC

  PublicSubnets:
    Description: A list of the public subnets
    Value: !Join [ ",", [ !Ref PublicSubnet1, !Ref PublicSubnet2 ]]

  PrivateSubnets:
    Description: A list of the private subnets
    Value: !Join [ ",", [ !Ref PrivateSubnet1, !Ref PrivateSubnet2 ]]

  PublicSubnet1:
    Description: A reference to the public subnet in the 1st Availability Zone
    Value: !Ref PublicSubnet1

  PublicSubnet2:
    Description: A reference to the public subnet in the 2nd Availability Zone
    Value: !Ref PublicSubnet2

  PrivateSubnet1:
    Description: A reference to the private subnet in the 1st Availability Zone
    Value: !Ref PrivateSubnet1

  PrivateSubnet2:
    Description: A reference to the private subnet in the 2nd Availability Zone
    Value: !Ref PrivateSubnet2

  NoIngressSecurityGroup:
    Description: Security group with no ingress rule
    Value: !Ref NoIngressSecurityGroup