serverless.yml

service: deep-util-services
frameworkVersion: "=3"

provider:
  name: aws
  runtime: python3.8
  stage: ${opt:stage, "local"}
  region: us-east-1
  versionFunctions: false
  memorySize: 512 # in MB
  timeout: 30 # in seconds
  environment:
    SERVICE: ${self:service}
    STAGE: ${self:provider.stage}
    SOURCE_TABLE_NAME: ${self:custom.sourceTableName}
    ASYNC_JOB_TABLE_NAME: ${self:custom.asyncJobTableName}
    JWT_SECRET_OR_PUBLIC_KEY: ${file(./secrets/${self:provider.stage}.json):JWT_SECRET_OR_PUBLIC_KEY}
    MEDIA_BUCKET_NAME: ${file(./secrets/${self:provider.stage}.json):MEDIA_BUCKET_NAME}
    MEDIA_BUCKET_ROOT: ${file(./secrets/${self:provider.stage}.json):MEDIA_BUCKET_ROOT}
    SENTRY_DNS: ${file(./secrets/${self:provider.stage}.json):SENTRY_DNS}
    NLTK_DATA: /opt/nltk_data
  iamRoleStatements:
    - Effect: Allow
      Action:
        - s3:ListBucket
        - s3:HeadObject
        - s3:GetObject
        - s3:PutObject
      Resource:
        - "arn:aws:s3:::${self:provider.environment.MEDIA_BUCKET_NAME}/*"
    - Effect: Allow
      Action:
        - dynamodb:Query
        - dynamodb:Scan
        - dynamodb:GetItem
        - dynamodb:BatchGetItem
        - dynamodb:BatchWriteItem
        - dynamodb:PutItem
        - dynamodb:UpdateItem
        - dynamodb:DeleteItem
        - dynamodb:DescribeTable
      Resource:
        - "arn:aws:dynamodb:${opt:region, self:provider.region}:*:table/${self:provider.environment.SOURCE_TABLE_NAME}"
        - "arn:aws:dynamodb:${opt:region, self:provider.region}:*:table/${self:provider.environment.ASYNC_JOB_TABLE_NAME}"
    - Effect: Allow
      Action:
        # - lambda:InvokeFunction
        # - lambda:InvokeAsync
        - states:StartExecution
      # TODO: NOT WORKING Resource: ${self:resources.Outputs.SourceExtractStateMachine.Value}
      Resource: "*"

custom:
  stepFunctionConfig:
    beta:
      MaxConcurrency: 50
    local:
      MaxConcurrency: 50
    MaxConcurrency: ${self:custom.stepFunctionConfig.${self:provider.stage}.MaxConcurrency, 1}
  dynamodbConfig:
    beta:
      BillingMode: PAY_PER_REQUEST
    dev:
      ProvisionedThroughput:
        ReadCapacityUnits: 1
        WriteCapacityUnits: 1
    nightly:
      ProvisionedThroughput:
        ReadCapacityUnits: 1
        WriteCapacityUnits: 1
    local:
      BillingMode: PAY_PER_REQUEST
      # ProvisionedThroughput:
      #   ReadCapacityUnits: 1
      #   WriteCapacityUnits: 1
    BillingMode: ${self:custom.dynamodbConfig.${self:provider.stage}.BillingMode, "PROVISIONED"}
    ProvisionedThroughput: ${self:custom.dynamodbConfig.${self:provider.stage}.ProvisionedThroughput, null}
  dynamodb:
    stages:
      - ${self:provider.stage}
  sourceTableName: "${self:service}-${self:provider.stage}-sources"
  asyncJobTableName: "${self:service}-${self:provider.stage}-async-jobs"
  authorizer: authorizerFunc
  cors:
    origins: ${file(./secrets/${self:provider.stage}.json):CORS_DOMAINS}
  pythonRequirements:
    dockerizePip: true
    noDeploy:
      # Dependencies of omitted packages must explicitly be omitted too
      - pytest
      - parameterized
  domains:
    beta: services.thedeep.io
    dev: services-alpha.thedeep.io
    nightly: services-nightly.thedeep.io
    local: services-local.thedeep.io
  customDomain:
    domainName: ${self:custom.domains.${self:provider.stage}}
    stage: ${self:provider.stage}
    createRoute53Record: false
    certificateArn: ${file(./secrets/${self:provider.stage}.json):CERTIFICATE_ARN}

package:
  exclude:
    - .git/**
    - .gitignore
    - .DS_Store
    - npm-debug.log
    - .serverless/**
    - .serverless_plugins/**
    - .python-venv/**
    - .pytest_cache/**

functions:
  webInfoExtract:
    handler: src.functions.source_extract.handler.web_info_extract
    # TODO: Seperate requirements (only handle common modules like utils as vendor)
    # module: src/functions/source_extract
    events:
      - http:
          path: web-info-extract
          method: get
          authorizer: ${self:custom.authorizer}
          cors: ${self:custom.cors}
  sourceExtract:
    handler: src.functions.source_extract.handler.main
    environment:
      SOURCE_EXTRACT_STATE_MACHINE_ARN: ${self:resources.Outputs.SourceExtractStateMachine.Value}
    events:
      - http:
          path: source-extract
          method: post
          authorizer: ${self:custom.authorizer}
          cors: ${self:custom.cors}
  handleEntries:
    handler: src.ngrams.handler.handle_entries
    events:
      - http:
          path: ngram-process
          method: post
          authorizer: ${self:custom.authorizer}
          cors: ${self:custom.cors}
    layers:
      - {Ref: NltkLambdaLayer}

  sourceExtractJobSuccess:
    handler: src.functions.source_extract.handler.source_extract_job_success
  sourceExtractJobFailure:
    handler: src.functions.source_extract.handler.source_extract_job_failure
  sourceExtractJobMappedTask:
    handler: src.functions.source_extract.handler.source_extract_job_mapped_task
    timeout: 120

  # TODO: Deploy without python dependencies
  authorizerFunc:
    runtime: nodejs12.x
    handler: src/authorizer/handler.auth

stepFunctions:
  stateMachines:
    SourceExtractStateMachine:
      name: "sourceExtractStateMachine-${self:provider.stage}"
      definition:
        StartAt: SourceExtractJobMappedTask
        States:
          # TODO: Handle errors
          SourceExtractJobMappedTask:
            Type: Map
            MaxConcurrency: ${self:custom.stepFunctionConfig.MaxConcurrency}
            ItemsPath: $.sources
            ResultPath: $.sources
            Catch:
              - ErrorEquals:
                - States.ALL
                ResultPath: $.error
                Next: SourceExtractJobFailure
            Next: SourceExtractJobSuccess
            Iterator:
              StartAt: SourceExtractJobMapTask
              States:
                SourceExtractJobMapTask:
                  Type: Task
                  Resource:
                    Fn::GetAtt: [sourceExtractJobMappedTask, Arn]
                  ResultPath: $.status
                  End: true
          SourceExtractJobSuccess:
            Type: Task
            Resource:
              Fn::GetAtt: [sourceExtractJobSuccess, Arn]
              End: true
            End: true
          SourceExtractJobFailure:
            Type: Task
            Resource:
              Fn::GetAtt: [sourceExtractJobFailure, Arn]
              End: true
            End: true

resources:
  Resources:
    AsyncJobTable:
      Type: "AWS::DynamoDB::Table"
      Properties:
        # Make sure to update PynamoDB models
        AttributeDefinitions:
          - AttributeName: uuid
            AttributeType: S
        KeySchema:
          - AttributeName: uuid
            KeyType: HASH
        BillingMode: ${self:custom.dynamodbConfig.BillingMode}
        ProvisionedThroughput: ${self:custom.dynamodbConfig.ProvisionedThroughput, null}
        TableName: ${self:custom.asyncJobTableName}
        TimeToLiveSpecification:
          AttributeName: ttl
          Enabled: true
    SourceTable:
      Type: "AWS::DynamoDB::Table"
      Properties:
        # Make sure to update PynamoDB models
        AttributeDefinitions:
          - AttributeName: key
            AttributeType: S
        KeySchema:
          - AttributeName: key
            KeyType: HASH
        BillingMode: ${self:custom.dynamodbConfig.BillingMode}
        ProvisionedThroughput: ${self:custom.dynamodbConfig.ProvisionedThroughput, null}
        TableName: ${self:custom.sourceTableName}
    GatewayResponseDefault4XX:
      Type: "AWS::ApiGateway::GatewayResponse"
      Properties:
        ResponseParameters:
          gatewayresponse.header.Access-Control-Allow-Origin: "'*'"
          gatewayresponse.header.Access-Control-Allow-Headers: "'*'"
        ResponseType: DEFAULT_4XX
        ResponseTemplates:
          application/json: ${file(./src/authorizer/error-template.txt)}
        RestApiId:
          Ref: "ApiGatewayRestApi"
    GatewayResponseDefault5XX:
      Type: "AWS::ApiGateway::GatewayResponse"
      Properties:
        ResponseParameters:
          gatewayresponse.header.Access-Control-Allow-Origin: "'*'"
          gatewayresponse.header.Access-Control-Allow-Headers: "'*'"
        ResponseType: DEFAULT_5XX
        ResponseTemplates:
          application/json: ${file(./src/authorizer/error-template.txt)}
        RestApiId:
          Ref: "ApiGatewayRestApi"
  Outputs:
    SourceExtractStateMachine:
      Description: The ARN of the source extraction state machine
      Value:
        Ref: "SourceExtractStateMachineDash${self:provider.stage}"

layers:
  nltk:
    path: src/ngrams/nltk
    name: NltkData
    description: Nltk data
    compatibleRuntimes:
      - python3.8

plugins:
  - serverless-domain-manager
  - serverless-python-requirements
  - serverless-plugin-include-dependencies
  - serverless-step-functions
  - serverless-dynamodb-local
  - serverless-offline  # always last