build.yml

name: TestNDeploy

on: [push, pull_request]

jobs:
  # Install and cache dependencies
  terraformcache:
    name: Caching Terraform dependencies
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ 3.8 ]

    steps:
    - name: Checkout Terraform Tests repo
      uses: actions/checkout@v2
      with:
        repository: localstack/localstack-terraform-test
        ref: build
        path: moto-terraform-tests
        submodules: 'true'
    - uses: actions/setup-go@v2
      with:
        go-version: '^1.16.0'
    - run: go version
    - name: cache
      id: terraformcache
      uses: actions/cache@v2
      with:
        path: '~/.cache'
        key: 'terraformcache_download2'
    - name: Download
      env:
        DOWNLOAD_TEST_BIN: 1
        TEST_BIN_URL: "https://moto-terraform-test.s3.amazonaws.com/aws.test"
      if: ${{ steps.terraformcache.outputs.cache-hit != 'true' }}
      run: |
        cd moto-terraform-tests
        bin/install-aws-test
        cd ..

  cache:
    name: Caching
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Get pip cache dir
        id: pip-cache-dir
        run: |
          echo "::set-output name=dir::$(pip cache dir)"
      - name: pip cache
        id: pip-cache
        uses: actions/cache@v2
        with:
          path: ${{ steps.pip-cache-dir.outputs.dir }}
          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
      - name: Update pip
        if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
        run: |
          python -m pip install --upgrade pip
      - name: Install project dependencies
        if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
        run: |
          pip install -r requirements-dev.txt

  lint:
    name: Linting
    runs-on: ubuntu-latest
    needs: cache
    strategy:
      matrix:
        python-version: [3.7]
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    # Retrieve the previously cached dependencies
    - name: Get pip cache dir
      id: pip-cache
      run: |
        echo "::set-output name=dir::$(pip cache dir)"
    - name: pip cache
      uses: actions/cache@v2
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
    # Update PIP
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    # Still need to properly install the dependencies - it will only skip the download part
    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
    - name: Lint with flake8
      run:
        make lint

  test:
    name: Unit test
    runs-on: ubuntu-latest
    needs: lint
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.6, 3.7, 3.8, 3.9, "3.10"]

    steps:
    - uses: actions/checkout@v2
      with:
        fetch-depth: 0
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Get pip cache dir
      id: pip-cache
      run: |
        echo "::set-output name=dir::$(pip cache dir)"
    - name: pip cache
      uses: actions/cache@v2
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
        pip install pytest-cov
        pip install pytest-github-actions-annotate-failures
        # https://github.com/aws/aws-xray-sdk-python/issues/196
        pip install "coverage<=4.5.4"
    - name: Test with pytest
      run: |
        make test-only
    - name: "Upload coverage to Codecov"
      if: ${{ github.repository == 'spulec/moto'}}
      uses: codecov/codecov-action@v1
      with:
        fail_ci_if_error: false
        flags: unittests

  testserver:
    name: Unit tests in Server Mode
    runs-on: ubuntu-latest
    needs: lint
    strategy:
      matrix:
        python-version: [3.6, 3.7, 3.8, 3.9, "3.10"]

    steps:
    - uses: actions/checkout@v2
      with:
        fetch-depth: 0
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Start MotoServer
      run: |
        python setup.py sdist
        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
        python scripts/ci_wait_for_server.py
    - name: Get pip cache dir
      id: pip-cache
      run: |
        echo "::set-output name=dir::$(pip cache dir)"
    - name: pip cache
      uses: actions/cache@v2
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
        pip install "coverage<=4.5.4"
    - name: Test ServerMode/Coverage
      env:
        TEST_SERVER_MODE: ${{ true }}
      run: |
        make test-only
    - name: "Upload coverage to Codecov"
      if: ${{ github.repository == 'spulec/moto'}}
      uses: codecov/codecov-action@v1
      with:
        fail_ci_if_error: false
        flags: servertests
    - name: "Stop MotoServer"
      if: always()
      run: |
        mkdir serverlogs
        pwd
        ls -la
        cp server_output.log serverlogs/server_output.log
        docker stop motoserver
    - name: Archive TF logs
      if: always()
      uses: actions/upload-artifact@v2
      with:
        name: motoserver-${{ matrix.python-version }}
        path: |
          serverlogs/*

  terraform:
    name: Terraform Tests
    runs-on: ubuntu-latest
    needs: terraformcache
    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.8 ]
        part: ["aa", "ab", "ac", "ad", "ae", "af", "ag"]

    steps:
    - uses: actions/checkout@v2
      with:
        fetch-depth: 0
    - name: Checkout Terraform Tests repo
      uses: actions/checkout@v2
      with:
        repository: localstack/localstack-terraform-test
        ref: build
        path: moto-terraform-tests
        submodules: 'true'
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: cache
      uses: actions/cache@v2
      with:
        path: '~/.cache'
        key: 'terraformcache_download2'
    - name: Start MotoServer
      run: |
        python setup.py sdist
        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e MOTO_PORT=4566 -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4566:4566 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
        MOTO_PORT=4566 python scripts/ci_wait_for_server.py
    # Poor man's parallelization
    # Running them sequentially takes too much time
    # And using the build in parallel-argument does not help with reducing runtime
    # So we simply split the list of tests, and ask our CI for separate VM's to run them in parallel
    - name: Get list of tests
      run: |
        cd moto-terraform-tests
        bin/list-tests -i ../tests/terraform-tests.success.txt -e ../tests/terraform-tests.failures.txt > tftestlist.txt
        split -n l/7 tftestlist.txt tf-split-
        cd ..
    - name: Run Terraform Tests
      run: |
        cd moto-terraform-tests
        echo "Copying random zipfile that is missing for SSM tests..."
        cp terraform-provider-aws/aws/test-fixtures/lambda_elb.zip terraform-provider-aws/aws/test-fixtures/ssm-doc-acc-test.zip
        AWS_DEFAULT_REGION=us-east-1 AWS_ALTERNATE_REGION=eu-west-1 bin/run-tests -t -i tf-split-${{ matrix.part }} -e ../tests/terraform-tests.failures.txt
        cd ..
    - name: "Create report"
      run: |
        ls -la
        cp server_output.log moto-terraform-tests/build/server_output.log
        cd moto-terraform-tests
        bin/create-report
        bin/create-report-cli
        cd ..
    - name: Archive TF logs
      if: always()
      uses: actions/upload-artifact@v2
      with:
        name: buildfolder-${{ matrix.part }}
        path: |
          moto-terraform-tests/build/*

  deploy:
    name: Deploy
    runs-on: ubuntu-latest
    needs: [test, testserver, terraform ]
    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'spulec/moto' }}
    strategy:
      matrix:
        python-version: [3.8]
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Update & Build
        run: |
          pip install wheel packaging
          python update_version_from_git.py
          python setup.py sdist bdist_wheel
      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@master
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}
      - name: Build Docker release
        run: |
          docker build -t motoserver/moto . --tag moto:latest
      # Required to get the correct Digest
      # See https://github.com/docker/build-push-action/issues/461
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Build and push
        uses: docker/build-push-action@v2
        with:
          push: true
          tags: motoserver/moto:latest
      - name: Get version number
        run: |
          version=$(grep -oP '(?<=__version__ = ")[0-9.a-z]+(?=")' moto/__init__.py)
          echo "moto_version=$version" >> $GITHUB_ENV
      - uses: octokit/graphql-action@v2.x
        name: Get PR info
        id: get_pr
        with:
          query: |
            query get_pr($owner:String!,$repo:String!,$commit:GitObjectID) {
              repository(owner:$owner,name:$repo) {
                object(oid:$commit) {
                  ... on Commit {
                    associatedPullRequests(last: 1){
                      edges {
                        node {
                          baseRepository {
                            nameWithOwner
                          }
                          merged
                          number
                        }
                      }
                    }
                  }
                }
              }
            }
          owner: ${{ github.event.repository.owner.name }}
          repo: ${{ github.event.repository.name }}
          commit: "${{ github.sha }}"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Get PR number
        run: |
          nr="${{ fromJSON(steps.get_pr.outputs.data).repository.object.associatedPullRequests.edges[0].node.number }}"
          repo="${{ fromJSON(steps.get_pr.outputs.data).repository.object.associatedPullRequests.edges[0].node.baseRepository.nameWithOwner }}"
          if [ -z "$nr" ]
          then
            echo "PR nr not found in $msg"
            echo "pr_found=false" >> $GITHUB_ENV
          else
            echo "PR NR: $nr"
            echo "pr_nr=$nr" >> $GITHUB_ENV
            echo "pr_repo=$repo" >> $GITHUB_ENV
            echo "pr_found=true" >> $GITHUB_ENV
          fi
      - name: Leave PR comment with Moto version
        uses: peter-evans/create-or-update-comment@v1
        if: env.pr_found == 'true' && env.pr_repo == 'spulec/moto'
        with:
          issue-number: ${{ env.pr_nr }}
          body: |
            This is now part of moto >= ${{ env.moto_version }}