cloud-tests.yaml

name: Cloud tests
on:
  schedule:
    # Every Sunday at 11PM
    - cron: '0 23 * * 0'
  workflow_dispatch:
    inputs:
      cnspecImageTag:
        required: true
        type: string
        default: edge-latest-rootless
        description: The image tag to use for the cnspec image
      mondooOperatorImageTag:
        required: true
        type: string
        default: main
        description: The image tag to use for the mondoo operator image
    secrets:
      MONDOO_CLIENT:
        required: true
      AZURE_CLIENT_ID:
        required: true
      AZURE_CLIENT_SECRET:
        required: true
      AZURE_SUBSCRIPTION_ID:
        required: true
      AZURE_TENANT_ID:
        required: true
      AWS_ACCESS_KEY_ID:
        required: true
      AWS_SECRET_ACCESS_KEY:
        required: true
      GCP_SERVICE_ACCOUNT:
        required: true

env:
  MONDOO_OPERATOR_IMAGE_TAG: ${{ github.event.inputs.mondooOperatorImageTag || 'main' }}
  CNSPEC_IMAGE_TAG: ${{ github.event.inputs.cnspecImageTag || 'edge-latest-rootless' }}

jobs:
  aks-integration-test:
    runs-on: ubuntu-latest
    name: AKS integration tests

    env:
      ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
      ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/aks/kubeconfig') }}
    
    strategy:
      fail-fast: false
      matrix:
        k8s-version: ["1.25", "1.26", "1.27"]

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile

      - name: Import environment variables from file
        run: cat ".github/env" >> $GITHUB_ENV

      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v3

      - name: Terraform init
        run: terraform init
        working-directory: .github/terraform/aks
      
      - name: Terraform plan
        run: terraform plan -out aks-${{ matrix.k8s-version }}.json
        env:
          TF_VAR_k8s_version: ${{ matrix.k8s-version }}
        working-directory: .github/terraform/aks

      - name: Terraform apply
        run: terraform apply -auto-approve aks-${{ matrix.k8s-version }}.json
        env:
          TF_VAR_k8s_version: ${{ matrix.k8s-version }}
        working-directory: .github/terraform/aks

      - uses: actions/setup-go@v5
        with:
          go-version: "${{ env.golang-version }}"
          cache: true

      - name: Get operator version
        run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

      - name: Wait a bit for the cluster to become more stable
        run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s

      - name: Run integration tests
        env:
          MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }}
          MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342
          MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query
        run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=aks make test/integration/ci

      - name: Clean up AKS terraform
        run: terraform destroy -auto-approve
        if: success() || failure()
        working-directory: .github/terraform/aks
      
      - run: mv integration-tests.xml integration-tests-aks-${{ matrix.k8s-version }}.xml
        if: success() || failure()

      - name: Upload cloud test results
        uses: actions/upload-artifact@v3  # upload test results
        if: success() || failure()        # run this step even if previous step failed
        with:                             # upload a combined archive with unit and integration test results
          name: cloud-test-results
          path: |
            integration-tests-aks-${{ matrix.k8s-version }}.xml
            .github/terraform/aks/aks-${{ matrix.k8s-version }}.json

      - name: Upload test logs artifact
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: test-logs-aks-${{ matrix.k8s-version }}
          path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/

  eks-integration-test:
    runs-on: ubuntu-latest
    name: EKS integration tests
    
    strategy:
      fail-fast: false
      max-parallel: 3 # Otherwise we will hit AWS VPC quota limits
      matrix:
        k8s-version: ["1.24", "1.25", "1.26", "1.27", "1.28"]

    env:
      TF_VAR_test_name: ${{ github.event.inputs.mondooOperatorImageTag }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: us-east-2

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile

      - name: Import environment variables from file
        run: cat ".github/env" >> $GITHUB_ENV

      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v3

      - run: terraform init
        working-directory: .github/terraform/aws

      - name: Plan EKS
        run: terraform plan -out eks-${{ matrix.k8s-version }}.json
        env:
          TF_VAR_kubernetes_version: ${{ matrix.k8s-version }}
        working-directory: .github/terraform/aws
      
      - name: Apply EKS
        run: terraform apply -auto-approve eks-${{ matrix.k8s-version }}.json
        env:
          TF_VAR_kubernetes_version: ${{ matrix.k8s-version }}
        working-directory: .github/terraform/aws

      - uses: actions/setup-go@v5
        with:
          go-version: "${{ env.golang-version }}"
          cache: true

      - name: Get operator version
        run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

      - name: Wait a bit for the cluster to become more stable
        run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s

      - name: Run integration tests
        env:
          MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }}
          MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342
          MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query
        run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=eks make test/integration/ci

      - name: Clean up EKS terraform
        run: terraform destroy -auto-approve
        working-directory: .github/terraform/aws
        if: success() || failure()

      - run: mv integration-tests.xml integration-tests-eks-${{ matrix.k8s-version }}.xml
        if: success() || failure()

      - name: Upload test results
        uses: actions/upload-artifact@v3  # upload test results
        if: success() || failure()        # run this step even if previous step failed
        with:                             # upload a combined archive with unit and integration test results
          name: cloud-test-results
          path: integration-tests-eks-${{ matrix.k8s-version }}.xml

      - name: Upload test logs artifact
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: test-logs-eks-${{ matrix.k8s-version }}
          path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/

  gke-integration-test:
    runs-on: ubuntu-latest
    name: GKE integration tests
    
    strategy:
      fail-fast: false
      matrix:
        k8s-version: ["1.25", "1.26", "1.27"]

    env:
      GOOGLE_APPLICATION_CREDENTIALS: ${{ format('{0}/{1}', github.workspace, 'gcp_sa.json') }}
      KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/gke/kubeconfig') }}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile

      - name: Import environment variables from file
        run: cat ".github/env" >> $GITHUB_ENV

      - name: Setup GCP service account
        run: echo ${{ secrets.GCP_SERVICE_ACCOUNT }} | base64 -d > gcp_sa.json

      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v3

      - run: terraform init
        working-directory: .github/terraform/gke

      - name: Plan GKE
        run: terraform plan -out gke-${{ matrix.k8s-version }}.json
        env:
          TF_VAR_k8s_version: ${{ matrix.k8s-version }}
        working-directory: .github/terraform/gke
      
      - name: Apply GKE
        run: terraform apply -auto-approve gke-${{ matrix.k8s-version }}.json
        env:
          TF_VAR_k8s_version: ${{ matrix.k8s-version }}
        working-directory: .github/terraform/gke

      - uses: actions/setup-go@v5
        with:
          go-version: "${{ env.golang-version }}"
          cache: true

      - name: Get operator version
        run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

      - name: Wait a bit for the cluster to become more stable
        run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=300s

      - name: Run integration tests
        env:
          MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }}
          MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342
          MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query
        run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=gke make test/integration/ci

      - name: Clean up GKE terraform
        run: terraform destroy -auto-approve
        working-directory: .github/terraform/gke
        if: success() || failure()

      - name: Cleanup GCP service account
        run: rm gcp_sa.json
        if: success() || failure()

      - run: mv integration-tests.xml integration-tests-gke-${{ matrix.k8s-version }}.xml
        if: success() || failure()

      - name: Upload test results
        uses: actions/upload-artifact@v3  # upload test results
        if: success() || failure()        # run this step even if previous step failed
        with:                             # upload a combined archive with unit and integration test results
          name: cloud-test-results
          path: integration-tests-gke-${{ matrix.k8s-version }}.xml

      - name: Upload test logs artifact
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: test-logs-gke-${{ matrix.k8s-version }}
          path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/

  test-report:
    name: Report test results
    runs-on: ubuntu-latest
    needs: [eks-integration-test,aks-integration-test,gke-integration-test]
    if: always()
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile

    - name: Download test results
      uses: actions/download-artifact@v3
      with:
        name: cloud-test-results

    - name: Parse test results
      uses: dorny/test-reporter@v1
      with:
        name: Report cloud test results
        path: '*.xml'                     # Path to test results
        reporter: java-junit              # Format of test results

  discord-notification:
    runs-on: ubuntu-latest
    name: Send Discord notification
    needs: [eks-integration-test,aks-integration-test,gke-integration-test]
    # Run only if the previous job has failed and only if it's running against the main branch
    if: ${{ always() && contains(join(needs.*.result, ','), 'fail') && github.ref_name == 'main' }}
    steps:
      - uses: sarisia/actions-status-discord@v1
        with:
          webhook: ${{ secrets.DISCORD_WEBHOOK }}
          status: Failure
          url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          description: Workflow ${{ github.workflow }} failed for commit ${{ github.sha }}.
          color: 0xff4d4d