cloud-tests.yaml

name: Cloud tests
on:
  schedule:
    # Every Sunday at 11PM
    - cron: '0 23 * * 0'
  workflow_dispatch:
    inputs:
      cnspecImageTag:
        required: true
        type: string
        default: edge-latest-rootless
        description: The image tag to use for the cnspec image
      mondooOperatorImageTag:
        required: true
        type: string
        default: main
        description: The image tag to use for the mondoo operator image
    secrets:
      MONDOO_CLIENT:
        required: true
      AZURE_CLIENT_ID:
        required: true
      AZURE_CLIENT_SECRET:
        required: true
      AZURE_SUBSCRIPTION_ID:
        required: true
      AZURE_TENANT_ID:
        required: true
      AWS_ACCESS_KEY_ID:
        required: true
      AWS_SECRET_ACCESS_KEY:
        required: true
      GCP_SERVICE_ACCOUNT:
        required: true

env:
  MONDOO_OPERATOR_IMAGE_TAG: ${{ github.event.inputs.mondooOperatorImageTag || 'main' }}
  CNSPEC_IMAGE_TAG: ${{ github.event.inputs.cnspecImageTag || 'edge-latest-rootless' }}

jobs:
  # aks-integration-test:
  #   runs-on: ubuntu-latest
  #   name: AKS integration tests

  #   env:
  #     ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
  #     ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
  #     ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
  #     ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
  #     KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/aks/kubeconfig') }}
    
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       k8s-version: ["1.25", "1.26", "1.27"]

  #   steps:
  #     - uses: GitHubSecurityLab/actions-permissions/monitor@v1
  #       with:
  #         config: ${{ vars.PERMISSIONS_CONFIG }}
  #     - uses: actions/checkout@v4
  #       with:
  #         fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile

  #     - name: Import environment variables from file
  #       run: cat ".github/env" >> $GITHUB_ENV

  #     - name: Setup Terraform
  #       uses: hashicorp/setup-terraform@v2

  #     - name: Terraform init
  #       run: terraform init
  #       working-directory: .github/terraform/aks
      
  #     - name: Terraform plan
  #       run: terraform plan -out aks-${{ matrix.k8s-version }}.json
  #       env:
  #         TF_VAR_k8s_version: ${{ matrix.k8s-version }}
  #       working-directory: .github/terraform/aks

  #     - name: Terraform apply
  #       run: terraform apply -auto-approve aks-${{ matrix.k8s-version }}.json
  #       env:
  #         TF_VAR_k8s_version: ${{ matrix.k8s-version }}
  #       working-directory: .github/terraform/aks

  #     - uses: actions/setup-go@v4
  #       with:
  #         go-version: "${{ env.golang-version }}"
  #         cache: true

  #     - name: Get operator version
  #       run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

  #     - name: Wait a bit for the cluster to become more stable
  #       run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s

  #     - name: Run integration tests
  #       env:
  #         MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }}
  #         MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342
  #         MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query
  #       run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=aks make test/integration/ci

  #     - name: Clean up AKS terraform
  #       run: terraform destroy -auto-approve
  #       if: success() || failure()
  #       working-directory: .github/terraform/aks
      
  #     - run: mv integration-tests.xml integration-tests-aks-${{ matrix.k8s-version }}.xml
  #       if: success() || failure()

  #     - name: Upload cloud test results
  #       uses: actions/upload-artifact@v3  # upload test results
  #       if: success() || failure()        # run this step even if previous step failed
  #       with:                             # upload a combined archive with unit and integration test results
  #         name: cloud-test-results
  #         path: |
  #           integration-tests-aks-${{ matrix.k8s-version }}.xml
  #           .github/terraform/aks/aks-${{ matrix.k8s-version }}.json

  #     - name: Upload test logs artifact
  #       uses: actions/upload-artifact@v3
  #       if: failure()
  #       with:
  #         name: test-logs-aks-${{ matrix.k8s-version }}
  #         path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/

  # eks-integration-test:
  #   runs-on: ubuntu-latest
  #   name: EKS integration tests
    
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       k8s-version: ["1.23", "1.24", "1.25", "1.26", "1.27"]

  #   env:
  #     TF_VAR_test_name: ${{ github.event.inputs.mondooOperatorImageTag }}
  #     AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  #     AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  #     AWS_REGION: us-east-2

  #   steps:
  #     - uses: GitHubSecurityLab/actions-permissions/monitor@v1
  #       with:
  #         config: ${{ vars.PERMISSIONS_CONFIG }}
  #     - uses: actions/checkout@v4
  #       with:
  #         fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile

  #     - name: Import environment variables from file
  #       run: cat ".github/env" >> $GITHUB_ENV

  #     - name: Setup Terraform
  #       uses: hashicorp/setup-terraform@v2

  #     - run: terraform init
  #       working-directory: .github/terraform/aws

  #     - name: Plan EKS
  #       run: terraform plan -out eks-${{ matrix.k8s-version }}.json
  #       env:
  #         TF_VAR_kubernetes_version: ${{ matrix.k8s-version }}
  #       working-directory: .github/terraform/aws
      
  #     - name: Apply EKS
  #       run: terraform apply -auto-approve eks-${{ matrix.k8s-version }}.json
  #       env:
  #         TF_VAR_kubernetes_version: ${{ matrix.k8s-version }}
  #       working-directory: .github/terraform/aws

  #     - uses: actions/setup-go@v4
  #       with:
  #         go-version: "${{ env.golang-version }}"
  #         cache: true

  #     - name: Get operator version
  #       run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

  #     - name: Wait a bit for the cluster to become more stable
  #       run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s

  #     - name: Run integration tests
  #       env:
  #         MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }}
  #         MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342
  #         MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query
  #       run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=eks make test/integration/ci

  #     - name: Clean up EKS terraform
  #       run: terraform destroy -auto-approve
  #       working-directory: .github/terraform/aws
  #       if: success() || failure()

  #     - run: mv integration-tests.xml integration-tests-eks-${{ matrix.k8s-version }}.xml
  #       if: success() || failure()

  #     - name: Upload test results
  #       uses: actions/upload-artifact@v3  # upload test results
  #       if: success() || failure()        # run this step even if previous step failed
  #       with:                             # upload a combined archive with unit and integration test results
  #         name: cloud-test-results
  #         path: integration-tests-eks-${{ matrix.k8s-version }}.xml

  #     - name: Upload test logs artifact
  #       uses: actions/upload-artifact@v3
  #       if: failure()
  #       with:
  #         name: test-logs-eks-${{ matrix.k8s-version }}
  #         path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/

  gke-integration-test:
    runs-on: ubuntu-latest
    name: GKE integration tests
    
    strategy:
      fail-fast: false
      matrix:
        k8s-version: ["1.25"]

    env:
      GOOGLE_APPLICATION_CREDENTIALS: ${{ format('{0}/{1}', github.workspace, 'gcp_sa.json') }}
      KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/gke/kubeconfig') }}

    steps:
      - uses: GitHubSecurityLab/actions-permissions/monitor@v1
        with:
          config: ${{ vars.PERMISSIONS_CONFIG }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile

      - name: Import environment variables from file
        run: cat ".github/env" >> $GITHUB_ENV

      - name: Setup GCP service account
        run: echo ${{ secrets.GCP_SERVICE_ACCOUNT }} | base64 -d > gcp_sa.json

      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v2
        with:
          terraform_wrapper: false

      - run: terraform init
        working-directory: .github/terraform/gke

      - name: Plan GKE
        run: terraform plan -out gke-${{ matrix.k8s-version }}.json
        env:
          TF_VAR_k8s_version: ${{ matrix.k8s-version }}
        working-directory: .github/terraform/gke
      
      - name: Apply GKE
        run: terraform apply -auto-approve gke-${{ matrix.k8s-version }}.json
        env:
          TF_VAR_k8s_version: ${{ matrix.k8s-version }}
        working-directory: .github/terraform/gke

      - uses: actions/setup-go@v4
        with:
          go-version: "${{ env.golang-version }}"
          cache: true

      - name: Get operator version
        run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

      - name: Set terraform output as env
        run: |
          terraform output -json -state=.github/terraform/gke/terraform.tfstate > gke_output.json
          echo "GKE_PROJECT=$(jq -r '.cluster_project.value' gke_output.json)" >> $GITHUB_ENV
          echo "GKE_NAME=$(jq -r '.cluster_name.value' gke_output.json)" >> $GITHUB_ENV
          echo "GKE_LOCATION=$(jq -r '.cluster_location.value' gke_output.json)" >> $GITHUB_ENV
          echo "GKE_ENDPOINT=$(jq -r '.cluster_endpoint.value' gke_output.json)" >> $GITHUB_ENV

      - name: Add GKE CA cert as trusted
        run: |
          jq -r '.cluster_ca_cert.value' gke_output.json > gke_ca.crt
          #sudo cp gke_ca.crt /usr/local/share/ca-certificates/gke_ca.crt
          #sudo update-ca-certificates
          cat $KUBECONFIG | yq '.clusters[0].cluster.certificate-authority-data' | base64 -d > gke_ca_kubectl.crt
          echo "**************"
          diff gke_ca.crt gke_ca_kubectl.crt || true
          echo "***************"
          ls -l $KUBECONFIG
          openssl x509 -in gke_ca.crt -text
          echo "***************"
          kubectl cluster-info || true
          echo "***************"
          echo $GKE_ENDPOINT
          echo "***************"
          echo | openssl s_client -connect ${{ env.GKE_ENDPOINT }}:443  2>/dev/null | openssl x509 -text
          echo "***************"
          openssl x509 -text -in ${{ env.NODE_EXTRA_CA_CERTS }}
          echo "***************"
          kubectl --certificate-authority=${{ env.NODE_EXTRA_CA_CERTS }} cluster-info || true
          


      # https://github.com/actions/runner-images/issues/6778
      # https://github.com/actions/runner-images/issues/5925#issuecomment-1365975455
      # - name: Install gke auth plugin
      #   run: |
      #     echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
      #     curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
      #     sudo apt update -y
      #     sudo apt-get install -y google-cloud-sdk-gke-gcloud-auth-plugin
    
      # - name: Prepare kubeconfig
      #   run: |
      #     gcloud auth login --cred-file=${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
      #     gcloud config set project ${{ env.GKE_PROJECT }}
      #     gcloud container clusters get-credentials ${{ env.GKE_NAME }} --location ${{ env.GKE_LOCATION }}
          
      # - name: Update certs
      #   run: sudo apt -y install ca-certificates

      - name: Wait a bit for the cluster to become more stable
        run: |
          #sleep 60
          kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=300s

      - name: Run integration tests
        env:
          MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }}
          MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342
          MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query
        run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=gke make test/integration/ci

      - name: Clean up GKE terraform
        run: terraform destroy -auto-approve
        working-directory: .github/terraform/gke
        if: success() || failure()

      - name: Cleanup GCP service account
        run: rm gcp_sa.json
        if: success() || failure()

      - run: mv integration-tests.xml integration-tests-gke-${{ matrix.k8s-version }}.xml
        if: success() || failure()

      - name: Upload test results
        uses: actions/upload-artifact@v3  # upload test results
        if: success() || failure()        # run this step even if previous step failed
        with:                             # upload a combined archive with unit and integration test results
          name: cloud-test-results
          path: integration-tests-gke-${{ matrix.k8s-version }}.xml

      - name: Upload test logs artifact
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: test-logs-gke-${{ matrix.k8s-version }}
          path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/

  test-report:
    name: Report test results
    runs-on: ubuntu-latest
#    needs: [eks-integration-test,aks-integration-test,gke-integration-test]
    needs: [gke-integration-test]
    if: always()
    steps:
    - uses: GitHubSecurityLab/actions-permissions/monitor@v1
      with:
        config: ${{ vars.PERMISSIONS_CONFIG }}
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile

    - name: Download test results
      uses: actions/download-artifact@v3
      with:
        name: cloud-test-results

    - name: Parse test results
      uses: dorny/test-reporter@v1
      with:
        name: Report cloud test results
        path: '*.xml'                     # Path to test results
        reporter: java-junit              # Format of test results

  discord-notification:
    runs-on: ubuntu-latest
    name: Send Discord notification
#    needs: [eks-integration-test,aks-integration-test,gke-integration-test]
    needs: [gke-integration-test]
    # Run only if the previous job has failed and only if it's running against the main branch
    if: ${{ always() && contains(join(needs.*.result, ','), 'fail') && github.ref_name == 'main' }}
    steps:
      - uses: GitHubSecurityLab/actions-permissions/monitor@v1
        with:
          config: ${{ vars.PERMISSIONS_CONFIG }}
      - uses: sarisia/actions-status-discord@v1
        with:
          webhook: ${{ secrets.DISCORD_WEBHOOK }}
          status: Failure
          url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          description: Workflow ${{ github.workflow }} failed for commit ${{ github.sha }}.
          color: 0xff4d4d