From df0c005e80761eea7e1e79b7b7476b201cd0379e Mon Sep 17 00:00:00 2001 From: Christian Zunker Date: Mon, 30 Oct 2023 16:58:06 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Fix=20GKE=20cloud=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian Zunker --- .github/terraform/gke/outputs.tf | 16 ++ .github/workflows/cloud-tests.yaml | 371 ++++++++++++++++------------- 2 files changed, 219 insertions(+), 168 deletions(-) diff --git a/.github/terraform/gke/outputs.tf b/.github/terraform/gke/outputs.tf index 57b9f3cb9..cc59176aa 100644 --- a/.github/terraform/gke/outputs.tf +++ b/.github/terraform/gke/outputs.tf @@ -2,4 +2,20 @@ resource "local_file" "kubeconfig" { depends_on = [google_container_cluster.cluster] content = module.gke_auth.kubeconfig_raw filename = "kubeconfig" +} + +output "cluster_name" { + value = google_container_cluster.cluster.name +} + +output "cluster_location" { + value = google_container_cluster.cluster.location +} + +output "cluster_project" { + value = google_container_cluster.cluster.project +} + +output "cluster_ca_cert" { + value = module.gke_auth.cluster_ca_certificate } \ No newline at end of file diff --git a/.github/workflows/cloud-tests.yaml b/.github/workflows/cloud-tests.yaml index 15eb40646..38b19ee4a 100644 --- a/.github/workflows/cloud-tests.yaml +++ b/.github/workflows/cloud-tests.yaml @@ -38,177 +38,177 @@ env: CNSPEC_IMAGE_TAG: ${{ github.event.inputs.cnspecImageTag || 'edge-latest-rootless' }} jobs: - aks-integration-test: - runs-on: ubuntu-latest - name: AKS integration tests - - env: - ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} - ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/aks/kubeconfig') }} + # aks-integration-test: + # runs-on: ubuntu-latest + # name: AKS integration tests + + # env: + # ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + # ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + # ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + # ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + # KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/aks/kubeconfig') }} - strategy: - fail-fast: false - matrix: - k8s-version: ["1.25", "1.26", "1.27"] - - steps: - - uses: GitHubSecurityLab/actions-permissions/monitor@v1 - with: - config: ${{ vars.PERMISSIONS_CONFIG }} - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile - - - name: Import environment variables from file - run: cat ".github/env" >> $GITHUB_ENV - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - - - name: Terraform init - run: terraform init - working-directory: .github/terraform/aks + # strategy: + # fail-fast: false + # matrix: + # k8s-version: ["1.25", "1.26", "1.27"] + + # steps: + # - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + # with: + # config: ${{ vars.PERMISSIONS_CONFIG }} + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile + + # - name: Import environment variables from file + # run: cat ".github/env" >> $GITHUB_ENV + + # - name: Setup Terraform + # uses: hashicorp/setup-terraform@v2 + + # - name: Terraform init + # run: terraform init + # working-directory: .github/terraform/aks - - name: Terraform plan - run: terraform plan -out aks-${{ matrix.k8s-version }}.json - env: - TF_VAR_k8s_version: ${{ matrix.k8s-version }} - working-directory: .github/terraform/aks - - - name: Terraform apply - run: terraform apply -auto-approve aks-${{ matrix.k8s-version }}.json - env: - TF_VAR_k8s_version: ${{ matrix.k8s-version }} - working-directory: .github/terraform/aks - - - uses: actions/setup-go@v4 - with: - go-version: "${{ env.golang-version }}" - cache: true - - - name: Get operator version - run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV - - - name: Wait a bit for the cluster to become more stable - run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s - - - name: Run integration tests - env: - MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }} - MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342 - MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query - run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=aks make test/integration/ci - - - name: Clean up AKS terraform - run: terraform destroy -auto-approve - if: success() || failure() - working-directory: .github/terraform/aks + # - name: Terraform plan + # run: terraform plan -out aks-${{ matrix.k8s-version }}.json + # env: + # TF_VAR_k8s_version: ${{ matrix.k8s-version }} + # working-directory: .github/terraform/aks + + # - name: Terraform apply + # run: terraform apply -auto-approve aks-${{ matrix.k8s-version }}.json + # env: + # TF_VAR_k8s_version: ${{ matrix.k8s-version }} + # working-directory: .github/terraform/aks + + # - uses: actions/setup-go@v4 + # with: + # go-version: "${{ env.golang-version }}" + # cache: true + + # - name: Get operator version + # run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV + + # - name: Wait a bit for the cluster to become more stable + # run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s + + # - name: Run integration tests + # env: + # MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }} + # MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342 + # MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query + # run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=aks make test/integration/ci + + # - name: Clean up AKS terraform + # run: terraform destroy -auto-approve + # if: success() || failure() + # working-directory: .github/terraform/aks - - run: mv integration-tests.xml integration-tests-aks-${{ matrix.k8s-version }}.xml - if: success() || failure() - - - name: Upload cloud test results - uses: actions/upload-artifact@v3 # upload test results - if: success() || failure() # run this step even if previous step failed - with: # upload a combined archive with unit and integration test results - name: cloud-test-results - path: | - integration-tests-aks-${{ matrix.k8s-version }}.xml - .github/terraform/aks/aks-${{ matrix.k8s-version }}.json - - - name: Upload test logs artifact - uses: actions/upload-artifact@v3 - if: failure() - with: - name: test-logs-aks-${{ matrix.k8s-version }} - path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/ - - eks-integration-test: - runs-on: ubuntu-latest - name: EKS integration tests + # - run: mv integration-tests.xml integration-tests-aks-${{ matrix.k8s-version }}.xml + # if: success() || failure() + + # - name: Upload cloud test results + # uses: actions/upload-artifact@v3 # upload test results + # if: success() || failure() # run this step even if previous step failed + # with: # upload a combined archive with unit and integration test results + # name: cloud-test-results + # path: | + # integration-tests-aks-${{ matrix.k8s-version }}.xml + # .github/terraform/aks/aks-${{ matrix.k8s-version }}.json + + # - name: Upload test logs artifact + # uses: actions/upload-artifact@v3 + # if: failure() + # with: + # name: test-logs-aks-${{ matrix.k8s-version }} + # path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/ + + # eks-integration-test: + # runs-on: ubuntu-latest + # name: EKS integration tests - strategy: - fail-fast: false - matrix: - k8s-version: ["1.23", "1.24", "1.25", "1.26", "1.27"] - - env: - TF_VAR_test_name: ${{ github.event.inputs.mondooOperatorImageTag }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-east-2 - - steps: - - uses: GitHubSecurityLab/actions-permissions/monitor@v1 - with: - config: ${{ vars.PERMISSIONS_CONFIG }} - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile - - - name: Import environment variables from file - run: cat ".github/env" >> $GITHUB_ENV - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - - - run: terraform init - working-directory: .github/terraform/aws - - - name: Plan EKS - run: terraform plan -out eks-${{ matrix.k8s-version }}.json - env: - TF_VAR_kubernetes_version: ${{ matrix.k8s-version }} - working-directory: .github/terraform/aws + # strategy: + # fail-fast: false + # matrix: + # k8s-version: ["1.23", "1.24", "1.25", "1.26", "1.27"] + + # env: + # TF_VAR_test_name: ${{ github.event.inputs.mondooOperatorImageTag }} + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # AWS_REGION: us-east-2 + + # steps: + # - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + # with: + # config: ${{ vars.PERMISSIONS_CONFIG }} + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile + + # - name: Import environment variables from file + # run: cat ".github/env" >> $GITHUB_ENV + + # - name: Setup Terraform + # uses: hashicorp/setup-terraform@v2 + + # - run: terraform init + # working-directory: .github/terraform/aws + + # - name: Plan EKS + # run: terraform plan -out eks-${{ matrix.k8s-version }}.json + # env: + # TF_VAR_kubernetes_version: ${{ matrix.k8s-version }} + # working-directory: .github/terraform/aws - - name: Apply EKS - run: terraform apply -auto-approve eks-${{ matrix.k8s-version }}.json - env: - TF_VAR_kubernetes_version: ${{ matrix.k8s-version }} - working-directory: .github/terraform/aws - - - uses: actions/setup-go@v4 - with: - go-version: "${{ env.golang-version }}" - cache: true - - - name: Get operator version - run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV - - - name: Wait a bit for the cluster to become more stable - run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s - - - name: Run integration tests - env: - MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }} - MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342 - MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query - run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=eks make test/integration/ci - - - name: Clean up EKS terraform - run: terraform destroy -auto-approve - working-directory: .github/terraform/aws - if: success() || failure() - - - run: mv integration-tests.xml integration-tests-eks-${{ matrix.k8s-version }}.xml - if: success() || failure() - - - name: Upload test results - uses: actions/upload-artifact@v3 # upload test results - if: success() || failure() # run this step even if previous step failed - with: # upload a combined archive with unit and integration test results - name: cloud-test-results - path: integration-tests-eks-${{ matrix.k8s-version }}.xml - - - name: Upload test logs artifact - uses: actions/upload-artifact@v3 - if: failure() - with: - name: test-logs-eks-${{ matrix.k8s-version }} - path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/ + # - name: Apply EKS + # run: terraform apply -auto-approve eks-${{ matrix.k8s-version }}.json + # env: + # TF_VAR_kubernetes_version: ${{ matrix.k8s-version }} + # working-directory: .github/terraform/aws + + # - uses: actions/setup-go@v4 + # with: + # go-version: "${{ env.golang-version }}" + # cache: true + + # - name: Get operator version + # run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV + + # - name: Wait a bit for the cluster to become more stable + # run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s + + # - name: Run integration tests + # env: + # MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }} + # MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342 + # MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query + # run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=eks make test/integration/ci + + # - name: Clean up EKS terraform + # run: terraform destroy -auto-approve + # working-directory: .github/terraform/aws + # if: success() || failure() + + # - run: mv integration-tests.xml integration-tests-eks-${{ matrix.k8s-version }}.xml + # if: success() || failure() + + # - name: Upload test results + # uses: actions/upload-artifact@v3 # upload test results + # if: success() || failure() # run this step even if previous step failed + # with: # upload a combined archive with unit and integration test results + # name: cloud-test-results + # path: integration-tests-eks-${{ matrix.k8s-version }}.xml + + # - name: Upload test logs artifact + # uses: actions/upload-artifact@v3 + # if: failure() + # with: + # name: test-logs-eks-${{ matrix.k8s-version }} + # path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/ gke-integration-test: runs-on: ubuntu-latest @@ -222,6 +222,7 @@ jobs: env: GOOGLE_APPLICATION_CREDENTIALS: ${{ format('{0}/{1}', github.workspace, 'gcp_sa.json') }} KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/gke/kubeconfig') }} + USE_GKE_GCLOUD_AUTH_PLUGIN: True steps: - uses: GitHubSecurityLab/actions-permissions/monitor@v1 @@ -239,6 +240,8 @@ jobs: - name: Setup Terraform uses: hashicorp/setup-terraform@v2 + with: + terraform_wrapper: false - run: terraform init working-directory: .github/terraform/gke @@ -263,6 +266,36 @@ jobs: - name: Get operator version run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV + - name: Set terraform output as env + run: | + terraform output -json -state=.github/terraform/gke/terraform.tfstate > gke_output.json + cat gke_output.json + echo "GKE_PROJECT=$(jq -r '.cluster_project.value' gke_output.json)" >> $GITHUB_ENV + echo "GKE_NAME=$(jq -r '.cluster_name.value' gke_output.json)" >> $GITHUB_ENV + echo "GKE_LOCATION=$(jq -r '.cluster_location.value' gke_output.json)" >> $GITHUB_ENV + echo "GKE_CA_BASE64=$(jq -r '.cluster_ca_cert.value' gke_output.json)" >> $GITHUB_ENV + + - name: Add GKE CA cert as trusted + run: | + echo ${{ env.GKE_CA_BASE64 }} | base64 -d > gke_ca.crt + sudo cp gke_ca.crt /usr/local/share/ca-certificates/gke_ca.crt + sudo update-ca-certificates + + # https://github.com/actions/runner-images/issues/6778 + # https://github.com/actions/runner-images/issues/5925#issuecomment-1365975455 + # - name: Install gke auth plugin + # run: | + # echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + # curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + # sudo apt update -y + # sudo apt-get install -y google-cloud-sdk-gke-gcloud-auth-plugin + + # - name: Prepare kubeconfig + # run: | + # gcloud auth login --cred-file=${{ env.GOOGLE_APPLICATION_CREDENTIALS }} + # gcloud config set project ${{ env.GKE_PROJECT }} + # gcloud container clusters get-credentials ${{ env.GKE_NAME }} --location ${{ env.GKE_LOCATION }} + - name: Wait a bit for the cluster to become more stable run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s @@ -302,7 +335,8 @@ jobs: test-report: name: Report test results runs-on: ubuntu-latest - needs: [eks-integration-test,aks-integration-test,gke-integration-test] +# needs: [eks-integration-test,aks-integration-test,gke-integration-test] + needs: [gke-integration-test] if: always() steps: - uses: GitHubSecurityLab/actions-permissions/monitor@v1 @@ -327,7 +361,8 @@ jobs: discord-notification: runs-on: ubuntu-latest name: Send Discord notification - needs: [eks-integration-test,aks-integration-test,gke-integration-test] +# needs: [eks-integration-test,aks-integration-test,gke-integration-test] + needs: [gke-integration-test] # Run only if the previous job has failed and only if it's running against the main branch if: ${{ always() && contains(join(needs.*.result, ','), 'fail') && github.ref_name == 'main' }} steps: