From e9d112acdeb11fdd8b4cdd9e5dac9b357c51f5de Mon Sep 17 00:00:00 2001 From: Vadim Tsarfin <139264463+vtsarfin-cg@users.noreply.github.com> Date: Tue, 17 Sep 2024 08:32:20 +0300 Subject: [PATCH] Fix/ci artifact store access (#101) * initial * fix: ci- instead of ci_ * add cloud_binary_artifacts_store_access_key to atlantis secrets * fix: cleanup old cwft, megalinter upgrade to v8 * workflow-template cleanup --------- Co-authored-by: Alexander Ulyanov Co-authored-by: Serg Shalavin --- .../argo-workflows/application.yaml | 13 +--- .../black-cwft.yaml | 19 ------ .../build-chain-cwft.yaml | 53 --------------- .../crane-cwft.yaml | 20 ------ .../crane-s3-cwft.yaml | 26 ------- .../eslint-cwft.yaml | 24 ------- .../git-clone-cwft.yaml | 49 -------------- .../git-clone-no-proxy-cwft.yaml | 33 --------- .../kaniko-cwft.yaml | 36 ---------- .../kaniko-no-proxy-cwft.yaml | 26 ------- .../kaniko-s3-cwft.yaml | 36 ---------- .../kaniko-s3-p-experimental-cwft.yaml | 51 -------------- .../megalinter-cwft.yaml | 2 +- .../trivy-fs-scan-cwft.yaml | 15 ----- .../workflow-templates/.placeholder | 0 .../workflow-templates/black-wft.yaml | 15 ----- .../workflow-templates/crane-wft.yaml | 18 ----- .../workflow-templates/eslint-wft.yaml | 24 ------- .../workflow-templates/git-clone-wft.yaml | 46 ------------- .../workflow-templates/kaniko-wft.yaml | 25 ------- .../workflow-templates/promote-wft.yaml | 61 ----------------- .../workflow-templates/trivy-fs-scan-wft.yaml | 20 ------ .../workflow-templates/tslint-wft.yaml | 15 ----- .../version-changer-wft.yaml | 67 ------------------- platform/terraform/hosting_provider/output.tf | 11 +++ .../terraform/modules/cloud_aws/outputs.tf | 12 ++++ .../terraform/modules/cloud_azure/outputs.tf | 12 ++++ .../terraform/modules/cloud_gcp/output.tf | 13 ++++ .../modules/secrets_vault/secrets.tf | 56 ++++++++-------- .../modules/secrets_vault/variables.tf | 6 ++ platform/terraform/secrets/main.tf | 25 +++---- platform/terraform/secrets/variable.tf | 6 ++ tools/cli/commands/setup.py | 5 ++ tools/cli/commands/workload/bootstrap.py | 5 +- tools/cli/services/cloud/aws/aws_manager.py | 15 +++++ .../cli/services/cloud/azure/azure_manager.py | 14 ++++ .../services/cloud/cloud_provider_manager.py | 8 +++ tools/cli/services/cloud/gcp/gcp_manager.py | 7 ++ 38 files changed, 157 insertions(+), 732 deletions(-) delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/black-cwft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/build-chain-cwft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/crane-cwft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/crane-s3-cwft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/eslint-cwft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/git-clone-cwft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/git-clone-no-proxy-cwft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-cwft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-no-proxy-cwft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-s3-cwft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-s3-p-experimental-cwft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/trivy-fs-scan-cwft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/.placeholder delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/promote-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/trivy-fs-scan-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml index e0088837..87c98d88 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml @@ -107,19 +107,8 @@ spec: scopes: [email, openid, groups, user, profile] rbac: enabled: true - useDefaultArtifactRepo: true + useDefaultArtifactRepo: false useStaticCredentials: false - artifactRepository: - archiveLogs: false - s3: - insecure: false - bucket: - keyFormat: argo-workflows/artifacts/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{workflow.uid}}/{{workflow.name}}/{{pod.name}} - endpoint: s3.amazonaws.com - region: - useSDKCreds: true - encryptionOptions: - enableEncryption: false chart: argo-workflows destination: server: https://kubernetes.default.svc diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/black-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/black-cwft.yaml deleted file mode 100644 index 42c113d1..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/black-cwft.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: black-cwft -spec: - templates: - - name: black - container: - image: '{{workflow.parameters.dockerhub-registry-proxy}}/pyfound/black' - imagePullPolicy: IfNotPresent - command: - - black - - --check - - -v - - /workspace/{{workflow.parameters.wl-service-name}} - volumeMounts: - - name: build - mountPath: /workspace diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/build-chain-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/build-chain-cwft.yaml deleted file mode 100644 index 6300fc2c..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/build-chain-cwft.yaml +++ /dev/null @@ -1,53 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: build-chain-cwft -spec: - volumeClaimTemplates: - - metadata: - name: build - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 8Gi - volumes: - - name: kaniko-secret - secret: - secretName: proxy-docker-config - items: - - key: .dockerconfigjson - path: config.json - imagePullPolicy: IfNotPresent - imagePullSecrets: - - name: proxy-docker-config - entrypoint: ci-sequence - templates: - - name: ci-sequence - dag: - tasks: - - name: git-clone - templateRef: - name: git-clone-cwft - template: git-clone - clusterScope: true - - name: kaniko - dependencies: [git-clone] - templateRef: - name: kaniko-s3-cwft - template: kaniko - clusterScope: true - - name: trivy-fs-scan - dependencies: [git-clone] - templateRef: - name: trivy-fs-s3-cwft - template: trivy-fs-scan - clusterScope: true - - name: megalinter - dependencies: [git-clone] - templateRef: - name: megalinter-cwft - template: megalinter - clusterScope: true - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/crane-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/crane-cwft.yaml deleted file mode 100644 index 95c97e4a..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/crane-cwft.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: crane-cwft -spec: - templates: - - name: crane - container: - image: '{{workflow.parameters.gcr-registry-proxy}}/go-containerregistry/crane' - imagePullPolicy: IfNotPresent - args: - - push - - /workspace/{{workflow.parameters.wl-service-name}}.tar - - '{{workflow.parameters.registry}}/{{workflow.parameters.workload-name}}/{{workflow.parameters.wl-service-name}}:{{workflow.parameters.tag}}' - volumeMounts: - - name: crane-secret - mountPath: /home/nonroot/.docker - - name: build - mountPath: /workspace diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/crane-s3-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/crane-s3-cwft.yaml deleted file mode 100644 index 844f7e1c..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/crane-s3-cwft.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: crane-s3-cwft -spec: - templates: - - name: crane - inputs: - artifacts: - - name: tar-build - path: /tmp/{{workflow.parameters.wl-service-name}}.tar - s3: - key: "{{workflow.parameters.workload-name}}/{{workflow.parameters.tag}}/{{workflow.parameters.wl-service-name}}-tar" - - container: - image: '{{workflow.parameters.gcr-registry-proxy}}/go-containerregistry/crane' - imagePullPolicy: IfNotPresent - args: - - push - - /tmp/{{workflow.parameters.wl-service-name}}.tar - - '{{workflow.parameters.registry}}/{{workflow.parameters.workload-name}}/{{workflow.parameters.wl-service-name}}:{{workflow.parameters.tag}}' - volumeMounts: - - name: crane-secret - mountPath: /home/nonroot/.docker - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/eslint-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/eslint-cwft.yaml deleted file mode 100644 index 42f7807c..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/eslint-cwft.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: eslint-cwft -spec: - templates: - - name: eslint - script: - image: "{{workflow.parameters.dockerhub-registry-proxy}}/pipelinecomponents/eslint:latest" - imagePullPolicy: IfNotPresent - env: - - name: WL_SERVICE - value: "{{workflow.parameters.wl-service-name}}" - command: [sh] - source: | - echo $WL_SERVICE - cd /build/$WL_SERVICE - ls src/**/*.ts* > /dev/null 2>&1 || { echo "Nothing to lint, exiting"; exit 1;} - npm -s --prefix /app install eslint-config-standard-with-typescript eslint-plugin-prettier eslint-config-prettier eslint-plugin-only-warn - eslint --plugin only-warn 'src/**/*.ts?' - volumeMounts: - - name: build - mountPath: /build - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/git-clone-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/git-clone-cwft.yaml deleted file mode 100644 index e14cd832..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/git-clone-cwft.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: git-clone-cwft -spec: - templates: - - name: git-clone - inputs: - artifacts: - - name: git-src - path: /src - git: - repo: '{{workflow.parameters.repo}}' - revision: '{{workflow.parameters.tag}}' - sshPrivateKeySecret: - name: ci-secrets - key: SSH_PRIVATE_KEY - depth: 1 - script: - image: '{{workflow.parameters.dockerhub-registry-proxy}}/alpine/git' - imagePullPolicy: IfNotPresent - env: - - name: WL_SERVICE - value: '{{workflow.parameters.wl-service-name}}' - - name: WL_SERVICE_DIR - value: '{{workflow.parameters.wl-service-dir}}' - - name: REGISTRY_MIRROR - value: '{{workflow.parameters.kaniko-registry-mirror}}' - - name: DOCKERHUB_REGISTRY_PROXY - value: '{{workflow.parameters.dockerhub-registry-proxy}}' - - - command: [sh] - source: | - DOCKERHUB_PROXY_PREFIX=`basename $DOCKERHUB_REGISTRY_PROXY` - echo "WL_SERVICE: $WL_SERVICE" - echo "Dockerfile-parameter {{workflow.parameters.dockerfile}}" - cp -r /src/. /build/ && echo "Copy successful." - echo "REGISTRY_MIRROR: $REGISTRY_MIRROR" - if [ $REGISTRY_MIRROR != NOMIRROR ] - then - cd /build/$WL_SERVICE_DIR - sed -r -i.orig "s/FROM\s+(\S+(\/)\S+)/FROM $DOCKERHUB_PROXY_PREFIX\/\1/;s/FROM\s+(\S+)/FROM $DOCKERHUB_PROXY_PREFIX\/library\/\1/" Dockerfile - cat Dockerfile - fi - volumeMounts: - - mountPath: /build - name: build diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/git-clone-no-proxy-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/git-clone-no-proxy-cwft.yaml deleted file mode 100644 index 09d7a9d5..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/git-clone-no-proxy-cwft.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: git-clone-no-proxy-cwft -spec: - templates: - - name: git-clone - inputs: - artifacts: - - name: git-src - path: /src - git: - repo: '{{workflow.parameters.repo}}' - revision: '{{workflow.parameters.tag}}' - sshPrivateKeySecret: - name: ci-secrets - key: SSH_PRIVATE_KEY - depth: 1 - script: - image: '{{workflow.parameters.dockerhub-registry-proxy}}/alpine/git' - imagePullPolicy: IfNotPresent - env: - - name: WL_SERVICE - value: '{{workflow.parameters.wl-service-name}}' - command: [sh] - source: | - echo "WL_SERVICE: $WL_SERVICE" - echo "Dockerfile-parameter {{workflow.parameters.dockerfile}}" - cp -r /src/* /build/ && echo "Copy successful." - volumeMounts: - - mountPath: /build - name: build diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-cwft.yaml deleted file mode 100644 index 4aca4161..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-cwft.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: kaniko-cwft -spec: - templates: - - name: kaniko - container: - env: - - name: WL_SERVICE - value: '{{workflow.parameters.wl-service-name}}' - - name: DOCKERFILE - value: '{{workflow.parameters.dockerfile}}' - - name: BUILD_CONTEXT - value: '{{workflow.parameters.build-context}}' - image: '{{workflow.parameters.gcr-registry-proxy}}/kaniko-project/executor' - imagePullPolicy: IfNotPresent - args: - - --dockerfile={{workflow.parameters.dockerfile}} - - --context=dir:///workspace/{{workflow.parameters.build-context}} - - --no-push - - --tar-path=/workspace/{{workflow.parameters.wl-service-name}}.tar - - --registry-mirror={{workflow.parameters.registry}} - - --skip-default-registry-fallback - - --snapshot-mode=time - - --use-new-run - - --cache=true - - --cache-copy-layers - - --cache-run-layers - - --cache-repo={{workflow.parameters.registry}}/kaniko-cache/{{workflow.parameters.workload-name}}-{{workflow.parameters.wl-service-name}} - volumeMounts: - - name: kaniko-secret - mountPath: /kaniko/.docker - - name: build - mountPath: /workspace diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-no-proxy-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-no-proxy-cwft.yaml deleted file mode 100644 index 89699578..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-no-proxy-cwft.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: kaniko-no-proxy-cwft -spec: - templates: - - name: kaniko - container: - image: '{{workflow.parameters.gcr-registry-proxy}}/kaniko-project/executor' - imagePullPolicy: IfNotPresent - args: - - --dockerfile={{workflow.parameters.dockerfile}} - - --context=dir:///workspace/{{workflow.parameters.build-context}} - - --no-push - - --tar-path=/workspace/{{workflow.parameters.wl-service-name}}.tar - - --snapshot-mode=time - - --use-new-run - - --cache=false - - --cache-run-layers - - --cache-repo={{workflow.parameters.registry}}/kaniko-cache/{{workflow.parameters.workload-name}}-{{workflow.parameters.wl-service-name}} - volumeMounts: - - name: kaniko-secret - mountPath: /kaniko/.docker - - name: build - mountPath: /workspace diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-s3-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-s3-cwft.yaml deleted file mode 100644 index cbb8de12..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-s3-cwft.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: kaniko-s3-cwft -spec: - templates: - - name: kaniko - container: - image: '{{workflow.parameters.gcr-registry-proxy}}/kaniko-project/executor' - imagePullPolicy: IfNotPresent - args: - - --dockerfile={{workflow.parameters.dockerfile}} - - --context=dir:///workspace/{{workflow.parameters.build-context}} - - --no-push - - --tar-path=/tmp/{{workflow.parameters.wl-service-name}}.tar - - --registry-mirror={{workflow.parameters.kaniko-registry-mirror}} - - --snapshot-mode=time - - --use-new-run - - --compressed-caching=false - - --cache={{workflow.parameters.kaniko-cache}} - - --cache-run-layers - - --cache-repo={{workflow.parameters.kaniko-cache-repo}}/kaniko-cache/{{workflow.parameters.workload-name}}-{{workflow.parameters.wl-service-name}} - volumeMounts: - - name: kaniko-secret - mountPath: /kaniko/.docker - - name: build - mountPath: /workspace - outputs: - artifacts: - - name: tar-build - path: /tmp/{{workflow.parameters.wl-service-name}}.tar - s3: - key: "{{workflow.parameters.workload-name}}/{{workflow.parameters.tag}}/{{workflow.parameters.wl-service-name}}-tar" - artifactGC: - strategy: Never diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-s3-p-experimental-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-s3-p-experimental-cwft.yaml deleted file mode 100644 index ff0fb2c5..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/kaniko-s3-p-experimental-cwft.yaml +++ /dev/null @@ -1,51 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: kaniko-s3-p-experimental-cwft -spec: - templates: - - name: kaniko - archiveLocation: - archiveLogs: true - inputs: - artifacts: - - name: git-src - path: /build - git: - repo: '{{workflow.parameters.repo}}' - revision: '{{workflow.parameters.tag}}' - sshPrivateKeySecret: - name: ci-secrets - key: SSH_PRIVATE_KEY - depth: 1 - container: - image: '{{workflow.parameters.gcr-registry-proxy}}/kaniko-project/executor:latest' - imagePullPolicy: "IfNotPresent" - args: - - --registry-map=index.docker.io={{workflow.parameters.dockerhub-registry-proxy}};gcr.io={{workflow.parameters.gcr-registry-proxy}} - # - --registry-mirror={{workflow.parameters.kaniko-registry-mirror}} - # - --skip-default-registry-fallback - - --dockerfile={{workflow.parameters.dockerfile}} - - --context=dir:///build/{{workflow.parameters.build-context}} - - --no-push - - --tar-path=/tmp/{{workflow.parameters.wl-service-name}}.tar - - --snapshot-mode=time - - --use-new-run - # - --compressed-caching=false - # #- --cache={{workflow.parameters.kaniko-cache}} - # - --cache=false - # - --cache-run-layers - # - --cache-repo=oci://{{workflow.parameters.kaniko-cache-repo}}/kaniko-cache/{{workflow.parameters.workload-name}}-{{workflow.parameters.wl-service-name}} - # - --verbosity=debug - volumeMounts: - - name: kaniko-secret - mountPath: /kaniko/.docker - outputs: - artifacts: - - name: tar-build - path: /tmp/{{workflow.parameters.wl-service-name}}.tar - s3: - key: "{{workflow.parameters.workload-name}}/{{workflow.parameters.tag}}/{{workflow.parameters.wl-service-name}}-tar" - artifactGC: - strategy: Never diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/megalinter-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/megalinter-cwft.yaml index 2d1adbef..175a2f75 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/megalinter-cwft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/megalinter-cwft.yaml @@ -18,7 +18,7 @@ spec: key: SSH_PRIVATE_KEY depth: 1 container: - image: "{{workflow.parameters.dockerhub-registry-proxy}}/oxsecurity/megalinter:latest" + image: "{{workflow.parameters.dockerhub-registry-proxy}}/oxsecurity/megalinter:v8" imagePullPolicy: IfNotPresent env: - name: DEFAULT_WORKSPACE diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/trivy-fs-scan-cwft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/trivy-fs-scan-cwft.yaml deleted file mode 100644 index 1ee11b09..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/trivy-fs-scan-cwft.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: trivy-fs-scan-cwft -spec: - templates: - - name: trivy-fs-scan - container: - image: '{{workflow.parameters.dockerhub-registry-proxy}}/aquasec/trivy' - imagePullPolicy: IfNotPresent - args: [fs, '--no-progress', '/build/{{workflow.parameters.wl-service-dir}}'] - volumeMounts: - - name: build - mountPath: /build diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/.placeholder b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/.placeholder new file mode 100644 index 00000000..e69de29b diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml deleted file mode 100644 index 78242707..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: black-wft -spec: - templates: - - name: black - container: - image: "{{workflow.parameters.dockerhub-registry-proxy}}/pyfound/black" - imagePullPolicy: IfNotPresent - command: ["black", "--check", "-v", "/workspace/{{workflow.parameters.wl-service-name}}"] - volumeMounts: - - name: build - mountPath: /workspace - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml deleted file mode 100644 index 08b0ba42..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: crane-wft -spec: - templates: - - name: crane - container: - image: "{{workflow.parameters.gcr-registry-proxy}}/go-containerregistry/crane" - imagePullPolicy: IfNotPresent - args: ["push", "/workspace/{{workflow.parameters.wl-service-name}}.tar", - "{{workflow.parameters.registry}}/{{workflow.parameters.workload-name}}/{{workflow.parameters.wl-service-name}}:{{workflow.parameters.tag}}"] - volumeMounts: - - name: kaniko-secret - mountPath: /home/nonroot/.docker - - name: build - mountPath: /workspace - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml deleted file mode 100644 index 556d96e3..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: eslint-wft -spec: - templates: - - name: eslint - script: - image: "{{workflow.parameters.dockerhub-registry-proxy}}/pipelinecomponents/eslint:latest" - imagePullPolicy: IfNotPresent - env: - - name: WL_SERVICE - value: "{{workflow.parameters.wl-service-name}}" - command: [sh] - source: | - echo $WL_SERVICE - cd /build/$WL_SERVICE - ls src/**/*.ts* > /dev/null 2>&1 || { echo "Nothing to lint, exiting"; exit 1;} - npm -s --prefix /app install eslint-config-standard-with-typescript eslint-plugin-prettier eslint-config-prettier eslint-plugin-only-warn - eslint --plugin only-warn 'src/**/*.ts?' - volumeMounts: - - name: build - mountPath: /build - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml deleted file mode 100644 index be49405c..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: git-clone-wft -spec: - templates: - - name: git-clone - inputs: - artifacts: - - name: git-src - path: /src - git: - repo: "{{workflow.parameters.repo}}" - revision: "{{workflow.parameters.tag}}" - sshPrivateKeySecret: - name: ci-secrets - key: SSH_PRIVATE_KEY - depth: 1 - - script: - image: "{{workflow.parameters.dockerhub-registry-proxy}}/alpine/git" - imagePullPolicy: IfNotPresent - env: - - name: WL_SERVICE - value: "{{workflow.parameters.wl-service-name}}" - - name: REGISTRY_MIRROR - value: "{{workflow.parameters.registry}}" - - name: DOCKERHUB_REGISTRY_PROXY - value: "{{workflow.parameters.dockerhub-registry-proxy}}" - - - command: [sh] - source: | - DOCKERHUB_PROXY_PREFIX=`basename $DOCKERHUB_REGISTRY_PROXY` - echo "WL_SERVICE: $WL_SERVICE" - cp -rv /src/$WL_SERVICE /build/ - if [[ $REGISTRY_MIRROR ]] - then - cd /build/$WL_SERVICE - sed -r -i.orig "s/FROM\s+(\S+(\/)\S+)/FROM $DOCKERHUB_PROXY_PREFIX\/\1/;s/FROM\s+(\S+)/FROM $DOCKERHUB_PROXY_PREFIX\/library\/\1/" Dockerfile - cat Dockerfile - fi - volumeMounts: - - mountPath: "/build" - name: build - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml deleted file mode 100644 index e25f53b3..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: kaniko-wft -spec: - templates: - - name: kaniko - container: - env: - - name: WL_SERVICE - value: "{{workflow.parameters.wl-service-name}}" - image: "{{workflow.parameters.gcr-registry-proxy}}/kaniko-project/executor" - imagePullPolicy: IfNotPresent - args: ["--dockerfile=Dockerfile", - "--context=dir:///workspace/{{workflow.parameters.wl-service-name}}/", - "--no-push", "--tar-path=/workspace/{{workflow.parameters.wl-service-name}}.tar", - "--registry-mirror={{workflow.parameters.registry}}", - "--skip-default-registry-fallback" - ] - volumeMounts: - - name: kaniko-secret - mountPath: /kaniko/.docker - - name: build - mountPath: /workspace - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/promote-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/promote-wft.yaml deleted file mode 100644 index e7e9dc6b..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/promote-wft.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: promote-wft -spec: - entrypoint: promote - templates: - - name: promote - inputs: - artifacts: - - name: git-src - path: /src - git: - repo: "{{workflow.parameters.repo}}" - sshPrivateKeySecret: - name: ci-secrets - key: SSH_PRIVATE_KEY - depth: 1 - - script: - image: "{{workflow.parameters.dockerhub-registry-proxy}}/alpine/git" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: ssh-key-vol - mountPath: "/etc/ssh-key" - env: - - name: ENV_PATH - value: "{{workflow.parameters.env-path}}" - - name: SOURCE_ENV - value: "{{workflow.parameters.source-env}}" - - name: TARGET_ENV - value: "{{workflow.parameters.target-env}}" - - name: PROMOTE_SETTINGS - value: "{{workflow.parameters.promote-settings}}" - - name: PROMOTE_CONTAINER - value: "{{workflow.parameters.promote-container}}" - - name: PROMOTE_CONFIGMAPS - value: "{{workflow.parameters.promote-configmaps}}" - - name: COMMIT_MESSAGE - value: "{{workflow.parameters.commit-message}}" - command: [sh] - source: | - cd /src - [[ $PROMOTE_CONTAINER == 'true' ]] && cp -v "${ENV_PATH}/${SOURCE_ENV}/version.yaml" "${ENV_PATH}/${TARGET_ENV}" - [[ $PROMOTE_SETTINGS == 'true' ]] && cp -v "${ENV_PATH}/${SOURCE_ENV}/settings.yaml" "${ENV_PATH}/${TARGET_ENV}" - [[ $PROMOTE_CONFIGMAPS == 'true' ]] && cp -v "${ENV_PATH}/${SOURCE_ENV}/cm.yaml" "${ENV_PATH}/${TARGET_ENV}" - mkdir ~/.ssh - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - cp /etc/ssh-key/SSH_PRIVATE_KEY ~/.ssh/id_rsa - ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub - git config --global user.email "" - git config --global user.name "" - git add . && \ - git commit -m "${COMMIT_MESSAGE}" \ - && git push - if [ $? -gt 0 ] - then - echo "Please read error explanation above." - else - echo "Commit and Push successful." - fi diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/trivy-fs-scan-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/trivy-fs-scan-wft.yaml deleted file mode 100644 index 628578d5..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/trivy-fs-scan-wft.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: trivy-fs-scan-wft -#TODO: -# in the end: clean up unpacked-fs -# script for untar and run trivy fs -spec: - templates: - - name: trivy-fs-scan - container: - image: "{{workflow.parameters.dockerhub-registry-proxy}}/aquasec/trivy" - imagePullPolicy: IfNotPresent - args: - - fs - - /build/{{workflow.parameters.wl-service-name}} - volumeMounts: - - name: build - mountPath: /build - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml deleted file mode 100644 index 4598480f..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: tslint-wft -spec: - templates: - - name: tslint - container: - image: "{{workflow.parameters.dockerhub-registry-proxy}}/pipelinecomponents/tslint" - imagePullPolicy: IfNotPresent - command: ["tslint", "-c", "/build/{{workflow.parameters.wl-service-name}}/tslint.json", "/build/{{workflow.parameters.wl-service-name}}/src/**/*.ts"] - volumeMounts: - - name: build - mountPath: /build - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml deleted file mode 100644 index d9489984..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml +++ /dev/null @@ -1,67 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: version-changer-wft - namespace: argo -spec: - entrypoint: version-changer - templates: - - name: version-changer - inputs: - artifacts: - - name: git-src - path: /src - git: - repo: '{{workflow.parameters.gitops-repo}}' - # revision: "{{workflow.parameters.tag}}" - sshPrivateKeySecret: - name: ci-secrets - key: SSH_PRIVATE_KEY - depth: 1 - script: - image: '{{workflow.parameters.dockerhub-registry-proxy}}/alpine/git' - imagePullPolicy: IfNotPresent - volumeMounts: - - name: ssh-key-vol - mountPath: /etc/ssh-key - env: - - name: WL_SERVICE_LIST - value: '{{workflow.parameters.wl-service-list}}' - - name: ENV_PATH - value: '{{workflow.parameters.env-path}}' - - name: ENV_NAME - value: '{{workflow.parameters.env-name}}' - - name: NEW_TAG - value: '{{workflow.parameters.tag}}' - - name: MIRROR_PREFIX - value: '{{workflow.parameters.dockerhub-registry-proxy}}' - command: [sh] - source: |- - cd /src/${ENV_PATH}/${ENV_NAME} - ls -lr - if [ -e version.yaml ] - then - for SVC_NAME in $WL_SERVICE_LIST - do - echo $SVC_NAME - sed -i -r "s/(^\s*image:.+$SVC_NAME).*$/\1:$NEW_TAG\"/" version.yaml - done - cat version.yaml - mkdir ~/.ssh - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - cp /etc/ssh-key/SSH_PRIVATE_KEY ~/.ssh/id_rsa - ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub - ls -l ~/.ssh - git config --global user.email "" - git config --global user.name "" - git add . && \ - git commit -m "Tag updated for $WL_SERVICE_LIST to $NEW_TAG" \ - && git push - fi - if [ $? -gt 0 ] - then - echo "Please read error explanation above." - else - echo "Commit and Push successful." - fi diff --git a/platform/terraform/hosting_provider/output.tf b/platform/terraform/hosting_provider/output.tf index ee09d73e..a38469a9 100644 --- a/platform/terraform/hosting_provider/output.tf +++ b/platform/terraform/hosting_provider/output.tf @@ -90,3 +90,14 @@ output "artifact_storage" { value = module.hosting-provider.artifacts_storage description = "Continuous Integration Artifact Repository storage backend" } + +output "artifact_storage_endpoint" { + value = module.hosting-provider.artifacts_storage_endpoint + description = "Continuous Integration Artifact Repository storage backend access endpoint" +} + +output "artifacts_storage_access_key" { + value = module.hosting-provider.artifacts_storage_access_key + sensitive = true + description = "Continuous Integration Artifact Repository storage account primary access key" +} \ No newline at end of file diff --git a/platform/terraform/modules/cloud_aws/outputs.tf b/platform/terraform/modules/cloud_aws/outputs.tf index 49d6797d..f4d317f4 100644 --- a/platform/terraform/modules/cloud_aws/outputs.tf +++ b/platform/terraform/modules/cloud_aws/outputs.tf @@ -290,6 +290,18 @@ output "artifacts_storage" { value = module.artifacts_repository.s3_bucket_id } +output "artifacts_storage_endpoint" { + description = "The artifact storage S3 bucket domain name" + value = module.artifacts_repository.s3_bucket_bucket_domain_name +} + +# stub value for module compatibility +output "artifacts_storage_access_key" { + value = "" + sensitive = true + description = "Continuous Integration Artifact Repository storage account primary access key" +} + # stub value for module compatibility output "kube_config_raw" { value = "" diff --git a/platform/terraform/modules/cloud_azure/outputs.tf b/platform/terraform/modules/cloud_azure/outputs.tf index 049d2af1..e4d860a8 100644 --- a/platform/terraform/modules/cloud_azure/outputs.tf +++ b/platform/terraform/modules/cloud_azure/outputs.tf @@ -90,6 +90,18 @@ output "artifacts_storage" { description = "Continuous Integration Artifact Repository storage backend" } +output "artifacts_storage_endpoint" { + value = azurerm_storage_account.storage_account.primary_blob_endpoint + description = "Continuous Integration Artifact Repository storage account primary endpoint" +} + +output "artifacts_storage_access_key" { + value = azurerm_storage_account.storage_account.primary_access_key + sensitive = true + description = "Continuous Integration Artifact Repository storage account primary access key" +} + + # stub value for module compatibility output "cluster_oidc_provider_arn" { value = "" diff --git a/platform/terraform/modules/cloud_gcp/output.tf b/platform/terraform/modules/cloud_gcp/output.tf index bb335d8b..0dbb7700 100644 --- a/platform/terraform/modules/cloud_gcp/output.tf +++ b/platform/terraform/modules/cloud_gcp/output.tf @@ -83,6 +83,19 @@ output "artifacts_storage" { description = "Continuous Integration Artifact Repository storage backend" } +output "artifacts_storage_endpoint" { + value = google_storage_bucket.artifacts_repository.url + description = "Continuous Integration Artifact Repository storage account primary endpoint" +} + +# stub value for module compatibility +output "artifacts_storage_access_key" { + value = "" + sensitive = true + description = "Continuous Integration Artifact Repository storage account primary access key" +} + + # stub value for module compatibility output "cluster_oidc_provider_arn" { value = "" diff --git a/platform/terraform/modules/secrets_vault/secrets.tf b/platform/terraform/modules/secrets_vault/secrets.tf index 1dfa1c14..b9560d96 100644 --- a/platform/terraform/modules/secrets_vault/secrets.tf +++ b/platform/terraform/modules/secrets_vault/secrets.tf @@ -1,5 +1,5 @@ locals { - b64_docker_auth = base64encode("robot@main-robot:${random_password.harbor_main_robot_password.result}") + b64_docker_auth = base64encode("robot@main-robot:${random_password.harbor_main_robot_password.result}") harbor_admin_user = "admin" grafana_admin_user = "admin" atlantis_admin_user = "admin" @@ -35,8 +35,9 @@ resource "vault_generic_secret" "ci_secrets" { data_json = jsonencode( { - SSH_PRIVATE_KEY = var.vcs_bot_ssh_private_key, - PERSONAL_ACCESS_TOKEN = var.vcs_token, + SSH_PRIVATE_KEY = var.vcs_bot_ssh_private_key, + PERSONAL_ACCESS_TOKEN = var.vcs_token, + ARTIFACT_STORAGE_ACCESS_KEY = var.cloud_binary_artifacts_store_access_key, } ) @@ -48,7 +49,7 @@ resource "vault_generic_secret" "cd_secrets" { data_json = jsonencode( { - cd_webhook_secret = var.cd_webhook_secret, + cd_webhook_secret = var.cd_webhook_secret, } ) @@ -63,33 +64,34 @@ resource "vault_generic_secret" "atlantis_secrets" { data_json = jsonencode( { - ARGO_SERVER_URL = "argo.argo.svc.cluster.local:2746", + ARGO_SERVER_URL = "argo.argo.svc.cluster.local:2746", # github specific section - ATLANTIS_GH_HOSTNAME = "github.com", - ATLANTIS_GH_TOKEN = var.vcs_token, - ATLANTIS_GH_USER = "", - ATLANTIS_GH_WEBHOOK_SECRET = var.atlantis_repo_webhook_secret, - GITHUB_OWNER = "", - GITHUB_TOKEN = var.vcs_token, + ATLANTIS_GH_HOSTNAME = "github.com", + ATLANTIS_GH_TOKEN = var.vcs_token, + ATLANTIS_GH_USER = "", + ATLANTIS_GH_WEBHOOK_SECRET = var.atlantis_repo_webhook_secret, + GITHUB_OWNER = "", + GITHUB_TOKEN = var.vcs_token, # ---- - TF_VAR_cd_webhook_secret = var.cd_webhook_secret, - TF_VAR_atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret, - TF_VAR_atlantis_repo_webhook_url = var.atlantis_repo_webhook_url, - TF_VAR_vcs_token = var.vcs_token, - TF_VAR_cluster_endpoint = var.cluster_endpoint, - TF_VAR_tf_backend_storage_access_key = var.tf_backend_storage_access_key, - TF_VAR_cluster_ssh_public_key = var.cluster_ssh_public_key, + TF_VAR_cd_webhook_secret = var.cd_webhook_secret, + TF_VAR_atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret, + TF_VAR_atlantis_repo_webhook_url = var.atlantis_repo_webhook_url, + TF_VAR_vcs_token = var.vcs_token, + TF_VAR_cluster_endpoint = var.cluster_endpoint, + TF_VAR_tf_backend_storage_access_key = var.tf_backend_storage_access_key, + TF_VAR_cluster_ssh_public_key = var.cluster_ssh_public_key, + TF_VAR_cloud_binary_artifacts_store_access_key = var.cloud_binary_artifacts_store_access_key, # - TF_VAR_hosted_zone_name = "", - TF_VAR_vcs_bot_ssh_public_key = var.vcs_bot_ssh_public_key, - TF_VAR_vcs_bot_ssh_private_key = var.vcs_bot_ssh_private_key, + TF_VAR_hosted_zone_name = "", + TF_VAR_vcs_bot_ssh_public_key = var.vcs_bot_ssh_public_key, + TF_VAR_vcs_bot_ssh_private_key = var.vcs_bot_ssh_private_key, # harbor specific section - TF_VAR_registry_oidc_client_id = module.harbor.vault_oidc_client_id, - TF_VAR_registry_oidc_client_secret = module.harbor.vault_oidc_client_secret, - TF_VAR_registry_main_robot_password = random_password.harbor_main_robot_password.result, - HARBOR_URL = "https://", - HARBOR_USERNAME = local.harbor_admin_user, - HARBOR_PASSWORD = random_password.harbor_password.result, + TF_VAR_registry_oidc_client_id = module.harbor.vault_oidc_client_id, + TF_VAR_registry_oidc_client_secret = module.harbor.vault_oidc_client_secret, + TF_VAR_registry_main_robot_password = random_password.harbor_main_robot_password.result, + HARBOR_URL = "https://", + HARBOR_USERNAME = local.harbor_admin_user, + HARBOR_PASSWORD = random_password.harbor_password.result, # ---- # vault specific section diff --git a/platform/terraform/modules/secrets_vault/variables.tf b/platform/terraform/modules/secrets_vault/variables.tf index 44a6e4f8..8031f48c 100644 --- a/platform/terraform/modules/secrets_vault/variables.tf +++ b/platform/terraform/modules/secrets_vault/variables.tf @@ -76,3 +76,9 @@ variable "tf_backend_storage_access_key" { type = string default = "" } + +variable "cloud_binary_artifacts_store_access_key" { + description = "Specifies the access key for CI artifact store backend storage. Only applicable to AKS." + type = string + default = "" +} diff --git a/platform/terraform/secrets/main.tf b/platform/terraform/secrets/main.tf index e26a7eb7..c1e31d78 100644 --- a/platform/terraform/secrets/main.tf +++ b/platform/terraform/secrets/main.tf @@ -22,16 +22,17 @@ locals { module "secrets" { source = "../modules/secrets_vault" - cluster_name = local.cluster_name - workloads = var.workloads - vcs_bot_ssh_public_key = var.vcs_bot_ssh_public_key - vcs_bot_ssh_private_key = var.vcs_bot_ssh_private_key - vcs_token = var.vcs_token - atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret - atlantis_repo_webhook_url = var.atlantis_repo_webhook_url - cd_webhook_secret = var.cd_webhook_secret - vault_token = var.vault_token - cluster_endpoint = var.cluster_endpoint - cluster_ssh_public_key = var.cluster_ssh_public_key - tf_backend_storage_access_key = var.tf_backend_storage_access_key + cluster_name = local.cluster_name + workloads = var.workloads + vcs_bot_ssh_public_key = var.vcs_bot_ssh_public_key + vcs_bot_ssh_private_key = var.vcs_bot_ssh_private_key + vcs_token = var.vcs_token + atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret + atlantis_repo_webhook_url = var.atlantis_repo_webhook_url + cd_webhook_secret = var.cd_webhook_secret + vault_token = var.vault_token + cluster_endpoint = var.cluster_endpoint + cluster_ssh_public_key = var.cluster_ssh_public_key + tf_backend_storage_access_key = var.tf_backend_storage_access_key + cloud_binary_artifacts_store_access_key = var.cloud_binary_artifacts_store_access_key } diff --git a/platform/terraform/secrets/variable.tf b/platform/terraform/secrets/variable.tf index 3b143595..e17faa42 100644 --- a/platform/terraform/secrets/variable.tf +++ b/platform/terraform/secrets/variable.tf @@ -69,3 +69,9 @@ variable "tf_backend_storage_access_key" { type = string default = "" } + +variable "cloud_binary_artifacts_store_access_key" { + description = "Specifies the access key for CI artifact store backend storage. Only applicable to AKS." + type = string + default = "" +} diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index fb730ed4..7d9e9b95 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -380,6 +380,8 @@ def setup( # artifact storage p.parameters[""] = hp_out["artifact_storage"] + p.parameters[""] = hp_out["artifact_storage_endpoint"] + p.internals["CLOUD_BINARY_ARTIFACTS_STORE_ACCESS_KEY"] = hp_out["artifacts_storage_access_key"] # kms keys p.parameters[""] = hp_out["secret_manager_unseal_key"] p.parameters[""] = hp_out["secret_manager_unseal_key_ring"] @@ -752,6 +754,9 @@ def setup( if "TF_BACKEND_STORAGE_ACCESS_KEY" in p.internals: sec_man_tf_params["tf_backend_storage_access_key"] = p.internals["TF_BACKEND_STORAGE_ACCESS_KEY"] + if "CLOUD_BINARY_ARTIFACTS_STORE_ACCESS_KEY" in p.internals: + sec_man_tf_params["cloud_binary_artifacts_store_access_key"] = p.internals["CLOUD_BINARY_ARTIFACTS_STORE_ACCESS_KEY"] + tf_wrapper.apply(sec_man_tf_params) sec_man_out = tf_wrapper.output() diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index 4e691bdd..1873d728 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -144,6 +144,7 @@ def bootstrap( owner_email = state_store.parameters[""] ci_iam_role_rn = state_store.parameters[""] artifact_store = state_store.parameters[""] + artifact_store_endpoint = state_store.parameters[""] ci_ingress_url = state_store.parameters[""] click.echo("1/11: Configuration loaded.") @@ -191,6 +192,7 @@ def bootstrap( "": str(wl_svc_port), "# ": cloud_man.create_k8s_cluster_role_mapping_snippet(), "# ": cloud_man.create_additional_labels(), + "# ": cloud_man.create_ci_artifact_store_config_snippet(), "# ": cloud_man.create_iac_backend_snippet( location=tf_backend_storage_name, service=f"workloads/{wl_name}/secrets" @@ -217,7 +219,8 @@ def bootstrap( "": construct_wl_iam_role( state_store.cloud_provider, cloud_account, cluster_name, wl_name, wl_svc_name ), - "": artifact_store + "": artifact_store, + "": artifact_store_endpoint } # set cloud provider specific params diff --git a/tools/cli/services/cloud/aws/aws_manager.py b/tools/cli/services/cloud/aws/aws_manager.py index cde1cae5..bf373640 100644 --- a/tools/cli/services/cloud/aws/aws_manager.py +++ b/tools/cli/services/cloud/aws/aws_manager.py @@ -202,3 +202,18 @@ def get_cloud_provider_k8s_dns_deployment_name(self) -> str: :rtype: str """ return "coredns" + + def create_ci_artifact_store_config_snippet(self) -> str: + """ + Creates Cloud Provider specific configuration section for Argo Workflow artifact storage + :return: Artifact storage configuration section + """ + return textwrap.dedent('''s3: + bucket: + endpoint: s3.amazonaws.com + insecure: false + keyFormat: "{{workflow.parameters.workload-name}}/{{workflow.parameters.tag}}/{{pod.name}}/" + region: + useSDKCreds: true + encryptionOptions: + enableEncryption: false''') diff --git a/tools/cli/services/cloud/azure/azure_manager.py b/tools/cli/services/cloud/azure/azure_manager.py index 65ece45c..cf6b83c6 100644 --- a/tools/cli/services/cloud/azure/azure_manager.py +++ b/tools/cli/services/cloud/azure/azure_manager.py @@ -301,3 +301,17 @@ def get_cloud_provider_k8s_dns_deployment_name(self) -> str: :rtype: str """ return "coredns" + + def create_ci_artifact_store_config_snippet(self) -> str: + """ + Creates Cloud Provider specific configuration section for Argo Workflow artifact storage + :return: Artifact storage configuration section + """ + return textwrap.dedent('''azure: + endpoint: + container: + blobNameFormat: "{{workflow.parameters.workload-name}}/{{workflow.parameters.tag}}/{{pod.name}}/" + # useSDKCreds: true + accountKeySecret: + name: ci-secrets + key: ARTIFACT_STORAGE_ACCESS_KEY''') diff --git a/tools/cli/services/cloud/cloud_provider_manager.py b/tools/cli/services/cloud/cloud_provider_manager.py index 4af31d68..a6bf7eaf 100644 --- a/tools/cli/services/cloud/cloud_provider_manager.py +++ b/tools/cli/services/cloud/cloud_provider_manager.py @@ -191,3 +191,11 @@ def get_cloud_provider_k8s_dns_deployment_name(self) -> str: :rtype: str """ pass + + @abstractmethod + def create_ci_artifact_store_config_snippet(self) -> str: + """ + Creates Cloud Provider specific configuration section for Argo Workflow artifact storage + :return: Artifact storage configuration section + """ + pass diff --git a/tools/cli/services/cloud/gcp/gcp_manager.py b/tools/cli/services/cloud/gcp/gcp_manager.py index ab606552..76e3e575 100644 --- a/tools/cli/services/cloud/gcp/gcp_manager.py +++ b/tools/cli/services/cloud/gcp/gcp_manager.py @@ -357,3 +357,10 @@ def get_cloud_provider_k8s_dns_deployment_name(self) -> str: :rtype: str """ return "kube-dns" + + def create_ci_artifact_store_config_snippet(self) -> str: + """ + Creates Cloud Provider specific configuration section for Argo Workflow artifact storage + :return: Artifact storage configuration section + """ + return textwrap.dedent('''''')