From 93c31c46fdfa10257c3965d49a435a49fb653465 Mon Sep 17 00:00:00 2001 From: Alex <101551092+alexmance@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:13:19 +0100 Subject: [PATCH] simplify ipfs process (#2241) --- .github/actions/dev-ipfs-pin/action.yml | 95 ---------------- .github/actions/ipfs-pin/action.yml | 51 +++++++++ .../actions/production-ipfs-pin/action.yml | 100 ----------------- .github/workflows/build-test-deploy-dev.yml | 5 +- .github/workflows/test-deploy-fork.yml | 5 +- .github/workflows/update-prod-staging.yml | 103 ++++++++---------- 6 files changed, 96 insertions(+), 263 deletions(-) delete mode 100644 .github/actions/dev-ipfs-pin/action.yml create mode 100644 .github/actions/ipfs-pin/action.yml delete mode 100644 .github/actions/production-ipfs-pin/action.yml diff --git a/.github/actions/dev-ipfs-pin/action.yml b/.github/actions/dev-ipfs-pin/action.yml deleted file mode 100644 index 661a793d78..0000000000 --- a/.github/actions/dev-ipfs-pin/action.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Pin to ipfs cluster and nodes -description: Upload build to ipfs node -inputs: - BUILD_PATH: - description: path to the build directory - required: true - DOMAIN: - description: Domain to unpin - required: true - GCP_SA_KEY: - description: '' - required: true - GKE_CLUSTER: - description: '' - required: true - GKE_CLUSTER_REGION: - description: '' - required: true - PINATA_HASH: - description: '' - required: true - IPFS_SERVER_PORT: - description: '' - required: true - IPFS_SERVER: - description: '' - required: true - IPFS_SERVER_KEY: - description: '' - required: true - -runs: - using: 'composite' - steps: - - name: GCP Auth - uses: google-github-actions/auth@ceee102ec2387dd9e844e01b530ccd4ec87ce955 # v0.8.0 - with: - credentials_json: '${{ inputs.GCP_SA_KEY }}' - - name: Get GKE credentials - uses: google-github-actions/get-gke-credentials@054fdb05e32039f72764f03e69e6acb20caa6f56 # v0.8.0 - with: - cluster_name: '${{ inputs.GKE_CLUSTER }}' - location: '${{ inputs.GKE_CLUSTER_REGION }}' - - uses: azure/setup-kubectl@7f7e5ba5ea3e491b92e6e8e5819963f8f3a1f076 # v3 - with: - version: 'v1.22.13' - - - name: GKE ipfs cluster pin - shell: bash - run: | - export POD_NAME="ipfs-cluster-$(($RANDOM % 3))" - tar -czf app.tar.gz './${{ inputs.BUILD_PATH }}' - kubectl -n ipfs-cluster cp -c ipfs ./app.tar.gz ${POD_NAME}:/root/ - - echo "Uploading build..." - IPFS_HASH="$(kubectl -n ipfs-cluster exec -i ${POD_NAME} -c ipfs -- sh -c \ - 'cd /root/ && tar -zxf app.tar.gz && \ - ipfs add -rQ --cid-version=1 --pin=false ./${{ inputs.BUILD_PATH }}/ && \ - rm -rf ./app.tar.gz ./${{ inputs.BUILD_PATH }}')" - - echo "Pinning to cluster..." - kubectl -n ipfs-cluster exec -i ${POD_NAME} -c ipfs-cluster -- \ - ipfs-cluster-ctl pin add --wait --wait-timeout=220s \ - --replication-min=1 --name='app-aave-${{ github.sha }}' \ - "${IPFS_HASH}" - - - name: Prepare env and Compress build on IPFS servers - id: ipfs-prepare - continue-on-error: false - shell: bash - run: | - rm -rf ./key - echo -n "${{ inputs.IPFS_SERVER_KEY }}" | base64 -d >> ./key - chmod 400 ./key - - - name: Upload build to IPFS Primary Server - id: ipfs-scp-primary - continue-on-error: false - shell: bash - run: | - scp -o StrictHostKeyChecking=no \ - -i ./key \ - -P \${{ inputs.IPFS_SERVER_PORT }} app.tar.gz ipfs@\${{ inputs.IPFS_SERVER }}:. - ssh -o StrictHostKeyChecking=no \ - -p \${{ inputs.IPFS_SERVER_PORT }} \ - -i ./key ipfs@\${{ inputs.IPFS_SERVER }} "tar zxf app.tar.gz && ipfs add -rQ --cid-version=1 ./${{ inputs.BUILD_PATH }}/" - - - name: Clean build and tmp directory on Servers - continue-on-error: false - id: ipfs-clean - shell: bash - run: | - ssh -o StrictHostKeyChecking=no \ - -p \${{ inputs.IPFS_SERVER_PORT }} \ - -i ./key ipfs@\${{ inputs.IPFS_SERVER }} "rm -rf ./${{ inputs.BUILD_PATH }}/ app.tar.gz" diff --git a/.github/actions/ipfs-pin/action.yml b/.github/actions/ipfs-pin/action.yml new file mode 100644 index 0000000000..ee94fe5164 --- /dev/null +++ b/.github/actions/ipfs-pin/action.yml @@ -0,0 +1,51 @@ +name: Pin to ipfs cluster and nodes +description: Upload build to ipfs node +inputs: + BUILD_PATH: + description: path to the build directory + required: true + PINATA_HASH: + description: '' + required: true + IPFS_SERVER_PORT: + description: '' + required: true + IPFS_SERVER: + description: '' + required: true + IPFS_SERVER_KEY: + description: '' + required: true + +runs: + using: 'composite' + steps: + - name: Prepare env and Compress build on IPFS servers + id: ipfs-prepare + continue-on-error: false + shell: bash + run: | + tar -czf app.tar.gz './${{ inputs.BUILD_PATH }}' + rm -rf ./key + echo -n "${{ inputs.IPFS_SERVER_KEY }}" | base64 -d >> ./key && chmod 400 ./key + + - name: Upload build to IPFS Primary Server + id: ipfs-scp-primary + continue-on-error: false + shell: bash + run: | + scp -o StrictHostKeyChecking=no \ + -i ./key \ + -P \${{ inputs.IPFS_SERVER_PORT }} app.tar.gz ipfs@\${{ inputs.IPFS_SERVER }}:. + ssh -o StrictHostKeyChecking=no \ + -p \${{ inputs.IPFS_SERVER_PORT }} \ + -i ./key ipfs@\${{ inputs.IPFS_SERVER }} "tar zxf app.tar.gz && ipfs add -rQ --cid-version=1 ./${{ inputs.BUILD_PATH }}/" + + - name: Clean build and tmp directory on Servers + continue-on-error: false + id: ipfs-clean + shell: bash + run: | + ssh -o StrictHostKeyChecking=no \ + -p \${{ inputs.IPFS_SERVER_PORT }} \ + -i ./key ipfs@\${{ inputs.IPFS_SERVER }} "rm -rf ./${{ inputs.BUILD_PATH }}/ app.tar.gz" diff --git a/.github/actions/production-ipfs-pin/action.yml b/.github/actions/production-ipfs-pin/action.yml deleted file mode 100644 index eef8886fde..0000000000 --- a/.github/actions/production-ipfs-pin/action.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: Pin to private cluster -description: Upload build to ipfs nodes running in GKE -inputs: - BUILD_PATH: - description: path to the build directory - required: true - DOMAIN: - description: Domain to unpin - required: true - GCP_SA_KEY: - description: '' - required: true - GKE_CLUSTER: - description: '' - required: true - GKE_CLUSTER_REGION: - description: '' - required: true - PINATA_HASH: - description: '' - required: true - IPFS_SERVER_PORT: - description: '' - required: true - IPFS_SERVER: - description: '' - required: true - IPFS_SERVER_KEY: - description: '' - required: true - -runs: - using: 'composite' - steps: - - name: GCP Auth - uses: google-github-actions/auth@ceee102ec2387dd9e844e01b530ccd4ec87ce955 # v0.8.0 - with: - credentials_json: '${{ inputs.GCP_SA_KEY }}' - - name: Get GKE credentials - uses: google-github-actions/get-gke-credentials@054fdb05e32039f72764f03e69e6acb20caa6f56 # v0.8.0 - with: - cluster_name: '${{ inputs.GKE_CLUSTER }}' - location: '${{ inputs.GKE_CLUSTER_REGION }}' - - uses: azure/setup-kubectl@7f7e5ba5ea3e491b92e6e8e5819963f8f3a1f076 # v3 - with: - version: 'v1.22.13' - - - name: ipfs cluster pin - shell: bash - run: | - export POD_NAME="ipfs-cluster-$(($RANDOM % 3))" - tar -czf app.tar.gz './${{ inputs.BUILD_PATH }}' - kubectl -n ipfs-cluster cp -c ipfs ./app.tar.gz ${POD_NAME}:/root/ - - echo "Uploading build..." - IPFS_HASH="$(kubectl -n ipfs-cluster exec -i ${POD_NAME} -c ipfs -- sh -c \ - 'cd /root/ && tar -zxf app.tar.gz && \ - ipfs add -rQ --cid-version=1 --pin=false ./${{ inputs.BUILD_PATH }}/ && \ - rm -rf ./app.tar.gz ./${{ inputs.BUILD_PATH }}')" - - test "${IPFS_HASH}" = '${{ inputs.PINATA_HASH }}' || \ - { echo -e "Hashes differ, aborting pinning\nPINATA: ${{ inputs.PINATA_HASH }}\nLOCAL: ${IPFS_HASH}" && exit 1; } - - echo "Pinning to cluster..." - kubectl -n ipfs-cluster exec -i ${POD_NAME} -c ipfs-cluster -- \ - ipfs-cluster-ctl pin rm '/ipns/${{ inputs.DOMAIN }}' || true - kubectl -n ipfs-cluster exec -i ${POD_NAME} -c ipfs-cluster -- \ - ipfs-cluster-ctl pin add --wait --wait-timeout=220s \ - --replication-min=1 --name='app-aave-${{ github.sha }}' \ - "${IPFS_HASH}" - - - name: Prepare env and Compress build on IPFS servers - id: ipfs-prepare - continue-on-error: false - shell: bash - run: | - rm -rf ./key - echo -n "${{ inputs.IPFS_SERVER_KEY }}" | base64 -d >> ./key - chmod 400 ./key - - - name: Upload build to IPFS Primary Server - id: ipfs-scp-primary - continue-on-error: false - shell: bash - run: | - scp -o StrictHostKeyChecking=no \ - -i ./key \ - -P \${{ inputs.IPFS_SERVER_PORT }} app.tar.gz ipfs@\${{ inputs.IPFS_SERVER }}:. - ssh -o StrictHostKeyChecking=no \ - -p \${{ inputs.IPFS_SERVER_PORT }} \ - -i ./key ipfs@\${{ inputs.IPFS_SERVER }} "tar zxf app.tar.gz && ipfs add -rQ --cid-version=1 ./${{ inputs.BUILD_PATH }}/" - - - name: Clean build and tmp directory on Servers - continue-on-error: false - id: ipfs-clean - shell: bash - run: | - ssh -o StrictHostKeyChecking=no \ - -p \${{ inputs.IPFS_SERVER_PORT }} \ - -i ./key ipfs@\${{ inputs.IPFS_SERVER }} "rm -rf ./${{ inputs.BUILD_PATH }}/ app.tar.gz" diff --git a/.github/workflows/build-test-deploy-dev.yml b/.github/workflows/build-test-deploy-dev.yml index b968af9eb7..303e0c57c1 100644 --- a/.github/workflows/build-test-deploy-dev.yml +++ b/.github/workflows/build-test-deploy-dev.yml @@ -127,11 +127,8 @@ jobs: - name: Pin to primary IPFS server id: ipfs-primary - uses: ./.github/actions/dev-ipfs-pin + uses: ./.github/actions/ipfs-pin with: - GCP_SA_KEY: '${{ secrets.GCP_SA_KEY }}' - GKE_CLUSTER: '${{ secrets.GKE_CLUSTER }}' - GKE_CLUSTER_REGION: '${{ secrets.GKE_CLUSTER_REGION }}' IPFS_SERVER_PORT: '${{ secrets.IPFS_PRIMARY_SERVER_PORT }}' IPFS_SERVER: '${{ secrets.IPFS_PRIMARY_SERVER }}' IPFS_SERVER_KEY: '${{ secrets.IPFS_PRIMARY_SERVER_KEY }}' diff --git a/.github/workflows/test-deploy-fork.yml b/.github/workflows/test-deploy-fork.yml index 03e876f976..874afe2830 100644 --- a/.github/workflows/test-deploy-fork.yml +++ b/.github/workflows/test-deploy-fork.yml @@ -138,11 +138,8 @@ jobs: - name: Pin to primary IPFS server id: ipfs-primary - uses: ./.github/actions/dev-ipfs-pin + uses: ./.github/actions/ipfs-pin with: - GCP_SA_KEY: '${{ secrets.GCP_SA_KEY }}' - GKE_CLUSTER: '${{ secrets.GKE_CLUSTER }}' - GKE_CLUSTER_REGION: '${{ secrets.GKE_CLUSTER_REGION }}' IPFS_SERVER_PORT: '${{ secrets.IPFS_PRIMARY_SERVER_PORT }}' IPFS_SERVER: '${{ secrets.IPFS_PRIMARY_SERVER }}' IPFS_SERVER_KEY: '${{ secrets.IPFS_PRIMARY_SERVER_KEY }}' diff --git a/.github/workflows/update-prod-staging.yml b/.github/workflows/update-prod-staging.yml index 527e28c64f..202118e49f 100644 --- a/.github/workflows/update-prod-staging.yml +++ b/.github/workflows/update-prod-staging.yml @@ -9,97 +9,80 @@ on: required: true jobs: - production: + deploy: runs-on: ubuntu-latest + strategy: + matrix: + environment: [ + { + name: 'Production', + domain: 'app.aave.com', + artifact: 'out', + build_path: 'out' + }, + { + name: 'Staging', + domain: 'staging.aave.com', + artifact: 'out_staging', + build_path: 'out_staging' + } + ] environment: - name: Production - url: 'https://app.aave.com' + name: ${{ matrix.environment.name }} + url: https://${{ matrix.environment.domain }} steps: - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2 - name: download build uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741 # v3.0.0 with: - name: out - path: out + name: ${{ matrix.environment.artifact }} + path: ${{ matrix.environment.build_path }} + + # Only for staging environment + - name: pinata + id: pinata + if: matrix.environment.name == 'Staging' + uses: aave/pinata-action@35662944e92cf082457973f10aa42e48df179b47 + with: + PINATA_API_KEY: '${{ secrets.PINATA_API_KEY }}' + PINATA_SECRET_KEY: '${{ secrets.PINATA_SECRET_KEY }}' + PIN_ALIAS: 'app-aave-staging' + BUILD_LOCATION: './out_staging' + CID_VERSION: 1 - name: Pin to primary IPFS server id: ipfs-primary - uses: ./.github/actions/production-ipfs-pin + uses: ./.github/actions/ipfs-pin with: - DOMAIN: app.aave.com - GCP_SA_KEY: '${{ secrets.GCP_SA_KEY }}' - GKE_CLUSTER: '${{ secrets.GKE_CLUSTER }}' - GKE_CLUSTER_REGION: '${{ secrets.GKE_CLUSTER_REGION }}' + DOMAIN: ${{ matrix.environment.domain }} IPFS_SERVER_PORT: '${{ secrets.IPFS_PRIMARY_SERVER_PORT }}' IPFS_SERVER: '${{ secrets.IPFS_PRIMARY_SERVER }}' IPFS_SERVER_KEY: '${{ secrets.IPFS_PRIMARY_SERVER_KEY }}' - BUILD_PATH: 'out' - PINATA_HASH: '${{ inputs.PINATA_HASH }}' + BUILD_PATH: ${{ matrix.environment.build_path }} + PINATA_HASH: ${{ matrix.environment.name == 'Production' && inputs.PINATA_HASH || steps.pinata.outputs.hash }} - uses: aave/cloudflare-update-action@5c1b528c9c6e0aed18a7dbdd7f957e0b8815a75e with: CF_API_TOKEN: '${{ secrets.CF_API_TOKEN }}' CF_ZONE_ID: '${{ secrets.CF_ZONE_ID }}' - HASH: '${{ inputs.PINATA_HASH }}' - CF_DEPLOYMENT_DOMAIN: app.aave.com + HASH: ${{ matrix.environment.name == 'Production' && inputs.PINATA_HASH || steps.pinata.outputs.hash }} + CF_DEPLOYMENT_DOMAIN: ${{ matrix.environment.domain }} + # Only for production environment - name: prepare release + if: matrix.environment.name == 'Production' run: | cp .github/release-template.md ./release-notes.md sed -i 's||${{ inputs.PINATA_HASH }}|g' ./release-notes.md echo "TAG=release-$(date '+%Y-%m-%d_%H-%M')" >> ${GITHUB_ENV} - name: Create GH release + if: matrix.environment.name == 'Production' uses: ncipollo/release-action@4c75f0f2e4ae5f3c807cf0904605408e319dcaac # v1.10.0 with: name: Production release artifacts: app.tar.gz bodyFile: release-notes.md commit: '${{ github.sha }}' - tag: '${{ env.TAG }}' - - staging: - runs-on: ubuntu-latest - environment: - name: Staging - url: 'https://staging.aave.com' - steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2 - - - name: download build - uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741 # v3.0.0 - with: - name: out_staging - path: out_staging - - - name: pinata - id: pinata - uses: aave/pinata-action@35662944e92cf082457973f10aa42e48df179b47 - with: - PINATA_API_KEY: '${{ secrets.PINATA_API_KEY }}' - PINATA_SECRET_KEY: '${{ secrets.PINATA_SECRET_KEY }}' - PIN_ALIAS: 'app-aave-staging' - BUILD_LOCATION: './out_staging' - CID_VERSION: 1 - - - name: Pin to primary IPFS server - id: ipfs-primary - uses: ./.github/actions/production-ipfs-pin - with: - DOMAIN: staging.aave.com - GCP_SA_KEY: '${{ secrets.GCP_SA_KEY }}' - GKE_CLUSTER: '${{ secrets.GKE_CLUSTER }}' - GKE_CLUSTER_REGION: '${{ secrets.GKE_CLUSTER_REGION }}' - IPFS_SERVER_PORT: '${{ secrets.IPFS_PRIMARY_SERVER_PORT }}' - IPFS_SERVER: '${{ secrets.IPFS_PRIMARY_SERVER }}' - IPFS_SERVER_KEY: '${{ secrets.IPFS_PRIMARY_SERVER_KEY }}' - BUILD_PATH: 'out_staging' - PINATA_HASH: '${{ steps.pinata.outputs.hash }}' - - - uses: aave/cloudflare-update-action@8b9720cbb79a5b824e8e2e618df8955dd9c3a93c - with: - CF_API_TOKEN: '${{ secrets.CF_API_TOKEN }}' - CF_ZONE_ID: '${{ secrets.CF_ZONE_ID }}' - HASH: '${{ steps.pinata.outputs.hash }}' - CF_DEPLOYMENT_DOMAIN: staging.aave.com + tag: '${{ env.TAG }}' \ No newline at end of file