More compute for fw2 (#3117) #1370
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-License-Identifier: Apache-2.0 | |
| # Copyright 2022 The HuggingFace Authors. | |
| name: Continuous deployment | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| repository-prefix: huggingface/datasets-server- | |
| jobs: | |
| build-and-push-images: | |
| name: Build and push docker images to public Docker Hub | |
| strategy: | |
| matrix: | |
| include: | |
| - directory: jobs | |
| project: mongodb_migration | |
| - directory: jobs | |
| project: cache_maintenance | |
| - directory: services | |
| project: admin | |
| - directory: services | |
| project: api | |
| - directory: services | |
| project: rows | |
| - directory: services | |
| project: search | |
| - directory: services | |
| project: sse-api | |
| - directory: services | |
| project: worker | |
| - directory: services | |
| project: webhook | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Set outputs | |
| id: vars | |
| run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.repository-prefix }}${{ matrix.directory }}-${{ matrix.project }} | |
| tags: | | |
| type=raw,value=sha-${{ steps.vars.outputs.sha_short }} | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ${{ matrix.directory }}/${{ matrix.project }}/Dockerfile | |
| build-args: COMMIT=${{ steps.vars.outputs.sha_short }} | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| # see https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#registry-cache | |
| cache-from: type=registry,ref=${{ env.repository-prefix }}${{ matrix.directory }}-${{ matrix.project }}:buildcache | |
| cache-to: type=registry,ref=${{ env.repository-prefix }}${{ matrix.directory }}-${{ matrix.project }}:buildcache,mode=max | |
| code-quality-helm: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Update dependencies | |
| run: helm dependencies update | |
| working-directory: chart | |
| - name: Lint chart with default values | |
| run: helm lint | |
| working-directory: chart | |
| - name: Lint chart with staging values | |
| run: helm lint --values env/staging.yaml | |
| working-directory: chart | |
| - name: Lint chart with prod values | |
| run: helm lint --values env/prod.yaml | |
| working-directory: chart | |
| deploy-staging-and-prod: | |
| if: ${{ endsWith(github.ref, '/main') }} | |
| runs-on: ubuntu-latest | |
| needs: [build-and-push-images] | |
| environment: production | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Set outputs | |
| id: vars | |
| run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| - name: Gen values | |
| run: | | |
| VALUES=$(cat <<-END | |
| images: | |
| jobs: | |
| mongodbMigration: | |
| tag: sha-${{ steps.vars.outputs.sha_short }} | |
| cacheMaintenance: | |
| tag: sha-${{ steps.vars.outputs.sha_short }} | |
| services: | |
| admin: | |
| tag: sha-${{ steps.vars.outputs.sha_short }} | |
| api: | |
| tag: sha-${{ steps.vars.outputs.sha_short }} | |
| rows: | |
| tag: sha-${{ steps.vars.outputs.sha_short }} | |
| search: | |
| tag: sha-${{ steps.vars.outputs.sha_short }} | |
| sseApi: | |
| tag: sha-${{ steps.vars.outputs.sha_short }} | |
| worker: | |
| tag: sha-${{ steps.vars.outputs.sha_short }} | |
| webhook: | |
| tag: sha-${{ steps.vars.outputs.sha_short }} | |
| END | |
| ) | |
| echo "VALUES=$(echo "$VALUES" | yq -o=json | jq tostring)" >> $GITHUB_ENV | |
| - name: Deploy on infra-deployments | |
| uses: aurelien-baudet/workflow-dispatch@v2 | |
| with: | |
| workflow: Update application values | |
| repo: huggingface/infra-deployments | |
| wait-for-completion: true | |
| ref: refs/heads/main | |
| token: ${{ secrets.GIT_TOKEN_INFRA_DEPLOYMENT }} | |
| inputs: '{"path": "datasets-server/*.yaml", "values": ${{ env.VALUES }}, "url": "${{ github.event.head_commit.url }}"}' |