diff --git a/.devcontainer/devcontainer.env b/.devcontainer/devcontainer.env index 77294460..5af03832 100644 --- a/.devcontainer/devcontainer.env +++ b/.devcontainer/devcontainer.env @@ -2,6 +2,8 @@ PORT_PREFIX=${PORT_PREFIX} CONTAINER_PREFIX=${USER} AZURE_API_ENDPOINT=${AZURE_API_ENDPOINT} AZURE_GPT35_MODEL=${AZURE_GPT35_MODEL} +AZURE_GPT4_MODEL=${AZURE_GPT4_MODEL} +AZURE_GPT4o_MODEL=${AZURE_GPT4o_MODEL} AZURE_API_VERSION=${AZURE_API_VERSION} ADMIN_EMAILS=${ADMIN_EMAILS} GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID} @@ -12,3 +14,6 @@ ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} AZURE_OPENAI_API_KEY=${AZURE_OPENAI_API_KEY} OPENAI_API_KEY=${OPENAI_API_KEY} TOGETHER_API_KEY=${TOGETHER_API_KEY} + +# BING key +BING_API_KEY=${BING_API_KEY} diff --git a/.github/workflows/check_certs_expiry.yml b/.github/workflows/check-certs-expiry.yml similarity index 97% rename from .github/workflows/check_certs_expiry.yml rename to .github/workflows/check-certs-expiry.yml index d22b3136..3f5adb25 100644 --- a/.github/workflows/check_certs_expiry.yml +++ b/.github/workflows/check-certs-expiry.yml @@ -55,6 +55,6 @@ jobs: - run: chmod 600 key.pem - run: ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$DOMAIN" "ls -la" - - run: bash ./scripts/ci_check_certs.sh + - run: bash ./scripts/ci-check-certs.sh - run: rm key.pem diff --git a/.github/workflows/deploy-nats.yaml b/.github/workflows/deploy-nats.yaml index 8a141ab1..8a113e79 100644 --- a/.github/workflows/deploy-nats.yaml +++ b/.github/workflows/deploy-nats.yaml @@ -67,6 +67,6 @@ jobs: - run: chmod 600 key.pem - run: ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$DOMAIN" "docker images" - - run: bash scripts/deploy_nats.sh + - run: bash scripts/deploy-nats.sh - run: rm key.pem diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 00000000..bdedf651 --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,246 @@ +name: Deploy + +on: + push: + branches: + - main + - dev + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + detect-deployment-environment: + runs-on: ubuntu-latest + outputs: + environment: ${{ steps.set-env.outputs.environment }} + steps: + - name: Determine deployment environment + id: set-env + run: | + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + echo "environment=production" >> $GITHUB_OUTPUT + elif [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then + echo "environment=staging" >> $GITHUB_OUTPUT + else + echo "environment=none" >> $GITHUB_OUTPUT + fi + + docker-build-push-node: + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + env: + PORT: ${{ vars.PORT }} + steps: + - name: Checkout repository with cached git lfs + uses: nschloe/action-cached-lfs-checkout@v1 + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install wasp + run: curl -sSL https://get.wasp-lang.dev/installer.sh | sh + + - name: Temporary wasp fix + run: | + PATCH_FILE_PATH=$(cat $(whereis wasp | cut -d " " -f 2) | tail -1 | cut -d " " -f 1 | cut -d "=" -f 2)/Generator/templates/server/package.json + echo $PATCH_FILE_PATH + sed -i 's/"postinstall": "patch-package"/"postinstall": ""/' $PATCH_FILE_PATH + + - name: Log in to the Container registry + uses: docker/login-action@v3.2.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - run: docker pull ghcr.io/$GITHUB_REPOSITORY-node:$GITHUB_REF_NAME || docker pull ghcr.io/$GITHUB_REPOSITORY-node:dev || true + - name: Build wasp + run: cd app && wasp build + - run: docker build --build-arg PORT=$PORT -t ghcr.io/$GITHUB_REPOSITORY-node:${GITHUB_REF_NAME////-} ./app/.wasp/build/ + - name: Add tag latest if branch is main + if: github.ref_name == 'main' + run: docker tag ghcr.io/$GITHUB_REPOSITORY-node:$GITHUB_REF_NAME ghcr.io/$GITHUB_REPOSITORY-node:latest + - name: Push only if branch name is main or dev + if: github.ref_name == 'main' || github.ref_name == 'dev' + run: docker push ghcr.io/$GITHUB_REPOSITORY-node --all-tags + + docker-build-push-fastapi: + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Install wasp + run: curl -sSL https://get.wasp-lang.dev/installer.sh | sh + + - name: Log in to the Container registry + uses: docker/login-action@v3.2.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - run: docker pull ghcr.io/$GITHUB_REPOSITORY:$GITHUB_REF_NAME || docker pull ghcr.io/$GITHUB_REPOSITORY:dev || true + - run: docker build --build-arg PORT=$PORT -t ghcr.io/$GITHUB_REPOSITORY:${GITHUB_REF_NAME////-} . + - name: Add tag latest if branch is main + if: github.ref_name == 'main' + run: docker tag ghcr.io/$GITHUB_REPOSITORY:$GITHUB_REF_NAME ghcr.io/$GITHUB_REPOSITORY:latest + - name: Push only if branch name is main or dev + if: github.ref_name == 'main' || github.ref_name == 'dev' + run: docker push ghcr.io/$GITHUB_REPOSITORY --all-tags + + deploy-fastapi: + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash + needs: + - docker-build-push-fastapi + - detect-deployment-environment + environment: + name: ${{ needs.detect-deployment-environment.outputs.environment }} + env: + GITHUB_USERNAME: ${{ github.actor }} + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + DEVELOPER_TOKEN: ${{ secrets.DEVELOPER_TOKEN }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + PY_DATABASE_URL: ${{ secrets.PY_DATABASE_URL }} + FASTAGENCY_SERVER_URL: ${{ vars.FASTAGENCY_SERVER_URL }} + DOMAIN: ${{ vars.DOMAIN }} + SSH_KEY: ${{ secrets.SSH_KEY }} + AZURE_API_VERSION: ${{ vars.AZURE_API_VERSION }} + AZURE_API_ENDPOINT: ${{ vars.AZURE_API_ENDPOINT }} + AZURE_GPT35_MODEL: ${{ vars.AZURE_GPT35_MODEL }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + + steps: + - uses: actions/checkout@v3 # Don't change it to cheackout@v4. V4 is not working with container image. + # This is to fix GIT not liking owner of the checkout dir - https://github.com/actions/runner/issues/2033#issuecomment-1204205989 + - run: chown -R $(id -u):$(id -g) $PWD + + - run: if [[ $GITHUB_REF_NAME == "main" ]]; then echo "TAG=latest" >> $GITHUB_ENV ; else echo "TAG=dev" >> $GITHUB_ENV ; fi; + + - run: echo "PATH=$PATH:/github/home/.local/bin" >> $GITHUB_ENV + - run: "which ssh-agent || ( apt-get update -y && apt-get install openssh-client git gettext -y )" + - run: eval $(ssh-agent -s) + - run: mkdir -p ~/.ssh + - run: chmod 700 ~/.ssh + - run: ssh-keyscan "$DOMAIN" >> ~/.ssh/known_hosts + - run: chmod 644 ~/.ssh/known_hosts + - run: echo "$SSH_KEY" | base64 --decode > key.pem + - run: chmod 600 key.pem + + - run: ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$DOMAIN" "docker images" + - run: bash scripts/deploy.sh + + - run: rm key.pem + + deploy-node: + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash + needs: + - docker-build-push-node + - detect-deployment-environment + environment: + name: ${{ needs.detect-deployment-environment.outputs.environment }} + env: + GITHUB_USERNAME: ${{ github.actor }} + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + PORT: ${{ vars.PORT }} + GOOGLE_CLIENT_ID: ${{ vars.GOOGLE_CLIENT_ID }} + GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} + ADMIN_EMAILS: ${{ vars.ADMIN_EMAILS }} + WASP_SERVER_URL: ${{ vars.WASP_SERVER_URL }} + FASTAGENCY_SERVER_URL: ${{ vars.FASTAGENCY_SERVER_URL }} + NODE_DOMAIN: ${{ vars.NODE_DOMAIN }} + WASP_WEB_CLIENT_URL: ${{ vars.WASP_WEB_CLIENT_URL }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + REACT_APP_API_URL: ${{ vars.REACT_APP_API_URL }} + JWT_SECRET: ${{ secrets.JWT_SECRET }} + SSH_KEY: ${{ secrets.SSH_KEY }} + steps: + - name: Checkout repository with cached git lfs + uses: nschloe/action-cached-lfs-checkout@v1 + # This is to fix GIT not liking owner of the checkout dir - https://github.com/actions/runner/issues/2033#issuecomment-1204205989 + - run: chown -R $(id -u):$(id -g) $PWD + + - run: if [[ $GITHUB_REF_NAME == "main" ]]; then echo "TAG=latest" >> $GITHUB_ENV ; else echo "TAG=dev" >> $GITHUB_ENV ; fi; + + - run: echo "PATH=$PATH:/github/home/.local/bin" >> $GITHUB_ENV + - run: "which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )" + - run: eval $(ssh-agent -s) + - run: mkdir -p ~/.ssh + - run: chmod 700 ~/.ssh + - run: ssh-keyscan "$NODE_DOMAIN" >> ~/.ssh/known_hosts + - run: chmod 644 ~/.ssh/known_hosts + - run: echo "$SSH_KEY" | base64 --decode > key.pem + - run: chmod 600 key.pem + + - run: ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$NODE_DOMAIN" "docker images" + - run: bash scripts/deploy-node.sh + + - run: rm key.pem + + deploy-frontend: + runs-on: ubuntu-22.04 + permissions: + contents: write + needs: + - deploy-fastapi + - deploy-node + - detect-deployment-environment + environment: + name: ${{ needs.detect-deployment-environment.outputs.environment }} + env: + NODE_DOMAIN: ${{ vars.NODE_DOMAIN }} + SSH_KEY: ${{ secrets.SSH_KEY }} + REACT_APP_API_URL: ${{ vars.REACT_APP_API_URL }} + steps: + - name: Checkout repository with cached git lfs + uses: nschloe/action-cached-lfs-checkout@v1 + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install wasp + run: curl -sSL https://get.wasp-lang.dev/installer.sh | sh + + - name: Temporary wasp fix + run: | + PATCH_FILE_PATH=$(cat $(whereis wasp | cut -d " " -f 2) | tail -1 | cut -d " " -f 1 | cut -d "=" -f 2)/Generator/templates/server/package.json + echo $PATCH_FILE_PATH + sed -i 's/"postinstall": "patch-package"/"postinstall": ""/' $PATCH_FILE_PATH + + - name: Build wasp + run: cd app && wasp build + - name: Build frontend + run: cd app && cd .wasp/build/web-app && npm install && REACT_APP_API_URL=$REACT_APP_API_URL npm run build + - name: Copy 404.html + run: cp 404.html app/.wasp/build/web-app/build + + - name: Deploy UI to nginx directory + run: | + apt-get update -y && apt-get install openssh-client git -y + eval $(ssh-agent -s) + mkdir -p ~/.ssh + chmod 700 ~/.ssh + ssh-keyscan "$NODE_DOMAIN" >> ~/.ssh/known_hosts + chmod 644 ~/.ssh/known_hosts + echo "$SSH_KEY" | base64 --decode > key.pem + chmod 600 key.pem + ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$NODE_DOMAIN" "ls -lah /var/www/html/UI" + scp -i key.pem -r app/.wasp/build/web-app/build azureuser@"$NODE_DOMAIN":/var/www/html/UI + ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$NODE_DOMAIN" "ls -lah /var/www/html/UI" + rm key.pem diff --git a/.github/workflows/docker_cleanup.yml b/.github/workflows/docker-cleanup.yml similarity index 100% rename from .github/workflows/docker_cleanup.yml rename to .github/workflows/docker-cleanup.yml diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 6ab2ddd2..1b8b9588 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -2,7 +2,9 @@ name: Pipeline on: push: - merge_group: + branches-ignore: + # - main + - dev workflow_dispatch: env: @@ -10,22 +12,7 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - detect-deployment-environment: - runs-on: ubuntu-latest - outputs: - environment: ${{ steps.set-env.outputs.environment }} - steps: - - name: Determine deployment environment - id: set-env - run: | - if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - echo "environment=production" >> $GITHUB_OUTPUT - elif [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then - echo "environment=staging" >> $GITHUB_OUTPUT - else - echo "environment=none" >> $GITHUB_OUTPUT - fi - static_analysis: + static-analysis: strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -63,21 +50,65 @@ jobs: with: python-version: ${{ matrix.python-version }} environment: null - use-llms: false + use-llms: "" + secrets: inherit # pragma: allowlist secret + + test-with-anthropic: + uses: ./.github/workflows/test.yaml + with: + python-version: "3.9" + environment: testing + use-llms: "anthropic" + secrets: inherit # pragma: allowlist secret + needs: + - test-without-llms + + test-with-azure_oai: + uses: ./.github/workflows/test.yaml + with: + python-version: "3.9" + environment: testing + use-llms: "azure_oai" + secrets: inherit # pragma: allowlist secret + needs: + - test-without-llms + + test-with-openai: + uses: ./.github/workflows/test.yaml + with: + python-version: "3.9" + environment: testing + use-llms: "openai" + secrets: inherit # pragma: allowlist secret + needs: + - test-without-llms + + test-with-togetherai: + uses: ./.github/workflows/test.yaml + with: + python-version: "3.9" + environment: testing + use-llms: "togetherai" secrets: inherit # pragma: allowlist secret + needs: + - test-without-llms - test-with-llms: + test-with-llm: uses: ./.github/workflows/test.yaml with: python-version: "3.9" environment: testing - use-llms: true + use-llms: "llm" secrets: inherit # pragma: allowlist secret needs: - test-without-llms + - test-with-anthropic + - test-with-azure_oai + - test-with-openai + - test-with-togetherai test-macos-latest: - if: github.event.pull_request.draft == false + if: github.ref != 'refs/heads/dev' && github.ref != 'refs/heads/main' && github.event.pull_request.draft == false runs-on: macos-latest steps: - uses: actions/checkout@v4 @@ -97,7 +128,7 @@ jobs: run: bash scripts/test.sh -m "not (db or nats or anthropic or azure_oai or openai or togetherai or llm)" test-windows-latest: - if: github.event.pull_request.draft == false + if: github.ref != 'refs/heads/dev' && github.ref != 'refs/heads/main' && github.event.pull_request.draft == false runs-on: windows-latest steps: - uses: actions/checkout@v4 @@ -117,10 +148,13 @@ jobs: run: bash scripts/test.sh -m "not (db or nats or anthropic or azure_oai or openai or togetherai or llm)" coverage-combine: - if: github.event.pull_request.draft == false needs: - test-without-llms - - test-with-llms + - test-with-llm + - test-with-anthropic + - test-with-azure_oai + - test-with-openai + - test-with-togetherai runs-on: ubuntu-latest steps: @@ -152,7 +186,14 @@ jobs: name: coverage-html path: htmlcov - unit_test_wasp: + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: airtai/fastagency + + unit-test-wasp: + if: github.ref != 'refs/heads/dev' && github.ref != 'refs/heads/main' && github.event.pull_request.draft == false runs-on: ubuntu-22.04 permissions: contents: read @@ -182,78 +223,6 @@ jobs: - name: Build frontend run: cd app && cd .wasp/build/web-app && npm install && REACT_APP_API_URL=$REACT_APP_API_URL npm run build - docker_build_push_node: - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - env: - PORT: ${{ vars.PORT }} - steps: - - name: Checkout repository with cached git lfs - uses: nschloe/action-cached-lfs-checkout@v1 - - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Install wasp - run: curl -sSL https://get.wasp-lang.dev/installer.sh | sh - - - name: Temporary wasp fix - run: | - PATCH_FILE_PATH=$(cat $(whereis wasp | cut -d " " -f 2) | tail -1 | cut -d " " -f 1 | cut -d "=" -f 2)/Generator/templates/server/package.json - echo $PATCH_FILE_PATH - sed -i 's/"postinstall": "patch-package"/"postinstall": ""/' $PATCH_FILE_PATH - - - name: Log in to the Container registry - uses: docker/login-action@v3.2.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - run: docker pull ghcr.io/$GITHUB_REPOSITORY-node:$GITHUB_REF_NAME || docker pull ghcr.io/$GITHUB_REPOSITORY-node:dev || true - - name: Build wasp - run: cd app && wasp build - - run: docker build --build-arg PORT=$PORT -t ghcr.io/$GITHUB_REPOSITORY-node:${GITHUB_REF_NAME////-} ./app/.wasp/build/ - - name: Add tag latest if branch is main - if: github.ref_name == 'main' - run: docker tag ghcr.io/$GITHUB_REPOSITORY-node:$GITHUB_REF_NAME ghcr.io/$GITHUB_REPOSITORY-node:latest - - name: Push only if branch name is main or dev - if: github.ref_name == 'main' || github.ref_name == 'dev' - run: docker push ghcr.io/$GITHUB_REPOSITORY-node --all-tags - - docker_build_push_fastapi: - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - name: Install wasp - run: curl -sSL https://get.wasp-lang.dev/installer.sh | sh - - - name: Log in to the Container registry - uses: docker/login-action@v3.2.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - run: docker pull ghcr.io/$GITHUB_REPOSITORY:$GITHUB_REF_NAME || docker pull ghcr.io/$GITHUB_REPOSITORY:dev || true - - run: docker build --build-arg PORT=$PORT -t ghcr.io/$GITHUB_REPOSITORY:${GITHUB_REF_NAME////-} . - - name: Add tag latest if branch is main - if: github.ref_name == 'main' - run: docker tag ghcr.io/$GITHUB_REPOSITORY:$GITHUB_REF_NAME ghcr.io/$GITHUB_REPOSITORY:latest - - name: Push only if branch name is main or dev - if: github.ref_name == 'main' || github.ref_name == 'dev' - run: docker push ghcr.io/$GITHUB_REPOSITORY --all-tags - pre-commit-check: runs-on: ubuntu-latest env: @@ -278,14 +247,12 @@ jobs: if: github.event.pull_request.draft == false needs: - - static_analysis + - static-analysis - pre-commit-check - coverage-combine - test-macos-latest - test-windows-latest - - unit_test_wasp - - docker_build_push_node - - docker_build_push_fastapi + - unit-test-wasp runs-on: ubuntu-latest @@ -294,151 +261,3 @@ jobs: uses: re-actors/alls-green@release/v1 # nosemgrep with: jobs: ${{ toJSON(needs) }} - - deploy_fastapi: - runs-on: ubuntu-22.04 - defaults: - run: - shell: bash - needs: - - check - - detect-deployment-environment - if: github.ref_name == 'main' || github.ref_name == 'dev' - environment: - name: ${{ needs.detect-deployment-environment.outputs.environment }} - env: - GITHUB_USERNAME: ${{ github.actor }} - GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - DEVELOPER_TOKEN: ${{ secrets.DEVELOPER_TOKEN }} - DATABASE_URL: ${{ secrets.DATABASE_URL }} - PY_DATABASE_URL: ${{ secrets.PY_DATABASE_URL }} - FASTAGENCY_SERVER_URL: ${{ vars.FASTAGENCY_SERVER_URL }} - DOMAIN: ${{ vars.DOMAIN }} - SSH_KEY: ${{ secrets.SSH_KEY }} - AZURE_API_VERSION: ${{ vars.AZURE_API_VERSION }} - AZURE_API_ENDPOINT: ${{ vars.AZURE_API_ENDPOINT }} - AZURE_GPT35_MODEL: ${{ vars.AZURE_GPT35_MODEL }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - - steps: - - uses: actions/checkout@v3 # Don't change it to cheackout@v4. V4 is not working with container image. - # This is to fix GIT not liking owner of the checkout dir - https://github.com/actions/runner/issues/2033#issuecomment-1204205989 - - run: chown -R $(id -u):$(id -g) $PWD - - - run: if [[ $GITHUB_REF_NAME == "main" ]]; then echo "TAG=latest" >> $GITHUB_ENV ; else echo "TAG=dev" >> $GITHUB_ENV ; fi; - - - run: echo "PATH=$PATH:/github/home/.local/bin" >> $GITHUB_ENV - - run: "which ssh-agent || ( apt-get update -y && apt-get install openssh-client git gettext -y )" - - run: eval $(ssh-agent -s) - - run: mkdir -p ~/.ssh - - run: chmod 700 ~/.ssh - - run: ssh-keyscan "$DOMAIN" >> ~/.ssh/known_hosts - - run: chmod 644 ~/.ssh/known_hosts - - run: echo "$SSH_KEY" | base64 --decode > key.pem - - run: chmod 600 key.pem - - - run: ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$DOMAIN" "docker images" - - run: bash scripts/deploy.sh - - - run: rm key.pem - - deploy_node: - runs-on: ubuntu-22.04 - defaults: - run: - shell: bash - needs: - - deploy_fastapi - - detect-deployment-environment - if: github.ref_name == 'main' || github.ref_name == 'dev' - environment: - name: ${{ needs.detect-deployment-environment.outputs.environment }} - env: - GITHUB_USERNAME: ${{ github.actor }} - GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - PORT: ${{ vars.PORT }} - GOOGLE_CLIENT_ID: ${{ vars.GOOGLE_CLIENT_ID }} - GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} - ADMIN_EMAILS: ${{ vars.ADMIN_EMAILS }} - WASP_SERVER_URL: ${{ vars.WASP_SERVER_URL }} - FASTAGENCY_SERVER_URL: ${{ vars.FASTAGENCY_SERVER_URL }} - NODE_DOMAIN: ${{ vars.NODE_DOMAIN }} - WASP_WEB_CLIENT_URL: ${{ vars.WASP_WEB_CLIENT_URL }} - DATABASE_URL: ${{ secrets.DATABASE_URL }} - REACT_APP_API_URL: ${{ vars.REACT_APP_API_URL }} - JWT_SECRET: ${{ secrets.JWT_SECRET }} - SSH_KEY: ${{ secrets.SSH_KEY }} - steps: - - name: Checkout repository with cached git lfs - uses: nschloe/action-cached-lfs-checkout@v1 - # This is to fix GIT not liking owner of the checkout dir - https://github.com/actions/runner/issues/2033#issuecomment-1204205989 - - run: chown -R $(id -u):$(id -g) $PWD - - - run: if [[ $GITHUB_REF_NAME == "main" ]]; then echo "TAG=latest" >> $GITHUB_ENV ; else echo "TAG=dev" >> $GITHUB_ENV ; fi; - - - run: echo "PATH=$PATH:/github/home/.local/bin" >> $GITHUB_ENV - - run: "which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )" - - run: eval $(ssh-agent -s) - - run: mkdir -p ~/.ssh - - run: chmod 700 ~/.ssh - - run: ssh-keyscan "$NODE_DOMAIN" >> ~/.ssh/known_hosts - - run: chmod 644 ~/.ssh/known_hosts - - run: echo "$SSH_KEY" | base64 --decode > key.pem - - run: chmod 600 key.pem - - - run: ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$NODE_DOMAIN" "docker images" - - run: bash scripts/deploy_node.sh - - - run: rm key.pem - - deploy_frontend: - runs-on: ubuntu-22.04 - permissions: - contents: write - needs: - - deploy_fastapi - - detect-deployment-environment - if: github.ref_name == 'main' || github.ref_name == 'dev' - environment: - name: ${{ needs.detect-deployment-environment.outputs.environment }} - env: - NODE_DOMAIN: ${{ vars.NODE_DOMAIN }} - SSH_KEY: ${{ secrets.SSH_KEY }} - REACT_APP_API_URL: ${{ vars.REACT_APP_API_URL }} - steps: - - name: Checkout repository with cached git lfs - uses: nschloe/action-cached-lfs-checkout@v1 - - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Install wasp - run: curl -sSL https://get.wasp-lang.dev/installer.sh | sh - - - name: Temporary wasp fix - run: | - PATCH_FILE_PATH=$(cat $(whereis wasp | cut -d " " -f 2) | tail -1 | cut -d " " -f 1 | cut -d "=" -f 2)/Generator/templates/server/package.json - echo $PATCH_FILE_PATH - sed -i 's/"postinstall": "patch-package"/"postinstall": ""/' $PATCH_FILE_PATH - - - name: Build wasp - run: cd app && wasp build - - name: Build frontend - run: cd app && cd .wasp/build/web-app && npm install && REACT_APP_API_URL=$REACT_APP_API_URL npm run build - - name: Copy 404.html - run: cp 404.html app/.wasp/build/web-app/build - - - name: Deploy UI to nginx directory - run: | - apt-get update -y && apt-get install openssh-client git -y - eval $(ssh-agent -s) - mkdir -p ~/.ssh - chmod 700 ~/.ssh - ssh-keyscan "$NODE_DOMAIN" >> ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts - echo "$SSH_KEY" | base64 --decode > key.pem - chmod 600 key.pem - ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$NODE_DOMAIN" "ls -lah /var/www/html/UI" - scp -i key.pem -r app/.wasp/build/web-app/build azureuser@"$NODE_DOMAIN":/var/www/html/UI - ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$NODE_DOMAIN" "ls -lah /var/www/html/UI" - rm key.pem diff --git a/.github/workflows/publish_coverage.yml b/.github/workflows/publish-coverage.yml similarity index 96% rename from .github/workflows/publish_coverage.yml rename to .github/workflows/publish-coverage.yml index 3e470da0..51165a35 100644 --- a/.github/workflows/publish_coverage.yml +++ b/.github/workflows/publish-coverage.yml @@ -4,6 +4,8 @@ on: workflow_run: workflows: [Pipeline] types: [completed] + ignore-branches: + - dev permissions: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bb9eb400..0b05cb42 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -13,13 +13,14 @@ on: required: true type: string use-llms: - description: 'Use LLMs in the tests' + description: 'Use LLM in the tests' required: true - type: boolean + type: string jobs: test: runs-on: ubuntu-latest + timeout-minutes: 15 environment: ${{ inputs.environment }} services: nats: @@ -53,6 +54,9 @@ jobs: - name: Set up environment variables run: | # check if an environment var or secret is defined and set env var to its value + + # vars + if [ -n "${{ vars.AZURE_API_VERSION }}" ]; then echo "AZURE_API_VERSION=${{ vars.AZURE_API_VERSION }}" >> $GITHUB_ENV fi @@ -62,6 +66,15 @@ jobs: if [ -n "${{ vars.AZURE_GPT35_MODEL }}" ]; then echo "AZURE_GPT35_MODEL=${{ vars.AZURE_GPT35_MODEL }}" >> $GITHUB_ENV fi + if [ -n "${{ vars.AZURE_GPT4_MODEL }}" ]; then + echo "AZURE_GPT4_MODEL=${{ vars.AZURE_GPT4_MODEL }}" >> $GITHUB_ENV + fi + if [ -n "${{ vars.AZURE_GPT4o_MODEL }}" ]; then + echo "AZURE_GPT4o_MODEL=${{ vars.AZURE_GPT4o_MODEL }}" >> $GITHUB_ENV + fi + + # secrets + if [ -n "${{ secrets.AZURE_OPENAI_API_KEY }}" ]; then echo "AZURE_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }}" >> $GITHUB_ENV fi @@ -103,14 +116,14 @@ jobs: - name: Prisma run: prisma migrate deploy && prisma generate - name: Test without LLMs - if: ${{ inputs.use-llms == false }} - run: bash scripts/test.sh -m "not (anthropic or azure_oai or openai or togetherai or llm)" + if: ${{ inputs.use-llms == '' }} + run: bash scripts/test.sh -vv -m "not (anthropic or azure_oai or openai or togetherai or llm)" env: COVERAGE_FILE: coverage/.coverage.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} CONTEXT: ${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} - name: Test with LLMs - if: ${{ inputs.use-llms == true }} - run: bash scripts/test.sh -m "anthropic or azure_oai or openai or togetherai or llm" + if: ${{ inputs.use-llms != '' }} + run: bash scripts/test.sh -vv -m "${{ inputs.use-llms }}" env: COVERAGE_FILE: coverage/.coverage.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} CONTEXT: ${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} diff --git a/.github/workflows/update_cron_script.yml b/.github/workflows/update-cron-script.yml similarity index 93% rename from .github/workflows/update_cron_script.yml rename to .github/workflows/update-cron-script.yml index b3f50b8a..3ac3a274 100644 --- a/.github/workflows/update_cron_script.yml +++ b/.github/workflows/update-cron-script.yml @@ -5,8 +5,8 @@ on: branches: - main paths: - - 'scripts/ci_check_certs.sh' - - '.github/workflows/update_cron_script.yml' + - 'scripts/ci-check-certs.sh' + - '.github/workflows/update-cron-script.yml' workflow_dispatch: jobs: @@ -59,7 +59,7 @@ jobs: - run: ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$DOMAIN" "ls -la" - - run: envsubst '${DOMAIN}' < scripts/cron_check_certs.sh > tmp.sh + - run: envsubst '${DOMAIN}' < scripts/cron-check-certs.sh > tmp.sh - run: chmod +x tmp.sh - run: cat tmp.sh - run: scp -i key.pem tmp.sh azureuser@"$DOMAIN":/home/azureuser/cron_check_certs.sh diff --git a/.github/workflows/update_release_notes.yaml b/.github/workflows/update-release-notes.yaml similarity index 100% rename from .github/workflows/update_release_notes.yaml rename to .github/workflows/update-release-notes.yaml diff --git a/.secrets.baseline b/.secrets.baseline index d1c08700..0bd81bff 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -118,7 +118,7 @@ "filename": ".github/workflows/test.yaml", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 33, + "line_number": 34, "is_secret": false }, { @@ -126,7 +126,7 @@ "filename": ".github/workflows/test.yaml", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 48, + "line_number": 49, "is_secret": false } ], @@ -141,5 +141,5 @@ } ] }, - "generated_at": "2024-07-02T05:22:08Z" + "generated_at": "2024-07-05T13:57:43Z" } diff --git a/Dockerfile b/Dockerfile index 49518aad..e6ddc413 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,4 +51,4 @@ ENV PATH="${PATH}:/root/.local/bin:${FLYCTL_INSTALL}/bin" EXPOSE ${PORT} ENTRYPOINT [] -CMD [ "/usr/bin/bash", "-c", "./run_server.sh" ] +CMD [ "/usr/bin/bash", "-c", "./run-server.sh" ] diff --git a/app/main.wasp b/app/main.wasp index fbdcfd4d..1a615fc2 100644 --- a/app/main.wasp +++ b/app/main.wasp @@ -174,7 +174,7 @@ page PlayGroundPage { component: import PlayGroundPageWithCustomAuth from "@src/client/app/PlayGroundPage" } -route BuildRoute { path: "/build", to: BuildPage } +route BuildRoute { path: "/build/:id?", to: BuildPage } page BuildPage { component: import BuildPageWithCustomAuth from "@src/client/app/BuildPage" } diff --git a/app/src/client/app/BuildPage.tsx b/app/src/client/app/BuildPage.tsx index f3777ef7..d7abf7bb 100644 --- a/app/src/client/app/BuildPage.tsx +++ b/app/src/client/app/BuildPage.tsx @@ -96,6 +96,8 @@ const BuildPage = ({ user }: BuildPageProps) => { const [sidebarOpen, setSidebarOpen] = useState(false); const [sideNavSelectedItem, setSideNavSelectedItem] = useState('secret'); const [togglePropertyList, setTogglePropertyList] = useState(false); + const { pathname } = location; + const activeBuildPageTab = pathname.split('/').pop(); const wrapperClass = document.body.classList.contains('server-error') ? 'h-[calc(100vh-173px)]' @@ -113,11 +115,13 @@ const BuildPage = ({ user }: BuildPageProps) => { }, [user, history]); useEffect(() => { - const selectedTab = sessionStorage.getItem('selectedBuildPageTab'); - if (selectedTab) { - setSideNavSelectedItem(selectedTab); + if (!activeBuildPageTab) return; + if (activeBuildPageTab === 'build') { + history.push(`/build/secret`); + } else { + setSideNavSelectedItem(activeBuildPageTab); } - }, []); + }, [activeBuildPageTab]); if (loading) { return ; @@ -126,7 +130,7 @@ const BuildPage = ({ user }: BuildPageProps) => { const handleSideNavItemClick = (selectedComponentName: string) => { setSideNavSelectedItem(selectedComponentName); setTogglePropertyList(!togglePropertyList); - sessionStorage.setItem('selectedBuildPageTab', selectedComponentName); + history.push(`/build/${selectedComponentName}`); }; return ( diff --git a/app/src/client/components/DynamicFormBuilder.tsx b/app/src/client/components/DynamicFormBuilder.tsx index 95581977..3580438c 100644 --- a/app/src/client/components/DynamicFormBuilder.tsx +++ b/app/src/client/components/DynamicFormBuilder.tsx @@ -1,73 +1,15 @@ -import React, { useState, useEffect, useRef, useCallback } from 'react'; -import _ from 'lodash'; - +import React, { useRef } from 'react'; import { useForm } from '../hooks/useForm'; -import { JsonSchema } from '../interfaces/BuildPageInterfaces'; -import { TextInput } from './form/TextInput'; -import { SelectInput } from './form/SelectInput'; -import { TextArea } from './form/TextArea'; -import { validateForm } from '../services/commonService'; -import { parseValidationErrors } from '../app/utils/formHelpers'; +import { useFormSubmission } from '../hooks/useFormSubmission'; +import { usePropertyReferenceValues } from '../hooks/usePropertyReferenceValues'; +import { useDeploymentInstructions } from '../hooks/useDeploymentInstructions'; +import { useEscapeKeyHandler } from '../hooks/useEscapeKeyHandler'; import Loader from '../admin/common/Loader'; import NotificationBox from './NotificationBox'; - -import { SelectedModelSchema } from '../interfaces/BuildPageInterfaces'; -import { - // getPropertyReferenceValues, - getFormSubmitValues, - getRefValues, - getMatchedUserProperties, - constructHTMLSchema, - getAllRefs, - checkForDependency, - getSecretUpdateFormSubmitValues, - getSecretUpdateValidationURL, -} from '../utils/buildPageUtils'; -import { set } from 'zod'; -import { NumericStepperWithClearButton } from './form/NumericStepperWithClearButton'; +import { DynamicFormBuilderProps } from '../interfaces/DynamicFormBuilderInterface'; import AgentConversationHistory from './AgentConversationHistory'; -import { DISCORD_URL } from '../../shared/constants'; - -interface DynamicFormBuilderProps { - allUserProperties: any; - type_name: string; - jsonSchema: JsonSchema; - validationURL: string; - updateExistingModel: SelectedModelSchema | null; - onSuccessCallback: (data: any) => void; - onCancelCallback: (event: React.FormEvent) => void; - onDeleteCallback: (data: any) => void; -} - -const SECRETS_TO_MASK = ['api_key', 'gh_token', 'fly_token']; - -const deploymentInprogressInstructions = `
GitHub Repository Created -- We have created a new GitHub repository in your GitHub account. -- The application code will be pushed to this repository in a few seconds. -Checking Deployment Status -- Once the application code is pushed, new workflows will be triggered to test and deploy the application -to Fly.io. You can check the status of the same on the GitHub repository's actions page. -Next Steps -- Wait for the workflows to complete: -- Workflow to run tests and verify the build (approx. 2 mins). -- Workflow to deploy the application to Fly.io (approx. 8 - 10 mins). - -- Once the "Fly Deployment Pipeline" completes. Please follow the below steps to access your application: -- Click on the "Fly Deployment Pipeline" action. -- Click on "onetime_app_setup" job. -- Click on "Deploy wasp application to fly" step. -- Scroll all the way to the bottom, you will see a sentence "Client has been deployed! Your Wasp -app is accessible" in the logs. Click on the link next to it to access your application. - -- Adding the fly.io configuration files: -- The above workflow might have also created a pull request in your GitHub repository -to update the fly.toml configuration files. -- Go to the Pull requests tab in your repository and merge the PR named "Add Fly.io configuration files". -You will be needing this to deploy your application to Fly.io in the future. -Need Help? -- If you encounter any issues or need assistance, please reach out to us on discord. -
-`; +import { DEPLOYMENT_PREREQUISITES } from '../utils/constants'; +import DynamicForm from './form/DynamicForm'; const DynamicFormBuilder: React.FC = ({ allUserProperties, @@ -83,286 +25,72 @@ const DynamicFormBuilder: React.FC = ({ jsonSchema, defaultValues: updateExistingModel, }); - const [isLoading, setIsLoading] = useState(false); - const [notification, setNotification] = useState({ - message: 'Oops. Something went wrong. Please try again later.', - show: false, - }); - const [refValues, setRefValues] = useState>({}); - const [missingDependency, setMissingDependency] = useState([]); - const [instructionForDeployment, setInstructionForDeployment] = useState | null>(null); - const cancelButtonRef = useRef(null); - - const isDeployment = type_name === 'deployment'; - - const missingDependencyNotificationMsg = `Please create atleast one item of type "${missingDependency.join( - ', ' - )}" to proceed.`; - - const handleSubmit = async (event: React.FormEvent) => { - event.preventDefault(); - // Avoid creating duplicate deployments - if (instructionForDeployment && !updateExistingModel) { - return; - } - setIsLoading(true); - const isSecretUpdate = type_name === 'secret' && !!updateExistingModel; - let formDataToSubmit: any = {}; - if (isSecretUpdate) { - formDataToSubmit = getSecretUpdateFormSubmitValues(formData, updateExistingModel); - validationURL = getSecretUpdateValidationURL(validationURL, updateExistingModel); - } else { - formDataToSubmit = getFormSubmitValues(refValues, formData, isSecretUpdate); // remove isSecretUpdate - } - try { - const response = await validateForm(formDataToSubmit, validationURL, isSecretUpdate); - const onSuccessCallbackResponse: any = await onSuccessCallback(response); - - isDeployment && - !updateExistingModel && - setInstructionForDeployment((prevState) => ({ - ...prevState, - gh_repo_url: response.gh_repo_url, - // @ts-ignore - instruction: deploymentInprogressInstructions.replaceAll( - '', - onSuccessCallbackResponse.gh_repo_url - ), - })); - } catch (error: any) { - try { - const errorMsgObj = JSON.parse(error.message); - const errors = parseValidationErrors(errorMsgObj); - setFormErrors(errors); - } catch (e: any) { - setNotification({ message: error.message || notification.message, show: true }); - } - } finally { - setIsLoading(false); - } - }; - const notificationOnClick = () => { - setNotification({ ...notification, show: false }); - }; - useEffect(() => { - async function fetchPropertyReferenceValues() { - if (jsonSchema) { - setIsLoading(true); - for (const [key, property] of Object.entries(jsonSchema.properties)) { - const propertyHasRef = _.has(property, '$ref') && property['$ref']; - const propertyHasAnyOf = (_.has(property, 'anyOf') || _.has(property, 'allOf')) && _.has(jsonSchema, '$defs'); - if (propertyHasRef || propertyHasAnyOf) { - const allRefList = propertyHasRef ? [property['$ref']] : getAllRefs(property); - const refUserProperties = getMatchedUserProperties(allUserProperties, allRefList); - const missingDependencyList = checkForDependency(refUserProperties, allRefList); - const title: string = property.hasOwnProperty('title') ? property.title || '' : key; - const selectedModelRefValues = _.get(updateExistingModel, key, null); - const htmlSchema = constructHTMLSchema(refUserProperties, title, property, selectedModelRefValues); - if (missingDependencyList.length > 0) { - setMissingDependency((prev) => { - const newMissingDependencies = missingDependencyList.filter((item) => !prev.includes(item)); - return prev.concat(newMissingDependencies); - }); - } - setRefValues((prev) => ({ - ...prev, - [key]: { - htmlSchema: htmlSchema, - refUserProperties: refUserProperties, - }, - })); - } - } - setIsLoading(false); - } - } - - fetchPropertyReferenceValues(); - }, [jsonSchema]); - - useEffect(() => { - if (missingDependency) { - if (missingDependency.length > 0) { - // missingDependency.length > 0 ? missingDependencyNotificationMsg - setNotification({ ...notification, show: true }); - } - } - }, [missingDependency?.length]); - - useEffect(() => { - if (updateExistingModel && type_name === 'deployment') { - const msg = deploymentInprogressInstructions; - - //@ts-ignore - setInstructionForDeployment((prevState) => ({ - ...prevState, - gh_repo_url: updateExistingModel.gh_repo_url, - flyio_app_url: updateExistingModel.flyio_app_url, - instruction: msg - //@ts-ignore - .replaceAll('', updateExistingModel.gh_repo_url) - //@ts-ignore - .replaceAll('', updateExistingModel.flyio_app_url), - })); - } - }, [isDeployment]); + const { + isLoading, + notification, + instructionForDeployment, + handleSubmit, + notificationOnClick, + onMissingDependencyClick, + setInstructionForDeployment, + } = useFormSubmission({ + type_name, + validationURL, + updateExistingModel, + onSuccessCallback, + setFormErrors, + }); - useEffect(() => { - const keyHandler = (event: KeyboardEvent) => { - if (event.key !== 'Escape') return; - cancelButtonRef.current?.click(); - }; - document.addEventListener('keydown', keyHandler); - return () => document.removeEventListener('keydown', keyHandler); + const refValues = usePropertyReferenceValues({ + jsonSchema, + allUserProperties, + updateExistingModel, }); - const appDeploymentPrerequisites = `
We've automated the application generation and deployment process so you can focus on building your application -without worrying about deployment complexities. + const cancelButtonRef = useRef(null); + const isDeployment = type_name === 'deployment'; -The deployment process includes: -- Automatically creating a new GitHub repository with the generated application code in your GitHub account. -- Automatically deploying the application to Fly.io using GitHub Actions. -Prerequisites: -Before you begin, ensure you have the following: -1. GitHub account: -- If you don't have a GitHub account, you can create one here. -- A GitHub personal access token. If you don't have one, you can generate it by following this guide. -Note: The minimum required scopes for the token are: repo, workflow, read:org, gist and user:email. + useDeploymentInstructions(updateExistingModel, type_name, setInstructionForDeployment); + useEscapeKeyHandler(cancelButtonRef); -2. Fly.io account: -- If you don't have a Fly.io account, you can create one here. Fly provides free allowances for up to 3 VMs, so deploying a Wasp app -to a new account is free but all plans require you to add your credit card information -- A Fly.io API token. If you don't have one, you can generate it by following the steps below. -- Go to your Fly.io dashboard and click on the Tokens tab (the one on the left sidebar). -- Enter a name and set the Optional Expiration to 999999h, then click on Create Organization Token to generate a token. -Note: If you already have a Fly.io account and created more than one organization, make sure you choose "Personal" as the organization - while creating the Fly.io API Token in the deployment steps below. -
-`; + const onSubmit = (event: React.FormEvent) => { + handleSubmit(event, formData, refValues); + }; return ( <> {!instructionForDeployment && isDeployment && (
)} - {/*
*/} - - {Object.entries(jsonSchema.properties).map(([key, property]) => { - if (key === 'uuid') { - return null; - } - const inputValue = formData[key] || ''; - - let formElementsObject = property; - if (_.has(property, '$ref') || _.has(property, 'anyOf') || _.has(property, 'allOf')) { - if (refValues[key]) { - formElementsObject = refValues[key].htmlSchema; - } - } - - // return formElementsObject?.enum?.length === 1 ? null : ( - return ( -
- - {formElementsObject.enum ? ( - formElementsObject.type === 'numericStepperWithClearButton' ? ( -
- handleChange(key, value)} - /> -
- ) : ( - handleChange(key, value)} - /> - ) - ) : key === 'system_message' ? ( -