From 71f4c94ff17985fa38196e7bfccafdc764696499 Mon Sep 17 00:00:00 2001 From: Harish Mohan Raj Date: Sat, 15 Jun 2024 20:48:35 +0530 Subject: [PATCH 01/17] Figure how to auto deploy to flyio in a single click (#331) * WIP * WIP * WIP: Add GH helper class * Add script to auto deploy to fly.io * Optimse the app creation flow * Pass gh token as env variable instead of authenticating with it * Fix test in windows * Fix failing tests * Refactor test_create_new_repository function to use Path constructor consistently * Merge remote-tracking branch 'origin/dev' into 298-figure-how-to-auto-deploy-to-flyio-in-a-single-click * WIP: Integrate one-click deployment logic * WIP: Integrate one-click deployment logic * WIP: Integrate one-click deployment logic * WIP: Integrate one-click deployment logic * WIP: Integrate one-click deployment logic * WIP: Integrate one-click deployment logic * Create gh_token and fly_token as secrets * Link gh_token and fly_token while creating an application * WIP: Refactoring * Integrate one-click deployment logic * Fix failing tests * Fix failing tests * Add docs for new classes * Include wasp, github cli, flyctl in dockerfile * Fix bug * Remove unnecessary sudo --------- Co-authored-by: Kumaran Rajendhiran --- Dockerfile | 14 + .../components/AgentConversationHistory.tsx | 3 + .../client/components/DynamicFormBuilder.tsx | 156 +++++---- app/src/client/components/TerminalDisplay.tsx | 19 +- .../buildPage/UserPropertyHandler.tsx | 8 +- .../client/interfaces/BuildPageInterfaces.ts | 3 + app/src/client/utils/buildPageUtils.ts | 1 + app/src/server/actions.ts | 4 +- docs/docs/SUMMARY.md | 8 + .../models/secrets/fly_token/FlyToken.md | 11 + .../secrets/github_token/GitHubToken.md | 11 + .../saas_app_generator/SaasAppGenerator.md | 11 + .../api/fastagency/saas_app_generator/main.md | 11 + fastagency/app.py | 89 ++++- fastagency/models/applications/application.py | 10 + fastagency/models/secrets/__init__.py | 1 + fastagency/models/secrets/fly_token.py | 22 ++ fastagency/models/secrets/github_token.py | 22 ++ fastagency/saas_app_generator.py | 260 ++++++++++++++ tests/app/test_get_schemas.py | 2 + tests/app/test_model_routes.py | 149 +++++++- tests/app/test_openai_extensively.py | 3 +- tests/models/agents/test_assistant.py | 6 + tests/models/agents/test_user_proxy.py | 2 + tests/models/agents/test_web_surfer.py | 5 + tests/models/applications/test_application.py | 95 +++++- tests/models/llms/test_azure.py | 4 + tests/models/llms/test_openai.py | 4 + tests/models/llms/test_together.py | 4 + tests/models/secrets/test_fly_token.py | 55 +++ tests/models/secrets/test_github_token.py | 55 +++ tests/models/teams/test_multi_agents_team.py | 6 + tests/models/teams/test_two_agents_team.py | 6 + tests/models/toolboxes/test_toolbox.py | 4 + tests/test_nats.py | 6 + tests/test_saas_app_generator.py | 319 ++++++++++++++++++ 36 files changed, 1297 insertions(+), 92 deletions(-) create mode 100644 docs/docs/en/api/fastagency/models/secrets/fly_token/FlyToken.md create mode 100644 docs/docs/en/api/fastagency/models/secrets/github_token/GitHubToken.md create mode 100644 docs/docs/en/api/fastagency/saas_app_generator/SaasAppGenerator.md create mode 100644 docs/docs/en/api/fastagency/saas_app_generator/main.md create mode 100644 fastagency/models/secrets/fly_token.py create mode 100644 fastagency/models/secrets/github_token.py create mode 100644 fastagency/saas_app_generator.py create mode 100644 tests/models/secrets/test_fly_token.py create mode 100644 tests/models/secrets/test_github_token.py create mode 100644 tests/test_saas_app_generator.py diff --git a/Dockerfile b/Dockerfile index aaa123f0..00b8a123 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,20 @@ COPY fastagency ./fastagency COPY scripts/* schema.prisma pyproject.toml README.md ./ RUN pip install -e ".[dev]" +# Install wasp +RUN curl -sSL https://get.wasp-lang.dev/installer.sh | sh +# Install github cli +RUN (type -p wget >/dev/null || (apt update && apt-get install wget -y)) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt update \ + && apt install gh -y \ + && rm -rf /var/lib/apt/lists/* +# Install flyctl +RUN curl -L https://fly.io/install.sh | sh + EXPOSE ${PORT} ENTRYPOINT [] diff --git a/app/src/client/components/AgentConversationHistory.tsx b/app/src/client/components/AgentConversationHistory.tsx index fcc1ed7a..fce09d7f 100644 --- a/app/src/client/components/AgentConversationHistory.tsx +++ b/app/src/client/components/AgentConversationHistory.tsx @@ -6,6 +6,7 @@ interface AgentConversationHistoryProps { initialState?: boolean; isAgentWindow?: boolean; isDeploymentInstructions?: boolean; + containerTitle?: string; } const AgentConversationHistory: React.FC = ({ @@ -13,6 +14,7 @@ const AgentConversationHistory: React.FC = ({ initialState = false, isAgentWindow = false, isDeploymentInstructions = false, + containerTitle, }) => { const [showHistory, setShowHistory] = useState(initialState); @@ -38,6 +40,7 @@ const AgentConversationHistory: React.FC = ({ maxHeight={maxH} isOpenOnLoad={isDeploymentInstructions ? isDeploymentInstructions : isAgentWindow} theme={isDeploymentInstructions ? 'modelDeployment' : null} + containerTitle={containerTitle} /> diff --git a/app/src/client/components/DynamicFormBuilder.tsx b/app/src/client/components/DynamicFormBuilder.tsx index 3599dad6..0e3b8d62 100644 --- a/app/src/client/components/DynamicFormBuilder.tsx +++ b/app/src/client/components/DynamicFormBuilder.tsx @@ -39,6 +39,36 @@ interface DynamicFormBuilderProps { onDeleteCallback: (data: any) => void; } +const SECRETS_TO_MASK = ['api_key', 'gh_token', 'fly_token']; + +const deploymentInprogressInstructions = `
+- Appication deployment status: In Progress + +GitHub Repository Created +- We have created a new GitHub repository in your GitHub account. +- The application code will be pushed to this repository in a few minutes. +Checking Deployment Status +- You can either check the status on the GitHub repository page or come back to this page after a few minutes. +Troubleshooting: +Common Issues: +- In Progress Status for Too Long: +- Ensure you have entered the correct gh_token and fly_token. +- Verify that you have added a payment method to your Fly.io account; otherwise, the deployment will fail. +- Review the deployment logs on Fly.io for any error messages. Access the logs by clicking on the server application + on the Fly.io dashboard, then clicking on the "Live Logs" tab. +Need Help? +- If you encounter any issues or need assistance, please reach out to us on discord. +
+`; + +const deploymentCompleteInstructions = `
- Hurrah! Your application has been successfully pushed to the GitHub repository. + +- A new workflow has been triggered to deploy the application to Fly.io. You can check the status on the GitHub repository actions page. + +- Once the deployment workflow is completed (approx 5 - 10 mins), you can access your application using this link +
+`; + const DynamicFormBuilder: React.FC = ({ allUserProperties, type_name, @@ -57,10 +87,10 @@ const DynamicFormBuilder: React.FC = ({ const [showNotification, setShowNotification] = useState(false); const [refValues, setRefValues] = useState>({}); const [missingDependency, setMissingDependency] = useState([]); - const [instructionForApplication, setInstructionForApplication] = useState(null); + const [instructionForApplication, setInstructionForApplication] = useState | null>(null); const cancelButtonRef = useRef(null); - const showDeployInstructions = type_name === 'application'; + const isApplication = type_name === 'application'; const handleSubmit = async (event: React.FormEvent) => { event.preventDefault(); @@ -79,8 +109,16 @@ const DynamicFormBuilder: React.FC = ({ } try { const response = await validateForm(formDataToSubmit, validationURL, isSecretUpdate); - onSuccessCallback(response); - showDeployInstructions && !updateExistingModel && setInstructionForApplication(response.uuid); + const onSuccessCallbackResponse = await onSuccessCallback(response); + + isApplication && + !updateExistingModel && + setInstructionForApplication((prevState) => ({ + ...prevState, + gh_repo_url: response.gh_repo_url, + // @ts-ignore + instruction: deploymentInprogressInstructions.replace('', onSuccessCallbackResponse.gh_repo_url), + })); } catch (error: any) { try { const errorMsgObj = JSON.parse(error.message); @@ -147,8 +185,20 @@ const DynamicFormBuilder: React.FC = ({ }, [missingDependency?.length]); useEffect(() => { - updateExistingModel && type_name === 'application' && setInstructionForApplication(updateExistingModel.uuid); - }, [showDeployInstructions]); + updateExistingModel && + type_name === 'application' && + //@ts-ignore + setInstructionForApplication((prevState) => ({ + ...prevState, + gh_repo_url: updateExistingModel.gh_repo_url, + flyio_app_url: updateExistingModel.flyio_app_url, + instruction: deploymentCompleteInstructions + //@ts-ignore + .replace('', updateExistingModel.gh_repo_url) + //@ts-ignore + .replace('', updateExistingModel.flyio_app_url), + })); + }, [isApplication]); useEffect(() => { const keyHandler = (event: KeyboardEvent) => { @@ -159,76 +209,41 @@ const DynamicFormBuilder: React.FC = ({ return () => document.removeEventListener('keydown', keyHandler); }); - const deploymentInstructions = showDeployInstructions - ? `
Introduction: -The application is based on Wasp, an open-source framework for building full-stack web apps. The generated application includes: -- A landing page and a chat page -- Username & Password based authentication -- Automated deployment to Fly.io using GitHub Actions + const appDeploymentPrerequisites = `
We've automated the application generation and deployment process so you can focus on building your application without worrying about deployment complexities. +The deployment process includes: +- Automatically creating a new GitHub repository with the generated application code in your GitHub account. +- Automatically deploying the application to Fly.io using GitHub Actions. Prerequisites: Before you begin, ensure you have the following: -1. Fly.io account: -- If you don't have a Fly.io account, you can create one here. -- Fly provides free allowances for up to 3 VMs (so deploying a Wasp app to a new account is free), but all plans -require you to add your credit card information before you can proceed. If you don't, the deployment will fail. +1. GitHub account: +- If you don't have a GitHub account, you can create one here. +- A GitHub personal access token. If you don't have one, you can generate it by following this guide. +Note: The minimum required scopes for the token are: repo, read:org, and gist. -Important: If you already have a Fly.io account and created more than one organization, make sure you choose "Personal" as the organization +2. Fly.io account: +- If you don't have a Fly.io account, you can create one here. Fly provides free allowances for up to 3 VMs, so deploying a Wasp app +to a new account is free but all plans require you to add your credit card information +- A Fly.io API token. If you don't have one, you can generate it by following the steps below. +- Go to your Fly.io dashboard and click on the Tokens tab (the one on the left sidebar). +- Enter a name and set the Optional Expiration to 999999h, then click on Create Organization Token to generate a token. +Note: If you already have a Fly.io account and created more than one organization, make sure you choose "Personal" as the organization while creating the Fly.io API Token in the deployment steps below. -Deployment Steps: -Step 1: Fork the GitHub Repository: -1.1 Fork this GitHub Repository to your account. Ensure the checkbox "Copy the main branch only" is checked. -Step 2: Generate Fly.io API Token: -2.1 Go to your Fly.io dashboard and click on the Tokens tab (the one on the left sidebar). -2.2 Enter a name and set the Optional Expiration to 999999h, then click on Create Organization Token to generate a token. -2.3 Copy the token, including the "FlyV1 " prefix and space at the beginning. -Step 3: Set necessary GitHub action secrets: -3.1 Create the below two "repository secrets" in your forked GitHub repository. -Note: If you don't know how to create a secret, follow this guide. -================================================================================== -FLY_API_TOKEN: <--- paste the value you copied from 2.3 ---> -FASTAGENCY_APPLICATION_UUID: ${instructionForApplication} -================================================================================== -Step 4: Set up applications Fly.io: -4.1 Go to the Actions tab in your forked repository on GitHub and click the -I understand my workflows, go ahead and enable them button. -4.2 On the left-hand side, you will see options like: All workflows, Fly Deployment Pipeline, Pipeline. -4.3 Click on the Fly Deployment Pipeline option and and then click the Run workflow button against the main branch. -4.4 Wait for the workflow to complete (approx. 2 mins). Once completed, you will see the Client, Server, and Database apps -created on Fly.io dashboard. -4.5 The workflow will only set up the applications in Fly.io and not deploy the actual application code which -will be done in the next step. -Step 5: Deploy the Application: -5.1 The above workflow might have also created a pull request in your GitHub repository to update the fly.toml files. -5.2 Go to the Pull requests tab in your forked repository on GitHub and merge the PR named "Add Fly.io configuration files". -5.3 It will trigger the below workflows in sequence: -- Pipeline to run tests and verify the build (approx. 2 mins). -- Pipeline to deploy the tested application to Fly.io (approx. 5 - 10 mins). -5.4 Once the workflow is completed, you can access your application using the hostname provided in the Fly.io dashboard. -5.5 Go to fly dashboard and click on the client application (similar to: fastagency-app-******-client). -5.6 The hostname is the URL of your application. Open the URL in your browser to launch your application. -Application customization (Optional): -- You can perform basic customization such as changing the app name and adding a support email address in the generated application by - setting the below optional "repository variables" (not repository secrets). Click here to learn how to set repository variables. -================================================================================== -REACT_APP_NAME: <--- Your App Name ---> -REACT_APP_SUPPORT_EMAIL: <--- Your Support Email Address ---> -================================================================================== -- After setting the repository variables, you can manualy trigger the Fly Deployment Pipeline workflow (refer 4.2 and 4.3) to deploy the changes. -- For further customization, you can refer to the Wasp documentation. -Troubleshooting: -If you encounter any issues during the deployment, check the following common problems: -Deployment Failures: -- Make sure you have added a payment method to your Fly.io account. Else, the deployment will fail. -- Review the deployment logs on Fly.io for any error messages. You can access the logs by clicking on the -server application on the Fly.io dashboard and then clicking on the Live Logs tab. -- If you need any help, please reach out to us on discord. +Note: We do not store your GitHub personal access token or Fly.io API token. So you need to provide them each time you deploy an application.
-` - : ''; +`; return ( <> + {!instructionForApplication && isApplication && ( +
+ +
+ )} {/*
*/} {Object.entries(jsonSchema.properties).map(([key, property]) => { @@ -276,7 +291,7 @@ Before you begin, ensure you have the following: ) : ( handleChange(key, value)} @@ -286,11 +301,12 @@ Before you begin, ensure you have the following:
); })} - {instructionForApplication && ( + {instructionForApplication && instructionForApplication.instruction && (
)} diff --git a/app/src/client/components/TerminalDisplay.tsx b/app/src/client/components/TerminalDisplay.tsx index 473bc9fd..0b231a72 100644 --- a/app/src/client/components/TerminalDisplay.tsx +++ b/app/src/client/components/TerminalDisplay.tsx @@ -2,17 +2,24 @@ import React, { useEffect, useRef, useState } from 'react'; interface TerminalDisplayProps { messages: string; - maxHeight: number; // Maximum height in pixels - isOpenOnLoad?: boolean; // Whether the terminal is open on load - theme?: string | null; // Title of the terminal + maxHeight: number; + isOpenOnLoad?: boolean; + theme?: string | null; + containerTitle?: string; } -const TerminalDisplay: React.FC = ({ messages, maxHeight, isOpenOnLoad, theme }) => { +const TerminalDisplay: React.FC = ({ + messages, + maxHeight, + isOpenOnLoad, + theme, + containerTitle, +}) => { const [isMinimized, setIsMinimized] = useState(isOpenOnLoad ? false : true); // Track if terminal is minimized const containerRef = useRef(null); // Reference to the scroll container const [isAutoScroll, setIsAutoScroll] = useState(true); // Track if auto-scroll is enabled const isModelDeploymentTheme = theme === 'modelDeployment'; // Check if the theme is modelDeployment - const containerTitle = isModelDeploymentTheme ? 'Deployment Guide for Your Application' : 'Agent conversations'; // Title of the terminal + const title = containerTitle ? containerTitle : 'Agent conversations'; // Title of the terminal // Convert ANSI codes to HTML with inline styles const convertAnsiToHtml = (text: string): string => { @@ -53,7 +60,7 @@ const TerminalDisplay: React.FC = ({ messages, maxHeight, } text-airt-font-base p-1 text-right bg-airt-secondary hover:cursor-pointer`} onClick={() => setIsMinimized(!isMinimized)} > -

{containerTitle}

+

{title}