Merge remote-tracking branch 'origin' into fix-flytectl-install-script

Signed-off-by: Eduardo Apolinario <[email protected]>
flyteorg · May 22, 2024 · 163b65b · 163b65b
2 parents 81dd7f1 + 2143948
commit 163b65b
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 48 deletions.
diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md
@@ -195,8 +195,11 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out |
 | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment |
 | flyteagent.enabled | bool | `false` |  |
-| flyteagent.plugin_config.plugins.agentService.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` |  |
-| flyteagent.plugin_config.plugins.agentService.defaultAgent.insecure | bool | `true` |  |
+| flyteagent.plugin_config.plugins.agent-service | object | `{"defaultAgent":{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true},"supportedTaskTypes":["sensor"]}` | Agent service configuration for propeller. |
+| flyteagent.plugin_config.plugins.agent-service.defaultAgent | object | `{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true}` | The default agent service to use for plugin tasks. |
+| flyteagent.plugin_config.plugins.agent-service.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` | The agent service endpoint propeller should connect to. |
+| flyteagent.plugin_config.plugins.agent-service.defaultAgent.insecure | bool | `true` | Whether the connection from propeller to the agent service should use TLS. |
+| flyteagent.plugin_config.plugins.agent-service.supportedTaskTypes | list | `["sensor"]` | The task types supported by the default agent. |
 | flyteagent.podLabels | object | `{}` | Labels for flyteagent pods |
 | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment |
 | flyteconsole.enabled | bool | `true` |  |

diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml
@@ -279,10 +279,19 @@ flyteagent:
   enabled: false
   plugin_config:
     plugins:
-      agentService:
+      # -- Agent service configuration for propeller.
+      agent-service:
+        # -- The default agent service to use for plugin tasks.
         defaultAgent:
+          # -- The agent service endpoint propeller should connect to.
           endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000"
+          # -- Whether the connection from propeller to the agent service should use TLS.
           insecure: true
+        # -- The task types supported by the default agent.
+        supportedTaskTypes:
+        - sensor
+        # -- Uncomment to enable task type that uses Flyte Agent
+        # - bigquery_query_job_task
   # -- Labels for flyteagent pods
   podLabels: {}
 

diff --git a/docs/deployment/configuration/auth_setup.rst b/docs/deployment/configuration/auth_setup.rst
@@ -172,7 +172,7 @@ Apply OIDC Configuration
           oidc:
             # baseUrl: https://accounts.google.com # Uncomment for Google
             # baseUrl: https://<keycloak-url>/auth/realms/<keycloak-realm> # Uncomment for Keycloak and update with your installation host and realm name
-            # baseUrl: https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/authorize # Uncomment for Azure AD
+            # baseUrl: https://login.microsoftonline.com/<tenant-id>/v2.0 # Uncomment for Azure AD
             # For Okta use the Issuer URI from Okta's default auth server
             baseUrl: https://dev-<org-id>.okta.com/oauth2/default
             # Replace with the client ID and secret created for Flyte in your IdP
@@ -488,7 +488,7 @@ Follow the steps in this section to configure `flyteadmin` to use an external au
            enabled: true
            oidc:
            # baseUrl: https://<keycloak-url>/auth/realms/<keycloak-realm> # Uncomment for Keycloak and update with your installation host and realm name
-           # baseUrl: https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/authorize # Uncomment for Azure AD
+           # baseUrl: https://login.microsoftonline.com/<tenant-id>/v2.0 # Uncomment for Azure AD
            # For Okta, use the Issuer URI of the custom auth server:
              baseUrl: https://dev-<org-id>.okta.com/oauth2/<auth-server-id>
            # Use the client ID and secret generated by your IdP for the first OIDC registration in the "Identity Management layer : OIDC" section of this guide
@@ -516,7 +516,7 @@ Follow the steps in this section to configure `flyteadmin` to use an external au
                authServerType: External
                externalAuthServer:
                # baseUrl: https://<keycloak-url>/auth/realms/<keycloak-realm> # Uncomment for Keycloak and update with your installation host and realm name
-               # baseUrl: https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/authorize # Uncomment for Azure AD
+               # baseUrl: https://login.microsoftonline.com/<tenant-id>/v2.0 # Uncomment for Azure AD
                # For Okta, use the Issuer URI of the custom auth server:
                  baseUrl: https://dev-<org-id>.okta.com/oauth2/<auth-server-id>
                  metadataUrl: .well-known/oauth-authorization-server
@@ -531,8 +531,8 @@ Follow the steps in this section to configure `flyteadmin` to use an external au
              userAuth:
                openId:
                # baseUrl: https://<keycloak-url>/auth/realms/<keycloak-realm> # Uncomment for Keycloak and update with your installation host and realm name
-               # baseUrl: https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/authorize # Uncomment for Azure AD
-               # For Okta, use the Issuer URI of the custom auth server:
+               # baseUrl: https://login.microsoftonline.com/<tenant-id>/v2.0 # Uncomment for Azure AD
+               # For Okta, use the Issuer URI of the custom auth server:  
                  baseUrl: https://dev-<org-id>.okta.com/oauth2/<auth-server-id>
                  scopes:
                  - profile
@@ -568,39 +568,29 @@ Follow the steps in this section to configure `flyteadmin` to use an external au
 
                  authServerType: External
 
-                 # 2. Optional: Set external auth server baseUrl if different from OpenId baseUrl.
-                 externalAuthServer:
-                 # baseUrl: https://<keycloak-url>/auth/realms/<keycloak-realm> # Uncomment for Keycloak and update with your installation host and realm name
-                 # baseUrl: https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/authorize # Uncomment for Azure AD
-                 # For Okta, use the Issuer URI of the custom auth server:
-                   baseUrl: https://dev-<org-id>.okta.com/oauth2/<auth-server-id>
-
-                   metadataUrl: .well-known/openid-configuration
-
-                 thirdPartyConfig:
-                    flyteClient:
-                       # 3. Replace with a new Native/Public Client ID provisioned in the custom authorization server.
-                       clientId: flytectl
-                       # This should not change
-                       redirectUri: http://localhost:53593/callback
-                       # 4. "all" is a required scope and must be configured in the custom authorization server.
-                       scopes:
-                       - offline
-                       - all
-
-               userAuth:
-                 openId:
-                 # baseUrl: https://<keycloak-url>/auth/realms/<keycloak-realm> # Uncomment for Keycloak and update with your installation host and realm name
-                 # baseUrl: https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/authorize # Uncomment for Azure AD
-                 # For Okta, use the Issuer URI of the custom auth server:
-                   baseUrl: https://dev-<org-id>.okta.com/oauth2/<auth-server-id>
-                   scopes:
-                   - profile
-                   - openid
-                   # - offline_access # Uncomment if OIdC supports issuing refresh tokens.
-                   clientId: <client id>
+               # 2. Optional: Set external auth server baseUrl if different from OpenId baseUrl.
+               externalAuthServer:
+               # baseUrl: https://<keycloak-url>/auth/realms/<keycloak-realm> # Uncomment for Keycloak and update with your installation host and realm name
+               # baseUrl: https://login.microsoftonline.com/<tenant-id>/v2.0 # Uncomment for Azure AD
+               # For Okta, use the Issuer URI of the custom auth server:  
+                 baseUrl: https://dev-<org-id>.okta.com/oauth2/<auth-server-id>
+               
+                 metadataUrl: .well-known/openid-configuration
 
 
+             userAuth:
+               openId:
+               # baseUrl: https://<keycloak-url>/auth/realms/<keycloak-realm> # Uncomment for Keycloak and update with your installation host and realm name
+               # baseUrl: https://login.microsoftonline.com/<tenant-id>/v2.0 # Uncomment for Azure AD
+               # For Okta, use the Issuer URI of the custom auth server:  
+                 baseUrl: https://dev-<org-id>.okta.com/oauth2/<auth-server-id>
+                 scopes:
+                 - profile
+                 - openid
+                 # - offline_access # Uncomment if OIdC supports issuing refresh tokens.
+                 clientId: <client id>
+                  
+   
          secrets:
            adminOauthClientCredentials:
              enabled: true # see the section "Disable Helm secret management" if you require to do so

diff --git a/docs/deployment/plugins/k8s/index.rst b/docs/deployment/plugins/k8s/index.rst
@@ -17,11 +17,11 @@ Select the integration you need and follow the steps to install the correspondin
 
   .. group-tab:: PyTorch/TensorFlow/MPI
 
-    1. Install the `Kubeflow training-operator <https://github.com/kubeflow/training-operator?tab=readme-ov-file#kubeflow-training-operator>`__:
+    1. Install the `Kubeflow training-operator <https://github.com/kubeflow/training-operator?tab=readme-ov-file#stable-release>`__ (Please install the stable release):
 
     .. code-block:: bash
 
-      kubectl apply -k "github.com/kubeflow/training-operator/manifests/overlays/standalone"
+      kubectl apply -k "github.com/kubeflow/training-operator/manifests/overlays/standalone?ref=v1.7.0"
 
     **Optional: Using a gang scheduler**
 

diff --git a/docs/flyte_agents/developing_agents.md b/docs/flyte_agents/developing_agents.md
@@ -133,21 +133,20 @@ class FileSensor(BaseSensor):
 
 ### 2. Test the agent
 
-You can test your agent in a {ref}`local Python environment <testing_agents_locally>` or in a {ref}<local development cluster `testing_agents_in_a_local_development_cluster`>.
+You can test your agent in a {ref}`local Python environment <testing_agents_locally>` or in a {ref}`local development cluster <testing_agents_in_a_local_development_cluster>`.
 
 ### 3. Build a new Docker image
 
 The following is a sample Dockerfile for building an image for a Flyte agent:
 
 ```Dockerfile
-FROM python:3.9-slim-buster
+FROM python:3.10-slim-bookworm
 
 MAINTAINER Flyte Team <[email protected]>
 LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit
 
-WORKDIR /root
-ENV PYTHONPATH /root
-
+# additional dependencies for running in k8s
+RUN pip install prometheus-client grpcio-health-checking
 # flytekit will autoload the agent if package is installed.
 RUN pip install flytekitplugins-bigquery
 CMD pyflyte serve agent --port 8000
@@ -193,7 +192,7 @@ By running agents independently, you can thoroughly test and validate your agent
 controlled environment before deploying them to the production cluster.
 
 By default, all agent requests will be sent to the default agent service. However,
-you can route particular task requests to designated agent services by adjusting the FlytePropeller configuration. 
+you can route particular task requests to designated agent services by adjusting the FlytePropeller configuration.
 
 ```yaml
  plugins:

diff --git a/monodocs-environment.yaml b/monodocs-environment.yaml
@@ -6,7 +6,7 @@ dependencies:
   - pip
   - codespell
   - furo
-  - flytekit>=1.10.2
+  - flytekit>=1.12.1b2
   - gitpython
   - ipython!=8.7.0
   - graphviz