From cfaedce95aa5327cc272682655eaa779a2f03f6d Mon Sep 17 00:00:00 2001 From: Jason Parraga Date: Tue, 21 May 2024 15:38:05 -0700 Subject: [PATCH 1/6] Fix link to test on local cluster (#5398) Signed-off-by: Jason Parraga Co-authored-by: Jason Parraga --- docs/flyte_agents/developing_agents.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/flyte_agents/developing_agents.md b/docs/flyte_agents/developing_agents.md index fe55630248..24b29f73ae 100644 --- a/docs/flyte_agents/developing_agents.md +++ b/docs/flyte_agents/developing_agents.md @@ -133,7 +133,7 @@ class FileSensor(BaseSensor): ### 2. Test the agent -You can test your agent in a {ref}`local Python environment ` or in a {ref}. +You can test your agent in a {ref}`local Python environment ` or in a {ref}`local development cluster `. ### 3. Build a new Docker image From 2f7bedfe80440f642f2331d875ecfb22b13a218e Mon Sep 17 00:00:00 2001 From: Erwin de Haan <1627021+EraYaN@users.noreply.github.com> Date: Wed, 22 May 2024 19:39:15 +0200 Subject: [PATCH 2/6] Replace Azure AD OIDC URL with correct one (#4075) Signed-off-by: Erwin de Haan --- docs/deployment/configuration/auth_setup.rst | 62 ++++++++------------ 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/docs/deployment/configuration/auth_setup.rst b/docs/deployment/configuration/auth_setup.rst index bb73bef8e3..9d628f5790 100644 --- a/docs/deployment/configuration/auth_setup.rst +++ b/docs/deployment/configuration/auth_setup.rst @@ -172,7 +172,7 @@ Apply OIDC Configuration oidc: # baseUrl: https://accounts.google.com # Uncomment for Google # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD + # baseUrl: https://login.microsoftonline.com//v2.0 # Uncomment for Azure AD # For Okta use the Issuer URI from Okta's default auth server baseUrl: https://dev-.okta.com/oauth2/default # Replace with the client ID and secret created for Flyte in your IdP @@ -488,7 +488,7 @@ Follow the steps in this section to configure `flyteadmin` to use an external au enabled: true oidc: # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD + # baseUrl: https://login.microsoftonline.com//v2.0 # Uncomment for Azure AD # For Okta, use the Issuer URI of the custom auth server: baseUrl: https://dev-.okta.com/oauth2/ # Use the client ID and secret generated by your IdP for the first OIDC registration in the "Identity Management layer : OIDC" section of this guide @@ -516,7 +516,7 @@ Follow the steps in this section to configure `flyteadmin` to use an external au authServerType: External externalAuthServer: # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD + # baseUrl: https://login.microsoftonline.com//v2.0 # Uncomment for Azure AD # For Okta, use the Issuer URI of the custom auth server: baseUrl: https://dev-.okta.com/oauth2/ metadataUrl: .well-known/oauth-authorization-server @@ -531,8 +531,8 @@ Follow the steps in this section to configure `flyteadmin` to use an external au userAuth: openId: # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD - # For Okta, use the Issuer URI of the custom auth server: + # baseUrl: https://login.microsoftonline.com//v2.0 # Uncomment for Azure AD + # For Okta, use the Issuer URI of the custom auth server: baseUrl: https://dev-.okta.com/oauth2/ scopes: - profile @@ -568,39 +568,29 @@ Follow the steps in this section to configure `flyteadmin` to use an external au authServerType: External - # 2. Optional: Set external auth server baseUrl if different from OpenId baseUrl. - externalAuthServer: - # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD - # For Okta, use the Issuer URI of the custom auth server: - baseUrl: https://dev-.okta.com/oauth2/ - - metadataUrl: .well-known/openid-configuration - - thirdPartyConfig: - flyteClient: - # 3. Replace with a new Native/Public Client ID provisioned in the custom authorization server. - clientId: flytectl - # This should not change - redirectUri: http://localhost:53593/callback - # 4. "all" is a required scope and must be configured in the custom authorization server. - scopes: - - offline - - all - - userAuth: - openId: - # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD - # For Okta, use the Issuer URI of the custom auth server: - baseUrl: https://dev-.okta.com/oauth2/ - scopes: - - profile - - openid - # - offline_access # Uncomment if OIdC supports issuing refresh tokens. - clientId: + # 2. Optional: Set external auth server baseUrl if different from OpenId baseUrl. + externalAuthServer: + # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name + # baseUrl: https://login.microsoftonline.com//v2.0 # Uncomment for Azure AD + # For Okta, use the Issuer URI of the custom auth server: + baseUrl: https://dev-.okta.com/oauth2/ + + metadataUrl: .well-known/openid-configuration + userAuth: + openId: + # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name + # baseUrl: https://login.microsoftonline.com//v2.0 # Uncomment for Azure AD + # For Okta, use the Issuer URI of the custom auth server: + baseUrl: https://dev-.okta.com/oauth2/ + scopes: + - profile + - openid + # - offline_access # Uncomment if OIdC supports issuing refresh tokens. + clientId: + + secrets: adminOauthClientCredentials: enabled: true # see the section "Disable Helm secret management" if you require to do so From 317ad3081aba30be07481e66f36c1f21c1376ae6 Mon Sep 17 00:00:00 2001 From: Jason Parraga Date: Wed, 22 May 2024 13:44:19 -0700 Subject: [PATCH 3/6] Update the example Dockerfile to run on k8s (#5412) Signed-off-by: Jason Parraga --- docs/flyte_agents/developing_agents.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/flyte_agents/developing_agents.md b/docs/flyte_agents/developing_agents.md index 24b29f73ae..ee989b812f 100644 --- a/docs/flyte_agents/developing_agents.md +++ b/docs/flyte_agents/developing_agents.md @@ -140,14 +140,13 @@ You can test your agent in a {ref}`local Python environment LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit -WORKDIR /root -ENV PYTHONPATH /root - +# additional dependencies for running in k8s +RUN pip install prometheus-client grpcio-health-checking # flytekit will autoload the agent if package is installed. RUN pip install flytekitplugins-bigquery CMD pyflyte serve agent --port 8000 @@ -193,7 +192,7 @@ By running agents independently, you can thoroughly test and validate your agent controlled environment before deploying them to the production cluster. By default, all agent requests will be sent to the default agent service. However, -you can route particular task requests to designated agent services by adjusting the FlytePropeller configuration. +you can route particular task requests to designated agent services by adjusting the FlytePropeller configuration. ```yaml plugins: From c1eddadfa597c59f1f2694b13864fbb82dee7e4b Mon Sep 17 00:00:00 2001 From: Chi-Sheng Liu Date: Thu, 23 May 2024 04:49:19 +0800 Subject: [PATCH 4/6] docs(kubeflow): Fix kubeflow webhook error (#5410) Signed-off-by: Chi-Sheng Liu --- docs/deployment/plugins/k8s/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/deployment/plugins/k8s/index.rst b/docs/deployment/plugins/k8s/index.rst index cf9a211f85..526a5975fe 100644 --- a/docs/deployment/plugins/k8s/index.rst +++ b/docs/deployment/plugins/k8s/index.rst @@ -17,11 +17,11 @@ Select the integration you need and follow the steps to install the correspondin .. group-tab:: PyTorch/TensorFlow/MPI - 1. Install the `Kubeflow training-operator `__: + 1. Install the `Kubeflow training-operator `__ (Please install the stable release): .. code-block:: bash - kubectl apply -k "github.com/kubeflow/training-operator/manifests/overlays/standalone" + kubectl apply -k "github.com/kubeflow/training-operator/manifests/overlays/standalone?ref=v1.7.0" **Optional: Using a gang scheduler** From 8a612524ef1b3971aaceef8c19f661c6f2020150 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 23 May 2024 02:29:37 +0530 Subject: [PATCH 5/6] update flytekit version to 1.12.1b2 in monodocs requirements (#5411) Signed-off-by: Samhita Alla --- monodocs-environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monodocs-environment.yaml b/monodocs-environment.yaml index 5a7774b8ed..ba32d33b3a 100644 --- a/monodocs-environment.yaml +++ b/monodocs-environment.yaml @@ -6,7 +6,7 @@ dependencies: - pip - codespell - furo - - flytekit>=1.10.2 + - flytekit>=1.12.1b2 - gitpython - ipython!=8.7.0 - graphviz From 2143948c9b7ef8e4fc2bb876756d6552ab19843f Mon Sep 17 00:00:00 2001 From: Jason Parraga Date: Wed, 22 May 2024 14:01:26 -0700 Subject: [PATCH 6/6] Add supported task types to agent service config and rename (#5402) Signed-off-by: Jason Parraga --- charts/flyte-core/README.md | 7 +++++-- charts/flyte-core/values.yaml | 11 ++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index 7e2da9da96..f2e0dbff1c 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -195,8 +195,11 @@ helm install gateway bitnami/contour -n flyte | flyteadmin.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out | | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment | | flyteagent.enabled | bool | `false` | | -| flyteagent.plugin_config.plugins.agentService.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` | | -| flyteagent.plugin_config.plugins.agentService.defaultAgent.insecure | bool | `true` | | +| flyteagent.plugin_config.plugins.agent-service | object | `{"defaultAgent":{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true},"supportedTaskTypes":["sensor"]}` | Agent service configuration for propeller. | +| flyteagent.plugin_config.plugins.agent-service.defaultAgent | object | `{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true}` | The default agent service to use for plugin tasks. | +| flyteagent.plugin_config.plugins.agent-service.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` | The agent service endpoint propeller should connect to. | +| flyteagent.plugin_config.plugins.agent-service.defaultAgent.insecure | bool | `true` | Whether the connection from propeller to the agent service should use TLS. | +| flyteagent.plugin_config.plugins.agent-service.supportedTaskTypes | list | `["sensor"]` | The task types supported by the default agent. | | flyteagent.podLabels | object | `{}` | Labels for flyteagent pods | | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyteconsole.enabled | bool | `true` | | diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index 4d1e77def0..4dd0fdf615 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -279,10 +279,19 @@ flyteagent: enabled: false plugin_config: plugins: - agentService: + # -- Agent service configuration for propeller. + agent-service: + # -- The default agent service to use for plugin tasks. defaultAgent: + # -- The agent service endpoint propeller should connect to. endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000" + # -- Whether the connection from propeller to the agent service should use TLS. insecure: true + # -- The task types supported by the default agent. + supportedTaskTypes: + - sensor + # -- Uncomment to enable task type that uses Flyte Agent + # - bigquery_query_job_task # -- Labels for flyteagent pods podLabels: {}