From 1b7d05933d9376e4aef6c5e93c50d239cdb46034 Mon Sep 17 00:00:00 2001 From: Aaron Siddhartha Mondal Date: Thu, 28 Nov 2024 11:27:48 -0800 Subject: [PATCH] Merge scheduler and cas for K8s (#1506) There is no reason for these two to be separate. The new setup reduces complexity and potentially increases performance due to in-process communication. --- .github/workflows/lre.yaml | 23 ++-- .../chromium-example/build_chromium_tests.sh | 5 +- flake.nix | 1 + kubernetes/base/kustomization.yaml | 3 +- kubernetes/base/{cas.yaml => nativelink.yaml} | 43 ++++--- kubernetes/base/scheduler.yaml | 52 -------- kubernetes/base/worker.yaml | 6 +- .../components/gateway-routes/routes.yaml | 21 +--- kubernetes/configmaps/kustomization.yaml | 10 +- .../{cas.json => nativelink-config.json} | 118 ++++++++++++++---- kubernetes/configmaps/scheduler.json | 94 -------------- kubernetes/configmaps/worker.json | 13 +- modules/nativelink.nix | 1 - .../embedded/nativelink-gateways.yaml | 15 +-- native-cli/components/loadbalancer.go | 14 +-- .../docs/deployment-examples/kubernetes.mdx | 16 ++- 16 files changed, 155 insertions(+), 280 deletions(-) rename kubernetes/base/{cas.yaml => nativelink.yaml} (58%) delete mode 100644 kubernetes/base/scheduler.yaml rename kubernetes/configmaps/{cas.json => nativelink-config.json} (50%) delete mode 100644 kubernetes/configmaps/scheduler.json diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml index f68860c03..478b848f3 100644 --- a/.github/workflows/lre.yaml +++ b/.github/workflows/lre.yaml @@ -185,15 +185,10 @@ jobs: --timeout=15m \ nativelink" - - name: Wait for CAS + - name: Wait for NativeLink run: > nix develop --impure --command - bash -c "kubectl rollout status deploy/nativelink-cas" - - - name: Wait for scheduler - run: > - nix develop --impure --command - bash -c "kubectl rollout status deploy/nativelink-scheduler" + bash -c "kubectl rollout status deploy/nativelink" - name: Wait for worker run: > @@ -203,8 +198,7 @@ jobs: - name: Get gateway IPs id: gateway-ips run: | - echo "cache_ip=$(kubectl get gtw cache-gateway -o=jsonpath='{.status.addresses[0].value}')" >> "$GITHUB_ENV" - echo "scheduler_ip=$(kubectl get gtw scheduler-gateway -o=jsonpath='{.status.addresses[0].value}')" >> "$GITHUB_ENV" + echo "nativelink_ip=$(kubectl get gtw nativelink-gateway -o=jsonpath='{.status.addresses[0].value}')" >> "$GITHUB_ENV" - name: Print cluster state run: | @@ -213,10 +207,8 @@ jobs: kubectl get svc -A kubectl get deployments -A kubectl describe gtw - echo "cas" - kubectl logs -l app=nativelink-cas - echo "scheduler" - kubectl logs -l app=nativelink-scheduler + echo "nativelink" + kubectl logs -l app=nativelink echo "worker" kubectl logs -l app=nativelink-worker @@ -224,8 +216,7 @@ jobs: run: > nix develop --impure --command bash -c "bazel run \ - --remote_instance_name=main \ - --remote_cache=grpc://$cache_ip \ - --remote_executor=grpc://$scheduler_ip \ + --remote_cache=grpc://$nativelink_ip \ + --remote_executor=grpc://$nativelink_ip \ --verbose_failures \ @local-remote-execution//examples:hello_lre" diff --git a/deploy/chromium-example/build_chromium_tests.sh b/deploy/chromium-example/build_chromium_tests.sh index 1cd953852..e76fc1206 100755 --- a/deploy/chromium-example/build_chromium_tests.sh +++ b/deploy/chromium-example/build_chromium_tests.sh @@ -62,8 +62,7 @@ echo "Generating ninja projects" gn gen --args="use_remoteexec=true rbe_cfg_dir=\"../../buildtools/reclient_cfgs/linux\"" out/Default # Fetch cache and schedular IP address for passing to ninja -CACHE=$(kubectl get gtw cache-gateway -o=jsonpath='{.status.addresses[0].value}') -SCHEDULER=$(kubectl get gtw scheduler-gateway -o=jsonpath='{.status.addresses[0].value}') +NATIVELINK=$(kubectl get gtw nativelink-gateway -o=jsonpath='{.status.addresses[0].value}') echo "Starting autoninja build" -RBE_service=${SCHEDULER}:80 RBE_cas_service=${CACHE}:80 RBE_instance=main RBE_reclient_timeout=60m RBE_exec_timeout=4m RBE_alsologtostderr=true RBE_service_no_security=true RBE_service_no_auth=true RBE_local_resource_fraction=0.00001 RBE_automatic_auth=false RBE_gcert_refresh_timeout=20 RBE_compression_threshold=-1 RBE_metrics_namespace=main RBE_platform= RBE_experimental_credentials_helper= RBE_experimental_credentials_helper_args= RBE_log_http_calls=true RBE_use_rpc_credentials=false RBE_exec_strategy=remote_local_fallback autoninja -v -j 50 -C out/Default cc_unittests +RBE_service=${NATIVELINK}:80 RBE_cas_service=${NATIVELINK}:80 RBE_instance="" RBE_reclient_timeout=60m RBE_exec_timeout=4m RBE_alsologtostderr=true RBE_service_no_security=true RBE_service_no_auth=true RBE_local_resource_fraction=0.00001 RBE_automatic_auth=false RBE_gcert_refresh_timeout=20 RBE_compression_threshold=-1 RBE_metrics_namespace="" RBE_platform= RBE_experimental_credentials_helper= RBE_experimental_credentials_helper_args= RBE_log_http_calls=true RBE_use_rpc_credentials=false RBE_exec_strategy=remote_local_fallback autoninja -v -j 50 -C out/Default cc_unittests diff --git a/flake.nix b/flake.nix index eed093dee..df0da8547 100644 --- a/flake.nix +++ b/flake.nix @@ -513,6 +513,7 @@ pkgs.fluxcd pkgs.go pkgs.kustomize + pkgs.kubectx ## Web pkgs.bun # got patched to the newest version (v.1.1.25) diff --git a/kubernetes/base/kustomization.yaml b/kubernetes/base/kustomization.yaml index 92d4e29f7..3a160e2ac 100644 --- a/kubernetes/base/kustomization.yaml +++ b/kubernetes/base/kustomization.yaml @@ -2,8 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - cas.yaml - - scheduler.yaml + - nativelink.yaml - worker.yaml - ../configmaps diff --git a/kubernetes/base/cas.yaml b/kubernetes/base/nativelink.yaml similarity index 58% rename from kubernetes/base/cas.yaml rename to kubernetes/base/nativelink.yaml index d93f8481e..ec18df291 100644 --- a/kubernetes/base/cas.yaml +++ b/kubernetes/base/nativelink.yaml @@ -2,40 +2,41 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: nativelink-cas + name: nativelink spec: replicas: 1 selector: matchLabels: - app: nativelink-cas + app: nativelink template: metadata: labels: - app: nativelink-cas + app: nativelink spec: containers: - - name: nativelink-cas + - name: nativelink # This image will be edited by kustomize. image: nativelink env: - name: RUST_LOG value: info ports: + - containerPort: 9090 - containerPort: 50051 + - containerPort: 50052 - containerPort: 50061 - - containerPort: 50071 volumeMounts: - - name: cas-config - mountPath: /cas.json - subPath: cas.json + - name: nativelink-config + mountPath: /nativelink-config.json + subPath: nativelink-config.json - name: tls-volume mountPath: /root readOnly: true - args: ["/cas.json"] + args: ["/nativelink-config.json"] volumes: - - name: cas-config + - name: nativelink-config configMap: - name: cas + name: nativelink-config - name: tls-volume secret: secretName: tls-secret @@ -43,20 +44,24 @@ spec: apiVersion: v1 kind: Service metadata: - name: nativelink-cas + name: nativelink spec: selector: - app: nativelink-cas + app: nativelink ports: - - name: http + - name: metrics + protocol: TCP + port: 9090 + targetPort: 9090 + - name: grpc protocol: TCP port: 50051 targetPort: 50051 - - name: metrics + - name: grpcs + protocol: TCP + port: 50052 + targetPort: 50052 + - name: worker-api protocol: TCP port: 50061 targetPort: 50061 - - name: https - protocol: TCP - port: 50071 - targetPort: 50071 diff --git a/kubernetes/base/scheduler.yaml b/kubernetes/base/scheduler.yaml deleted file mode 100644 index 603d3d63e..000000000 --- a/kubernetes/base/scheduler.yaml +++ /dev/null @@ -1,52 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nativelink-scheduler -spec: - replicas: 1 - selector: - matchLabels: - app: nativelink-scheduler - template: - metadata: - labels: - app: nativelink-scheduler - spec: - containers: - - name: nativelink-scheduler - # This image will be edited by kustomize. - image: nativelink - env: - - name: RUST_LOG - value: info - - name: CAS_ENDPOINT - value: nativelink-cas - ports: - - containerPort: 50052 - volumeMounts: - - name: scheduler-config - mountPath: /scheduler.json - subPath: scheduler.json - args: ["/scheduler.json"] - volumes: - - name: scheduler-config - configMap: - name: scheduler ---- -apiVersion: v1 -kind: Service -metadata: - name: nativelink-scheduler -spec: - selector: - app: nativelink-scheduler - ports: - - name: scheduler - protocol: TCP - port: 50052 - targetPort: 50052 - - name: worker-api - protocol: TCP - port: 50061 - targetPort: 50061 diff --git a/kubernetes/base/worker.yaml b/kubernetes/base/worker.yaml index dcf57bc2c..d683b6b0d 100644 --- a/kubernetes/base/worker.yaml +++ b/kubernetes/base/worker.yaml @@ -28,10 +28,8 @@ spec: env: - name: RUST_LOG value: info - - name: CAS_ENDPOINT - value: nativelink-cas - - name: SCHEDULER_ENDPOINT - value: nativelink-scheduler + - name: NATIVELINK_ENDPOINT + value: nativelink volumeMounts: - name: worker-config mountPath: /worker.json diff --git a/kubernetes/components/gateway-routes/routes.yaml b/kubernetes/components/gateway-routes/routes.yaml index fe4ac9ee3..065b8f633 100644 --- a/kubernetes/components/gateway-routes/routes.yaml +++ b/kubernetes/components/gateway-routes/routes.yaml @@ -2,25 +2,12 @@ apiVersion: gateway.networking.k8s.io/v1alpha2 kind: GRPCRoute metadata: - name: cache-route + name: nativelink-route spec: parentRefs: - - name: cache-gateway - sectionName: cache-gateway + - name: nativelink-gateway + sectionName: nativelink-gateway rules: - backendRefs: - - name: nativelink-cas + - name: nativelink port: 50051 ---- -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: GRPCRoute -metadata: - name: scheduler-route -spec: - parentRefs: - - name: scheduler-gateway - sectionName: scheduler-gateway - rules: - - backendRefs: - - name: nativelink-scheduler - port: 50052 diff --git a/kubernetes/configmaps/kustomization.yaml b/kubernetes/configmaps/kustomization.yaml index 91752af24..601abcdd2 100644 --- a/kubernetes/configmaps/kustomization.yaml +++ b/kubernetes/configmaps/kustomization.yaml @@ -3,15 +3,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization configMapGenerator: - - name: cas + - name: nativelink-config files: - - cas.json - options: - annotations: - kustomize.toolkit.fluxcd.io/substitute: disabled - - name: scheduler - files: - - scheduler.json + - nativelink-config.json options: annotations: kustomize.toolkit.fluxcd.io/substitute: disabled diff --git a/kubernetes/configmaps/cas.json b/kubernetes/configmaps/nativelink-config.json similarity index 50% rename from kubernetes/configmaps/cas.json rename to kubernetes/configmaps/nativelink-config.json index e26b9c1cb..21ad84b63 100644 --- a/kubernetes/configmaps/cas.json +++ b/kubernetes/configmaps/nativelink-config.json @@ -15,8 +15,7 @@ "content_path": "~/.cache/nativelink/content_path-cas", "temp_path": "~/.cache/nativelink/tmp_path-cas", "eviction_policy": { - // 10gb. - "max_bytes": 10000000000, + "max_bytes": "10Gb", } } } @@ -31,8 +30,7 @@ "content_path": "~/.cache/nativelink/content_path-ac", "temp_path": "~/.cache/nativelink/tmp_path-ac", "eviction_policy": { - // 500mb. - "max_bytes": 500000000, + "max_bytes": "500Mb", } } }, @@ -44,7 +42,46 @@ } } }, + "schedulers": { + "MAIN_SCHEDULER": { + // TODO(adams): use the right scheduler because reclient doesn't use the cached results? + // TODO(adams): max_bytes_per_stream + "simple": { + "supported_platform_properties": { + "cpu_count": "priority", + "memory_kb": "priority", + "network_kbps": "priority", + "disk_read_iops": "priority", + "disk_read_bps": "priority", + "disk_write_iops": "priority", + "disk_write_bps": "priority", + "shm_size": "priority", + "gpu_count": "priority", + "gpu_model": "priority", + "cpu_vendor": "priority", + "cpu_arch": "priority", + "cpu_model": "priority", + "kernel_version": "priority", + "OSFamily": "priority", + "container-image": "priority", + } + } + } + }, "servers": [{ + // Only publish metrics on a private port. + "listener": { + "http": { + "socket_address": "0.0.0.0:9090" + } + }, + "services": { + "experimental_prometheus": { + "path": "/metrics" + } + } + }, + { "listener": { "http": { "socket_address": "0.0.0.0:50051" @@ -52,40 +89,39 @@ }, "services": { "cas": { - "main": { + "": { "cas_store": "CAS_MAIN_STORE" } }, "ac": { - "main": { + "": { "ac_store": "AC_MAIN_STORE" } }, - "capabilities": {}, + "capabilities": { + "": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + }, + "execution": { + "": { + "cas_store": "CAS_MAIN_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, "bytestream": { "cas_stores": { - "main": "CAS_MAIN_STORE", + "": "CAS_MAIN_STORE", }, } } }, { - // Only publish metrics on a private port. "listener": { "http": { - "socket_address": "0.0.0.0:50061" - } - }, - "services": { - "experimental_prometheus": { - "path": "/metrics" - } - } - }, - { - "listener": { - "http": { - "socket_address": "0.0.0.0:50071", + "socket_address": "0.0.0.0:50052", "tls": { "cert_file": "/root/example-do-not-use-in-prod-rootca.crt", "key_file": "/root/example-do-not-use-in-prod-key.pem" @@ -94,20 +130,48 @@ }, "services": { "cas": { - "main": { + "": { "cas_store": "CAS_MAIN_STORE" } }, "ac": { - "main": { + "": { "ac_store": "AC_MAIN_STORE" } }, - "capabilities": {}, + "capabilities": { + "": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + }, + "execution": { + "": { + "cas_store": "CAS_MAIN_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, "bytestream": { "cas_stores": { - "main": "CAS_MAIN_STORE", - } + "": "CAS_MAIN_STORE", + }, + } + } + }, + { + "listener": { + "http": { + "socket_address": "0.0.0.0:50061", + } + }, + "services": { + // Note: This should be served on a different port, because it has + // a different permission set than the other services. + // In other words, this service is a backend api. The ones above + // are a frontend api. + "worker_api": { + "scheduler": "MAIN_SCHEDULER", }, "health": {}, } diff --git a/kubernetes/configmaps/scheduler.json b/kubernetes/configmaps/scheduler.json deleted file mode 100644 index e41060209..000000000 --- a/kubernetes/configmaps/scheduler.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "stores": { - "GRPC_LOCAL_STORE": { - // Note: This file is used to test GRPC store. - "grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "store_type": "cas" - } - }, - "GRPC_LOCAL_AC_STORE": { - // Note: This file is used to test GRPC store. - "grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "store_type": "ac" - } - } - }, - "schedulers": { - "MAIN_SCHEDULER": { - // TODO(adams): use the right scheduler because reclient doesn't use the cached results? - // TODO(adams): max_bytes_per_stream - "simple": { - "supported_platform_properties": { - "cpu_count": "priority", - "memory_kb": "priority", - "network_kbps": "priority", - "disk_read_iops": "priority", - "disk_read_bps": "priority", - "disk_write_iops": "priority", - "disk_write_bps": "priority", - "shm_size": "priority", - "gpu_count": "priority", - "gpu_model": "priority", - "cpu_vendor": "priority", - "cpu_arch": "priority", - "cpu_model": "priority", - "kernel_version": "priority", - "OSFamily": "priority", - "container-image": "priority", - } - } - } - }, - "servers": [{ - "listener": { - "http": { - "socket_address": "0.0.0.0:50052" - } - }, - "services": { - "ac": { - "main": { - "ac_store": "GRPC_LOCAL_AC_STORE" - } - }, - "execution": { - "main": { - "cas_store": "GRPC_LOCAL_STORE", - "scheduler": "MAIN_SCHEDULER", - } - }, - "capabilities": { - "main": { - "remote_execution": { - "scheduler": "MAIN_SCHEDULER", - } - } - } - } - }, - { - "listener": { - "http": { - "socket_address": "0.0.0.0:50061", - } - }, - "services": { - // Note: This should be served on a different port, because it has - // a different permission set than the other services. - // In other words, this service is a backend api. The ones above - // are a frontend api. - "worker_api": { - "scheduler": "MAIN_SCHEDULER", - }, - "health": {}, - } - }] -} diff --git a/kubernetes/configmaps/worker.json b/kubernetes/configmaps/worker.json index 2a3d2911d..b3dccb448 100644 --- a/kubernetes/configmaps/worker.json +++ b/kubernetes/configmaps/worker.json @@ -3,9 +3,9 @@ "GRPC_LOCAL_STORE": { // Note: This file is used to test GRPC store. "grpc": { - "instance_name": "main", + "instance_name": "", "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + {"address": "grpc://${NATIVELINK_ENDPOINT:-127.0.0.1}:50051"} ], "store_type": "cas" } @@ -13,9 +13,9 @@ "GRPC_LOCAL_AC_STORE": { // Note: This file is used to test GRPC store. "grpc": { - "instance_name": "main", + "instance_name": "", "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + {"address": "grpc://${NATIVELINK_ENDPOINT:-127.0.0.1}:50051"} ], "store_type": "ac" } @@ -27,8 +27,7 @@ "content_path": "~/.cache/nativelink/data-worker-test/content_path-cas", "temp_path": "~/.cache/nativelink/data-worker-test/tmp_path-cas", "eviction_policy": { - // 10gb. - "max_bytes": 10000000000, + "max_bytes": "10Gb" } } }, @@ -43,7 +42,7 @@ "workers": [{ "local": { "worker_api_endpoint": { - "uri": "grpc://${SCHEDULER_ENDPOINT:-127.0.0.1}:50061", + "uri": "grpc://${NATIVELINK_ENDPOINT:-127.0.0.1}:50061", }, "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE", "upload_action_result": { diff --git a/modules/nativelink.nix b/modules/nativelink.nix index eb7a498b2..5ff010747 100644 --- a/modules/nativelink.nix +++ b/modules/nativelink.nix @@ -17,7 +17,6 @@ defaultConfig = [ "--remote_cache=${config.endpoint}" "--remote_header=x-nativelink-api-key=${config.api-key}" - "--remote_instance_name=main" "--remote_header=x-nativelink-project=nativelink-ci" "--nogenerate_json_trace_profile" "--remote_upload_local_results=false" diff --git a/native-cli/components/embedded/nativelink-gateways.yaml b/native-cli/components/embedded/nativelink-gateways.yaml index 960289ad0..cc8829b60 100644 --- a/native-cli/components/embedded/nativelink-gateways.yaml +++ b/native-cli/components/embedded/nativelink-gateways.yaml @@ -4,22 +4,11 @@ apiVersion: gateway.networking.k8s.io/v1beta1 kind: Gateway metadata: - name: cache-gateway + name: nativelink-gateway spec: gatewayClassName: cilium listeners: - - name: cache-gateway - protocol: HTTP - port: 80 ---- -apiVersion: gateway.networking.k8s.io/v1beta1 -kind: Gateway -metadata: - name: scheduler-gateway -spec: - gatewayClassName: cilium - listeners: - - name: scheduler-gateway + - name: nativelink-gateway protocol: HTTP port: 80 --- diff --git a/native-cli/components/loadbalancer.go b/native-cli/components/loadbalancer.go index 61f3a421d..1820d235a 100644 --- a/native-cli/components/loadbalancer.go +++ b/native-cli/components/loadbalancer.go @@ -156,8 +156,7 @@ func gatewayAdded( len(gateway.Status.Addresses) > 0 && !addedGateways[gateway.Name] { gatewayIP := gateway.Status.Addresses[0].Value - http2 := gateway.Name == "cache-gateway" || - gateway.Name == "scheduler-gateway" + http2 := gateway.Name == "nativelink-gateway" gatewayInfo := InternalGateway{ Name: gateway.Name, IP: gatewayIP, @@ -305,12 +304,11 @@ func (component *Loadbalancer) Install( ctx.Context(), gatewayClientset, map[string]bool{ - "scheduler-gateway": false, - "cache-gateway": false, - "el-gateway": false, - "hubble-gateway": false, - "tkn-gateway": false, - "capacitor-gateway": false, + "nativelink-gateway": false, + "el-gateway": false, + "hubble-gateway": false, + "tkn-gateway": false, + "capacitor-gateway": false, }, ), component.Gateways) if err != nil { diff --git a/web/platform/src/content/docs/docs/deployment-examples/kubernetes.mdx b/web/platform/src/content/docs/docs/deployment-examples/kubernetes.mdx index 3fd7c6417..34caf3ab9 100644 --- a/web/platform/src/content/docs/docs/deployment-examples/kubernetes.mdx +++ b/web/platform/src/content/docs/docs/deployment-examples/kubernetes.mdx @@ -89,22 +89,20 @@ via your local docker network. You can pass the Gateway addresses to Bazel invocations to make builds run against the cluster: ```bash -CACHE=$(kubectl get gtw cache-gateway -o=jsonpath='{.status.addresses[0].value}') -SCHEDULER=$(kubectl get gtw scheduler-gateway -o=jsonpath='{.status.addresses[0].value}') +NATIVELINK=$(kubectl get gtw nativelink-gateway -o=jsonpath='{.status.addresses[0].value}') -echo "Cache IP: $CACHE" -echo "Scheduler IP: $SCHEDULER" +echo "NativeLink IP: $NATIVELINK" bazel build \ - --remote_cache=grpc://$CACHE \ - --remote_executor=grpc://$SCHEDULER \ + --remote_cache=grpc://$NATIVELINK \ + --remote_executor=grpc://$NATIVELINK \ //local-remote-execution/examples:hello_lre ``` :::caution While the Dashboard ports are static, the NativeLink endpoints aren't (yet). -If you shut down the cluster and reboot it, the `$CACHE` and `$SCHEDULER` IP -addresses will change. +If you shut down the cluster and reboot it, the `$NATIVELINK` IP addresses will +change. ::: :::tip @@ -114,7 +112,7 @@ resolve environment variables: ```bash # .bazelrc.user build --remote_cache=grpc://172.20.255.4 -build --remote_executor=grpc://172.20.255.5 +build --remote_executor=grpc://172.20.255.4 ``` ```bash