[k8s] Fix show-gpus when limited permissions are available (#4208)

* fixes

* fixes

* fixes

* lint

* update error str

docs/source/cloud-setup/cloud-permissions/kubernetes.rst

@@ -96,7 +96,29 @@ SkyPilot requires permissions equivalent to the following roles to be able to ma
 
 These roles must apply to both the user account configured in the kubeconfig file and the service account used by SkyPilot (if configured).
 
-If your tasks use object store mounting or require access to ingress resources, you will need to grant additional permissions as described below.
+If you need to view real-time GPU availability with ``sky show-gpus``, your tasks use object store mounting or your tasks require access to ingress resources, you will need to grant additional permissions as described below.
+
+Permissions for ``sky show-gpus``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``sky show-gpus`` needs to list all pods across all namespaces to calculate GPU availability. To do this, SkyPilot needs the ``get`` and ``list`` permissions for pods in a ``ClusterRole``:
+
+.. code-block:: yaml
+
+    apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+        name: sky-sa-cluster-role-pod-reader
+    rules:
+      - apiGroups: [""]
+        resources: ["pods"]
+        verbs: ["get", "list"]
+
+
+.. tip::
+
+    If this role is not granted to the service account, ``sky show-gpus`` will still work but it will only show the total GPUs on the nodes, not the number of free GPUs.
+
 
 Permissions for Object Store Mounting
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -225,6 +247,9 @@ To create a service account that has all necessary permissions for SkyPilot (inc
       - apiGroups: ["networking.k8s.io"]   # Required for exposing services through ingresses
         resources: ["ingressclasses"]
         verbs: ["get", "list", "watch"]
+      - apiGroups: [""]                 # Required for `sky show-gpus` command
+        resources: ["pods"]
+        verbs: ["get", "list"]
     ---
     # ClusterRoleBinding for the service account
     apiVersion: rbac.authorization.k8s.io/v1

docs/source/reference/kubernetes/kubernetes-getting-started.rst

@@ -296,6 +296,11 @@ FAQs
                 volumeMounts:       # Custom volume mounts for the pod
                   - mountPath: /foo
                     name: example-volume
+                resources:          # Custom resource requests and limits
+                  requests:
+                    rdma/rdma_shared_device_a: 1
+                  limits:
+                    rdma/rdma_shared_device_a: 1
             volumes:
               - name: example-volume
                 hostPath:

sky/cli.py

@@ -3102,6 +3102,7 @@ def show_gpus(
     kubernetes_autoscaling = kubernetes_utils.get_autoscaler_type() is not None
     kubernetes_is_enabled = sky_clouds.cloud_in_iterable(
         sky_clouds.Kubernetes(), global_user_state.get_cached_enabled_clouds())
+    no_permissions_str = '<no permissions>'
 
     def _list_to_str(lst):
         return ', '.join([str(e) for e in lst])
@@ -3146,9 +3147,11 @@ def _get_kubernetes_realtime_gpu_table(
                             debug_msg)
             raise ValueError(full_err_msg)
         for gpu, _ in sorted(counts.items()):
+            available_qty = available[gpu] if available[gpu] != -1 else (
+                no_permissions_str)
             realtime_gpu_table.add_row([
                 gpu,
-                _list_to_str(counts.pop(gpu)), capacity[gpu], available[gpu]
+                _list_to_str(counts.pop(gpu)), capacity[gpu], available_qty
             ])
         return realtime_gpu_table
 
@@ -3158,10 +3161,11 @@ def _get_kubernetes_node_info_table(context: Optional[str]):
 
         node_info_dict = kubernetes_utils.get_kubernetes_node_info(context)
         for node_name, node_info in node_info_dict.items():
+            available = node_info.free['nvidia.com/gpu'] if node_info.free[
+                'nvidia.com/gpu'] != -1 else no_permissions_str
             node_table.add_row([
                 node_name, node_info.gpu_type,
-                node_info.total['nvidia.com/gpu'],
-                node_info.free['nvidia.com/gpu']
+                node_info.total['nvidia.com/gpu'], available
             ])
         return node_table
 

sky/clouds/service_catalog/kubernetes_catalog.py

@@ -10,6 +10,7 @@
 from sky import check as sky_check
 from sky import sky_logging
 from sky.adaptors import common as adaptors_common
+from sky.adaptors import kubernetes
 from sky.clouds import Kubernetes
 from sky.clouds.service_catalog import CloudFilter
 from sky.clouds.service_catalog import common
@@ -22,6 +23,8 @@
 else:
     pd = adaptors_common.LazyImport('pandas')
 
+logger = sky_logging.init_logger(__name__)
+
 _PULL_FREQUENCY_HOURS = 7
 
 # We keep pull_frequency_hours so we can remotely update the default image paths
@@ -77,6 +80,11 @@ def list_accelerators_realtime(
     require_price: bool = True
 ) -> Tuple[Dict[str, List[common.InstanceTypeInfo]], Dict[str, int], Dict[str,
                                                                           int]]:
+    """List accelerators in the Kubernetes cluster.
+
+    If the user does not have sufficient permissions to list pods in all
+    namespaces, the function will return free GPUs as -1.
+    """
     # TODO(romilb): This should be refactored to use get_kubernetes_node_info()
     #   function from kubernetes_utils.
     del all_regions, require_price  # Unused.
@@ -108,7 +116,17 @@ def list_accelerators_realtime(
     key = label_formatter.get_label_key()
     nodes = kubernetes_utils.get_kubernetes_nodes(context)
     # Get the pods to get the real-time GPU usage
-    pods = kubernetes_utils.get_all_pods_in_kubernetes_cluster(context)
+    try:
+        pods = kubernetes_utils.get_all_pods_in_kubernetes_cluster(context)
+    except kubernetes.api_exception() as e:
+        if e.status == 403:
+            logger.warning('Failed to get pods in the Kubernetes cluster '
+                           '(forbidden). Please check if your account has '
+                           'necessary permissions to list pods. Realtime GPU '
+                           'availability information may be incorrect.')
+            pods = None
+        else:
+            raise
     # Total number of GPUs in the cluster
     total_accelerators_capacity: Dict[str, int] = {}
     # Total number of GPUs currently available in the cluster
@@ -141,6 +159,21 @@ def list_accelerators_realtime(
                 if accelerator_count not in accelerators_qtys:
                     accelerators_qtys.add((accelerator_name, accelerator_count))
 
+            if accelerator_count >= min_quantity_filter:
+                quantized_count = (min_quantity_filter *
+                                   (accelerator_count // min_quantity_filter))
+                if accelerator_name not in total_accelerators_capacity:
+                    total_accelerators_capacity[
+                        accelerator_name] = quantized_count
+                else:
+                    total_accelerators_capacity[
+                        accelerator_name] += quantized_count
+
+            if pods is None:
+                # If we can't get the pods, we can't get the GPU usage
+                total_accelerators_available[accelerator_name] = -1
+                continue
+
             for pod in pods:
                 # Get all the pods running on the node
                 if (pod.spec.node_name == node.metadata.name and
@@ -155,16 +188,6 @@ def list_accelerators_realtime(
 
             accelerators_available = accelerator_count - allocated_qty
 
-            if accelerator_count >= min_quantity_filter:
-                quantized_count = (min_quantity_filter *
-                                   (accelerator_count // min_quantity_filter))
-                if accelerator_name not in total_accelerators_capacity:
-                    total_accelerators_capacity[
-                        accelerator_name] = quantized_count
-                else:
-                    total_accelerators_capacity[
-                        accelerator_name] += quantized_count
-
             if accelerator_name not in total_accelerators_available:
                 total_accelerators_available[accelerator_name] = 0
             if accelerators_available >= min_quantity_filter:

sky/provision/kubernetes/utils.py

@@ -1801,13 +1801,22 @@ def get_kubernetes_node_info(
     number of GPUs available on the node and the number of free GPUs on the
     node.
 
+    If the user does not have sufficient permissions to list pods in all
+    namespaces, the function will return free GPUs as -1.
+
     Returns:
         Dict[str, KubernetesNodeInfo]: Dictionary containing the node name as
             key and the KubernetesNodeInfo object as value
     """
     nodes = get_kubernetes_nodes(context)
     # Get the pods to get the real-time resource usage
-    pods = get_all_pods_in_kubernetes_cluster(context)
+    try:
+        pods = get_all_pods_in_kubernetes_cluster(context)
+    except kubernetes.api_exception() as e:
+        if e.status == 403:
+            pods = None
+        else:
+            raise
 
     label_formatter, _ = detect_gpu_label_formatter(context)
     if not label_formatter:
@@ -1828,19 +1837,22 @@ def get_kubernetes_node_info(
         accelerator_count = int(node.status.allocatable.get(
             'nvidia.com/gpu', 0))
 
-        for pod in pods:
-            # Get all the pods running on the node
-            if (pod.spec.node_name == node.metadata.name and
-                    pod.status.phase in ['Running', 'Pending']):
-                # Iterate over all the containers in the pod and sum the
-                # GPU requests
-                for container in pod.spec.containers:
-                    if container.resources.requests:
-                        allocated_qty += int(
-                            container.resources.requests.get(
-                                'nvidia.com/gpu', 0))
-
-        accelerators_available = accelerator_count - allocated_qty
+        if pods is None:
+            accelerators_available = -1
+
+        else:
+            for pod in pods:
+                # Get all the pods running on the node
+                if (pod.spec.node_name == node.metadata.name and
+                        pod.status.phase in ['Running', 'Pending']):
+                    # Iterate over all the containers in the pod and sum the
+                    # GPU requests
+                    for container in pod.spec.containers:
+                        if container.resources.requests:
+                            allocated_qty += int(
+                                container.resources.requests.get(
+                                    'nvidia.com/gpu', 0))
+            accelerators_available = accelerator_count - allocated_qty
 
         node_info_dict[node.metadata.name] = KubernetesNodeInfo(
             name=node.metadata.name,

sky/utils/kubernetes/generate_kubeconfig.sh

@@ -112,6 +112,9 @@ rules:
   - apiGroups: ["networking.k8s.io"]   # Required for exposing services through ingresses
     resources: ["ingressclasses"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: [""]                 # Required for sky show-gpus command
+    resources: ["pods"]
+    verbs: ["get", "list"]
 ---
 # ClusterRoleBinding for the service account
 apiVersion: rbac.authorization.k8s.io/v1