Merge pull request #425 from chengcongdu/develop

add node READY state check for TAS

gke-topology-scheduler/schedule-daemon.py

@@ -286,6 +286,16 @@ def find_schedulable_nodes(
       )
       continue
 
+    # skip nodes that is not in Ready state
+    if any(
+      condition.type == "Ready" and condition.status != "True" for condition in node.status.conditions
+      ):
+        logging.info(
+          'Skipping node %s because it is NotReady',
+          node_name
+        )
+        continue
+
     allocatable = node.status.allocatable
     used_cpu, used_memory, used_gpu = 0, 0, 0
 

gpudirect-tcpxo/topology-scheduler/schedule-daemon.py

@@ -145,6 +145,7 @@ def find_schedulable_nodes(nodes, pods, tolerated_taints):
       continue
 
     skip_node = False
+    # check node taints
     if node.spec.taints is not None:
       for t in node.spec.taints:
         if t.key not in tolerated_taint_dict:
@@ -154,8 +155,14 @@ def find_schedulable_nodes(nodes, pods, tolerated_taints):
         else:
           tol = tolerated_taint_dict[t.key]
           if tol.operator == "Equal" and tol.value != t.value:
+            print(f'Skipping node {node_name} because it is tainted with key {t.key} with value {t.value}')
             skip_node = True
             break
+    # check node status
+    if any(condition.type == "Ready" and condition.status != "True" for condition in node.status.conditions):
+      print(f'Skipping node {node_name} because it is NotReady')
+      skip_node = True
+      break
 
     if skip_node:
       continue
@@ -321,7 +328,9 @@ def schedule_pod_on_node(v1, pod_name, pod_namespace, node, gate_name):
 
       v1.replace_namespaced_pod(pod_name, pod_namespace, pod)
 
-      print(f'Pod {pod_namespace}/{pod_name} scheduled on {node['name']} with topology: {node_topology_key(node)}')
+      print(
+        'Pod %s/%s scheduled on %s with topology %s', pod_namespace, pod_name, node['name'], node_topology_key(node)
+      )
   except ApiException as e:
     print(f'Exception when removing scheduling gate: {e}')