X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2FKubernetesUtils.py;h=69e7e832b4d7d0fc4bbcdb6409d22f3b73fe6240;hp=5faa056ddc946d9f46d7c9be269946293ea6b5be;hb=b015ecf08aba9659d0261dfa6eb0da9ab4950b92;hpb=9a261ea61549fc6a5c23369d2e236b002dc35038

diff --git a/resources/libraries/python/KubernetesUtils.py b/resources/libraries/python/KubernetesUtils.py
index 5faa056ddc..69e7e832b4 100644
--- a/resources/libraries/python/KubernetesUtils.py
+++ b/resources/libraries/python/KubernetesUtils.py
@@ -50,6 +50,9 @@ class KubernetesUtils(object):
             raise RuntimeError('Failed to setup Kubernetes on {node}.'
                                .format(node=node['host']))
 
+        KubernetesUtils.wait_for_kubernetes_pods_on_node(node,
+                                                         nspace='kube-system')
+
     @staticmethod
     def setup_kubernetes_on_all_duts(nodes):
         """Set up Kubernetes on all DUTs.
@@ -243,6 +246,36 @@ class KubernetesUtils(object):
                 KubernetesUtils.describe_kubernetes_resource_on_node(node,
                                                                      rtype)
 
+    @staticmethod
+    def get_kubernetes_logs_on_node(node, nspace='csit'):
+        """Get Kubernetes logs on node.
+
+        :param node: DUT node.
+        :param nspace: Kubernetes namespace.
+        :type node: dict
+        :type nspace: str
+        """
+        ssh = SSH()
+        ssh.connect(node)
+
+        cmd = "for p in $(kubectl get pods -n {namespace} --no-headers"\
+            " | cut -f 1 -d ' '); do echo $p; kubectl logs -n {namespace} $p; "\
+            "done".format(namespace=nspace)
+        ssh.exec_command(cmd, timeout=120)
+
+    @staticmethod
+    def get_kubernetes_logs_on_all_duts(nodes, nspace='csit'):
+        """Get Kubernetes logs on all DUTs.
+
+        :param nodes: Topology nodes.
+        :param nspace: Kubernetes namespace.
+        :type nodes: dict
+        :type nspace: str
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                KubernetesUtils.get_kubernetes_logs_on_node(node, nspace)
+
     @staticmethod
     def reset_kubernetes_on_node(node):
         """Reset Kubernetes on node.
@@ -272,23 +305,30 @@ class KubernetesUtils(object):
                 KubernetesUtils.reset_kubernetes_on_node(node)
 
     @staticmethod
-    def wait_for_kubernetes_pods_on_node(node):
+    def wait_for_kubernetes_pods_on_node(node, nspace='csit'):
         """Wait for Kubernetes PODs to become in 'Running' state on node.
 
         :param node: DUT node.
+        :param nspace: Kubernetes namespace.
         :type node: dict
+        :type nspace: str
         :raises RuntimeError: If Kubernetes PODs are not ready.
         """
         ssh = SSH()
         ssh.connect(node)
 
-        cmd = 'kubectl get -n csit pods --no-headers'
+        cmd = 'kubectl get -n {namespace} pods --no-headers'\
+            .format(namespace=nspace)
         for _ in range(48):
             (ret_code, stdout, _) = ssh.exec_command_sudo(cmd, timeout=120)
             if int(ret_code) == 0:
                 ready = True
                 for line in stdout.splitlines():
-                    if 'Running' not in line:
+                    try:
+                        state = line.split()[1].split('/')
+                        ready = True if 'Running' in line and\
+                            state == state[::-1] else False
+                    except ValueError, IndexError:
                         ready = False
                 if ready:
                     break
@@ -298,15 +338,17 @@ class KubernetesUtils(object):
                                .format(node=node['host']))
 
     @staticmethod
-    def wait_for_kubernetes_pods_on_all_duts(nodes):
+    def wait_for_kubernetes_pods_on_all_duts(nodes, nspace='csit'):
         """Wait for Kubernetes PODs to become in Running state on all DUTs.
 
         :param nodes: Topology nodes.
+        :param nspace: Kubernetes namespace.
         :type nodes: dict
+        :type nspace: str
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
-                KubernetesUtils.wait_for_kubernetes_pods_on_node(node)
+                KubernetesUtils.wait_for_kubernetes_pods_on_node(node, nspace)
 
     @staticmethod
     def create_kubernetes_vswitch_startup_config(**kwargs):