1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Library to control Kubernetes kubectl."""
19 from resources.libraries.python.constants import Constants
20 from resources.libraries.python.topology import NodeType
21 from resources.libraries.python.ssh import SSH
22 from resources.libraries.python.CpuUtils import CpuUtils
23 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
25 __all__ = ["KubernetesUtils"]
28 class KubernetesUtils(object):
29 """Kubernetes utilities class."""
32 """Initialize KubernetesUtils class."""
36 def setup_kubernetes_on_node(node):
37 """Set up Kubernetes on node.
39 :param node: DUT node.
41 :raises RuntimeError: If Kubernetes setup failed on node.
46 cmd = '{dir}/{lib}/k8s_setup.sh '.format(dir=Constants.REMOTE_FW_DIR,
47 lib=Constants.RESOURCES_LIB_SH)
48 (ret_code, _, _) = ssh.exec_command(cmd, timeout=120)
49 if int(ret_code) != 0:
50 raise RuntimeError('Failed to setup Kubernetes on {node}.'
51 .format(node=node['host']))
53 KubernetesUtils.wait_for_kubernetes_pods_on_node(node,
57 def setup_kubernetes_on_all_duts(nodes):
58 """Set up Kubernetes on all DUTs.
60 :param nodes: Topology nodes.
63 for node in nodes.values():
64 if node['type'] == NodeType.DUT:
65 KubernetesUtils.setup_kubernetes_on_node(node)
68 def apply_kubernetes_resource_on_node(node, yaml_file, **kwargs):
69 """Apply Kubernetes resource on node.
71 :param node: DUT node.
72 :param yaml_file: YAML configuration file.
73 :param kwargs: Key-value pairs to replace in YAML template.
77 :raises RuntimeError: If applying Kubernetes template failed.
82 stream = file('{tpl}/{yaml}'.format(tpl=Constants.RESOURCES_TPL_K8S,
85 for data in yaml.load_all(stream):
86 data = reduce(lambda a, kv: a.replace(*kv), kwargs.iteritems(),
87 yaml.dump(data, default_flow_style=False))
88 # Workaround to avoid using RAW string anotated with | in YAML as
89 # library + bash is misinterpreting spaces.
90 data = data.replace('.conf:\n', '.conf: |\n')
91 cmd = 'cat <<EOF | kubectl apply -f - \n{data}\nEOF'.format(
93 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
94 if int(ret_code) != 0:
95 raise RuntimeError('Failed to apply Kubernetes template {yaml} '
96 'on {node}.'.format(yaml=yaml_file,
100 def apply_kubernetes_resource_on_all_duts(nodes, yaml_file, **kwargs):
101 """Apply Kubernetes resource on all DUTs.
103 :param nodes: Topology nodes.
104 :param yaml_file: YAML configuration file.
105 :param kwargs: Key-value pairs to replace in YAML template.
110 for node in nodes.values():
111 if node['type'] == NodeType.DUT:
112 KubernetesUtils.apply_kubernetes_resource_on_node(node,
117 def create_kubernetes_cm_from_file_on_node(node, name, key, src_file):
118 """Create Kubernetes ConfigMap from file on node.
120 :param node: DUT node.
121 :param name: ConfigMap name.
122 :param key: Key (destination file).
123 :param src_file: Source file.
128 :raises RuntimeError: If creating Kubernetes ConfigMap failed.
133 cmd = 'kubectl create -n csit configmap {name} --from-file={key}='\
134 '{src_file}'.format(name=name, key=key, src_file=src_file)
135 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
136 if int(ret_code) != 0:
137 raise RuntimeError('Failed to create Kubernetes ConfigMap {name} '
138 'on {node}.'.format(name=name,
142 def create_kubernetes_cm_from_file_on_all_duts(nodes, name, key, src_file):
143 """Create Kubernetes ConfigMap from file on all DUTs.
145 :param nodes: Topology nodes.
146 :param name: ConfigMap name.
147 :param key: Key (destination file).
148 :param src_file: Source file.
154 for node in nodes.values():
155 if node['type'] == NodeType.DUT:
156 KubernetesUtils.create_kubernetes_cm_from_file_on_node(node,
162 def delete_kubernetes_resource_on_node(node, rtype='po,cm', name=None):
163 """Delete Kubernetes resource on node.
165 :param node: DUT node.
166 :param rtype: Kubernetes resource type.
167 :param name: Name of resource.
171 :raises RuntimeError: If deleting Kubernetes resource failed.
176 name = '{name}'.format(name=name) if name else '--all'
178 cmd = 'kubectl delete -n csit {rtype} {name}'\
179 .format(rtype=rtype, name=name)
180 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
181 if int(ret_code) != 0:
182 raise RuntimeError('Failed to delete Kubernetes resources in CSIT '
183 'namespace on {node}.'.format(node=node['host']))
185 cmd = 'kubectl get -n csit pods --no-headers'
187 (ret_code, stdout, _) = ssh.exec_command_sudo(cmd, timeout=120)
188 if int(ret_code) == 0:
190 for line in stdout.splitlines():
191 if 'No resources found.' not in line:
197 raise RuntimeError('Failed to delete Kubernetes resources in CSIT '
198 'namespace on {node}.'.format(node=node['host']))
201 def delete_kubernetes_resource_on_all_duts(nodes, rtype='po,cm', name=None):
202 """Delete all Kubernetes resource on all DUTs.
204 :param nodes: Topology nodes.
205 :param rtype: Kubernetes resource type.
206 :param name: Name of resource.
211 for node in nodes.values():
212 if node['type'] == NodeType.DUT:
213 KubernetesUtils.delete_kubernetes_resource_on_node(node, rtype,
217 def describe_kubernetes_resource_on_node(node, rtype='po,cm'):
218 """Describe Kubernetes resource on node.
220 :param node: DUT node.
221 :param rtype: Kubernetes resource type.
224 :raises RuntimeError: If describing Kubernetes resource failed.
229 cmd = 'kubectl describe -n csit {rtype}'.format(rtype=rtype)
230 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
231 if int(ret_code) != 0:
232 raise RuntimeError('Failed to describe Kubernetes resource on '
233 '{node}.'.format(node=node['host']))
236 def describe_kubernetes_resource_on_all_duts(nodes, rtype='po,cm'):
237 """Describe Kubernetes resource on all DUTs.
239 :param nodes: Topology nodes.
240 :param rtype: Kubernetes resource type.
244 for node in nodes.values():
245 if node['type'] == NodeType.DUT:
246 KubernetesUtils.describe_kubernetes_resource_on_node(node,
250 def get_kubernetes_logs_on_node(node, nspace='csit'):
251 """Get Kubernetes logs on node.
253 :param node: DUT node.
254 :param nspace: Kubernetes namespace.
261 cmd = "for p in $(kubectl get pods -n {namespace} --no-headers"\
262 " | cut -f 1 -d ' '); do echo $p; kubectl logs -n {namespace} $p; "\
263 "done".format(namespace=nspace)
264 ssh.exec_command(cmd, timeout=120)
267 def get_kubernetes_logs_on_all_duts(nodes, nspace='csit'):
268 """Get Kubernetes logs on all DUTs.
270 :param nodes: Topology nodes.
271 :param nspace: Kubernetes namespace.
275 for node in nodes.values():
276 if node['type'] == NodeType.DUT:
277 KubernetesUtils.get_kubernetes_logs_on_node(node, nspace)
280 def reset_kubernetes_on_node(node):
281 """Reset Kubernetes on node.
283 :param node: DUT node.
285 :raises RuntimeError: If resetting Kubernetes failed.
290 cmd = 'kubeadm reset && rm -rf $HOME/.kube'
291 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
292 if int(ret_code) != 0:
293 raise RuntimeError('Failed to reset Kubernetes on {node}.'
294 .format(node=node['host']))
297 def reset_kubernetes_on_all_duts(nodes):
298 """Reset Kubernetes on all DUTs.
300 :param nodes: Topology nodes.
303 for node in nodes.values():
304 if node['type'] == NodeType.DUT:
305 KubernetesUtils.reset_kubernetes_on_node(node)
308 def wait_for_kubernetes_pods_on_node(node, nspace='csit'):
309 """Wait for Kubernetes PODs to become in 'Running' state on node.
311 :param node: DUT node.
312 :param nspace: Kubernetes namespace.
315 :raises RuntimeError: If Kubernetes PODs are not ready.
320 cmd = 'kubectl get -n {namespace} pods --no-headers'\
321 .format(namespace=nspace)
323 (ret_code, stdout, _) = ssh.exec_command_sudo(cmd, timeout=120)
324 if int(ret_code) == 0:
326 for line in stdout.splitlines():
328 state = line.split()[1].split('/')
329 ready = True if 'Running' in line and\
330 state == state[::-1] else False
333 except ValueError, IndexError:
339 raise RuntimeError('Kubernetes PODs are not ready on {node}.'
340 .format(node=node['host']))
343 def wait_for_kubernetes_pods_on_all_duts(nodes, nspace='csit'):
344 """Wait for Kubernetes PODs to become in Running state on all DUTs.
346 :param nodes: Topology nodes.
347 :param nspace: Kubernetes namespace.
351 for node in nodes.values():
352 if node['type'] == NodeType.DUT:
353 KubernetesUtils.wait_for_kubernetes_pods_on_node(node, nspace)
356 def create_kubernetes_vswitch_startup_config(**kwargs):
357 """Create Kubernetes VSWITCH startup configuration.
359 :param kwargs: Key-value pairs used to create configuration.
363 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
364 cpu_node=kwargs['cpu_node'],
365 skip_cnt=kwargs['cpu_skip'],
366 cpu_cnt=kwargs['cpu_cnt'],
367 smt_used=kwargs['smt_used'])
369 # Create config instance
370 vpp_config = VppConfigGenerator()
371 vpp_config.set_node(kwargs['node'])
372 vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
373 vpp_config.add_unix_nodaemon()
374 vpp_config.add_dpdk_socketmem('1024,1024')
375 vpp_config.add_heapsize('3G')
376 vpp_config.add_ip6_hash_buckets('2000000')
377 vpp_config.add_ip6_heap_size('3G')
378 if kwargs['framesize'] < 1522:
379 vpp_config.add_dpdk_no_multi_seg()
380 vpp_config.add_dpdk_dev_default_rxq(kwargs['rxq'])
381 vpp_config.add_dpdk_dev(kwargs['if1'], kwargs['if2'])
382 # We will pop first core from list to be main core
383 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
384 # if this is not only core in list, the rest will be used as workers.
386 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
387 vpp_config.add_cpu_corelist_workers(corelist_workers)
388 vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)
391 def create_kubernetes_vnf_startup_config(**kwargs):
392 """Create Kubernetes VNF startup configuration.
394 :param kwargs: Key-value pairs used to create configuration.
397 skip_cnt = kwargs['cpu_skip'] + (kwargs['i'] - 1) * kwargs['cpu_cnt']
399 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
400 cpu_node=kwargs['cpu_node'],
402 cpu_cnt=kwargs['cpu_cnt'],
403 smt_used=kwargs['smt_used'])
405 # Create config instance
406 vpp_config = VppConfigGenerator()
407 vpp_config.set_node(kwargs['node'])
408 vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
409 vpp_config.add_unix_nodaemon()
410 # We will pop first core from list to be main core
411 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
412 # if this is not only core in list, the rest will be used as workers.
414 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
415 vpp_config.add_cpu_corelist_workers(corelist_workers)
416 vpp_config.add_plugin_disable('dpdk_plugin.so')
417 vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)