1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Library to control Kubernetes kubectl."""
19 from resources.libraries.python.constants import Constants
20 from resources.libraries.python.topology import NodeType
21 from resources.libraries.python.ssh import SSH
22 from resources.libraries.python.CpuUtils import CpuUtils
23 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
25 __all__ = ["KubernetesUtils"]
28 class KubernetesUtils(object):
29 """Kubernetes utilities class."""
32 """Initialize KubernetesUtils class."""
36 def setup_kubernetes_on_node(node):
37 """Set up Kubernetes on node.
39 :param node: DUT node.
41 :raises RuntimeError: If Kubernetes setup failed on node.
46 cmd = '{dir}/{lib}/k8s_setup.sh '.format(dir=Constants.REMOTE_FW_DIR,
47 lib=Constants.RESOURCES_LIB_SH)
48 (ret_code, _, _) = ssh.exec_command(cmd, timeout=120)
49 if int(ret_code) != 0:
50 raise RuntimeError('Failed to setup Kubernetes on {node}.'
51 .format(node=node['host']))
54 def setup_kubernetes_on_all_duts(nodes):
55 """Set up Kubernetes on all DUTs.
57 :param nodes: Topology nodes.
60 for node in nodes.values():
61 if node['type'] == NodeType.DUT:
62 KubernetesUtils.setup_kubernetes_on_node(node)
65 def apply_kubernetes_resource_on_node(node, yaml_file, **kwargs):
66 """Apply Kubernetes resource on node.
68 :param node: DUT node.
69 :param yaml_file: YAML configuration file.
70 :param kwargs: Key-value pairs to replace in YAML template.
74 :raises RuntimeError: If applying Kubernetes template failed.
79 stream = file('{tpl}/{yaml}'.format(tpl=Constants.RESOURCES_TPL_K8S,
82 for data in yaml.load_all(stream):
83 data = reduce(lambda a, kv: a.replace(*kv), kwargs.iteritems(),
84 yaml.dump(data, default_flow_style=False))
85 # Workaround to avoid using RAW string anotated with | in YAML as
86 # library + bash is misinterpreting spaces.
87 data = data.replace('.conf:\n', '.conf: |\n')
88 cmd = 'cat <<EOF | kubectl apply -f - \n{data}\nEOF'.format(
90 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
91 if int(ret_code) != 0:
92 raise RuntimeError('Failed to apply Kubernetes template {yaml} '
93 'on {node}.'.format(yaml=yaml_file,
97 def apply_kubernetes_resource_on_all_duts(nodes, yaml_file, **kwargs):
98 """Apply Kubernetes resource on all DUTs.
100 :param nodes: Topology nodes.
101 :param yaml_file: YAML configuration file.
102 :param kwargs: Key-value pairs to replace in YAML template.
107 for node in nodes.values():
108 if node['type'] == NodeType.DUT:
109 KubernetesUtils.apply_kubernetes_resource_on_node(node,
114 def create_kubernetes_cm_from_file_on_node(node, name, key, src_file):
115 """Create Kubernetes ConfigMap from file on node.
117 :param node: DUT node.
118 :param name: ConfigMap name.
119 :param key: Key (destination file).
120 :param src_file: Source file.
125 :raises RuntimeError: If creating Kubernetes ConfigMap failed.
130 cmd = 'kubectl create -n csit configmap {name} --from-file={key}='\
131 '{src_file}'.format(name=name, key=key, src_file=src_file)
132 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
133 if int(ret_code) != 0:
134 raise RuntimeError('Failed to create Kubernetes ConfigMap {name} '
135 'on {node}.'.format(name=name,
139 def create_kubernetes_cm_from_file_on_all_duts(nodes, name, key, src_file):
140 """Create Kubernetes ConfigMap from file on all DUTs.
142 :param nodes: Topology nodes.
143 :param name: ConfigMap name.
144 :param key: Key (destination file).
145 :param src_file: Source file.
151 for node in nodes.values():
152 if node['type'] == NodeType.DUT:
153 KubernetesUtils.create_kubernetes_cm_from_file_on_node(node,
159 def delete_kubernetes_resource_on_node(node, rtype='po,cm', name=None):
160 """Delete Kubernetes resource on node.
162 :param node: DUT node.
163 :param rtype: Kubernetes resource type.
164 :param name: Name of resource.
168 :raises RuntimeError: If deleting Kubernetes resource failed.
173 name = '{name}'.format(name=name) if name else '--all'
175 cmd = 'kubectl delete -n csit {rtype} {name}'\
176 .format(rtype=rtype, name=name)
177 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
178 if int(ret_code) != 0:
179 raise RuntimeError('Failed to delete Kubernetes resources in CSIT '
180 'namespace on {node}.'.format(node=node['host']))
182 cmd = 'kubectl get -n csit pods --no-headers'
184 (ret_code, stdout, _) = ssh.exec_command_sudo(cmd, timeout=120)
185 if int(ret_code) == 0:
187 for line in stdout.splitlines():
188 if 'No resources found.' not in line:
194 raise RuntimeError('Failed to delete Kubernetes resources in CSIT '
195 'namespace on {node}.'.format(node=node['host']))
198 def delete_kubernetes_resource_on_all_duts(nodes, rtype='po,cm', name=None):
199 """Delete all Kubernetes resource on all DUTs.
201 :param nodes: Topology nodes.
202 :param rtype: Kubernetes resource type.
203 :param name: Name of resource.
208 for node in nodes.values():
209 if node['type'] == NodeType.DUT:
210 KubernetesUtils.delete_kubernetes_resource_on_node(node, rtype,
214 def describe_kubernetes_resource_on_node(node, rtype='po,cm'):
215 """Describe Kubernetes resource on node.
217 :param node: DUT node.
218 :param rtype: Kubernetes resource type.
221 :raises RuntimeError: If describing Kubernetes resource failed.
226 cmd = 'kubectl describe -n csit {rtype}'.format(rtype=rtype)
227 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
228 if int(ret_code) != 0:
229 raise RuntimeError('Failed to describe Kubernetes resource on '
230 '{node}.'.format(node=node['host']))
233 def describe_kubernetes_resource_on_all_duts(nodes, rtype='po,cm'):
234 """Describe Kubernetes resource on all DUTs.
236 :param nodes: Topology nodes.
237 :param rtype: Kubernetes resource type.
241 for node in nodes.values():
242 if node['type'] == NodeType.DUT:
243 KubernetesUtils.describe_kubernetes_resource_on_node(node,
247 def get_kubernetes_logs_on_node(node, namespace='csit'):
248 """Get Kubernetes logs on node.
250 :param node: DUT node.
251 :param namespace: Kubernetes namespace.
258 cmd = "for p in $(kubectl get pods -n {namespace} --no-headers"\
259 " | cut -f 1 -d ' '); do echo $p; kubectl logs -n {namespace} $p; "\
260 "done".format(namespace=namespace)
261 ssh.exec_command(cmd, timeout=120)
264 def get_kubernetes_logs_on_all_duts(nodes, namespace='csit'):
265 """Get Kubernetes logs on all DUTs.
267 :param nodes: Topology nodes.
268 :param namespace: Kubernetes namespace.
272 for node in nodes.values():
273 if node['type'] == NodeType.DUT:
274 KubernetesUtils.get_kubernetes_logs_on_node(node, namespace)
277 def reset_kubernetes_on_node(node):
278 """Reset Kubernetes on node.
280 :param node: DUT node.
282 :raises RuntimeError: If resetting Kubernetes failed.
287 cmd = 'kubeadm reset && rm -rf $HOME/.kube'
288 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
289 if int(ret_code) != 0:
290 raise RuntimeError('Failed to reset Kubernetes on {node}.'
291 .format(node=node['host']))
294 def reset_kubernetes_on_all_duts(nodes):
295 """Reset Kubernetes on all DUTs.
297 :param nodes: Topology nodes.
300 for node in nodes.values():
301 if node['type'] == NodeType.DUT:
302 KubernetesUtils.reset_kubernetes_on_node(node)
305 def wait_for_kubernetes_pods_on_node(node):
306 """Wait for Kubernetes PODs to become in 'Running' state on node.
308 :param node: DUT node.
310 :raises RuntimeError: If Kubernetes PODs are not ready.
315 cmd = 'kubectl get -n csit pods --no-headers'
317 (ret_code, stdout, _) = ssh.exec_command_sudo(cmd, timeout=120)
318 if int(ret_code) == 0:
320 for line in stdout.splitlines():
321 if 'Running' in line and '1/1' in line:
329 raise RuntimeError('Kubernetes PODs are not ready on {node}.'
330 .format(node=node['host']))
333 def wait_for_kubernetes_pods_on_all_duts(nodes):
334 """Wait for Kubernetes PODs to become in Running state on all DUTs.
336 :param nodes: Topology nodes.
339 for node in nodes.values():
340 if node['type'] == NodeType.DUT:
341 KubernetesUtils.wait_for_kubernetes_pods_on_node(node)
344 def create_kubernetes_vswitch_startup_config(**kwargs):
345 """Create Kubernetes VSWITCH startup configuration.
347 :param kwargs: Key-value pairs used to create configuration.
351 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
352 cpu_node=kwargs['cpu_node'],
353 skip_cnt=kwargs['cpu_skip'],
354 cpu_cnt=kwargs['cpu_cnt'],
355 smt_used=kwargs['smt_used'])
357 # Create config instance
358 vpp_config = VppConfigGenerator()
359 vpp_config.set_node(kwargs['node'])
360 vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
361 vpp_config.add_unix_nodaemon()
362 vpp_config.add_dpdk_socketmem('1024,1024')
363 vpp_config.add_heapsize('3G')
364 vpp_config.add_ip6_hash_buckets('2000000')
365 vpp_config.add_ip6_heap_size('3G')
366 if kwargs['framesize'] < 1522:
367 vpp_config.add_dpdk_no_multi_seg()
368 vpp_config.add_dpdk_dev_default_rxq(kwargs['rxq'])
369 vpp_config.add_dpdk_dev(kwargs['if1'], kwargs['if2'])
370 # We will pop first core from list to be main core
371 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
372 # if this is not only core in list, the rest will be used as workers.
374 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
375 vpp_config.add_cpu_corelist_workers(corelist_workers)
376 vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)
379 def create_kubernetes_vnf_startup_config(**kwargs):
380 """Create Kubernetes VNF startup configuration.
382 :param kwargs: Key-value pairs used to create configuration.
386 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
387 cpu_node=kwargs['cpu_node'],
388 skip_cnt=kwargs['cpu_skip'],
389 cpu_cnt=kwargs['cpu_cnt'],
390 smt_used=kwargs['smt_used'])
392 # Create config instance
393 vpp_config = VppConfigGenerator()
394 vpp_config.set_node(kwargs['node'])
395 vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
396 vpp_config.add_unix_nodaemon()
397 # We will pop first core from list to be main core
398 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
399 # if this is not only core in list, the rest will be used as workers.
401 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
402 vpp_config.add_cpu_corelist_workers(corelist_workers)
403 vpp_config.add_plugin_disable('dpdk_plugin.so')
404 vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)