1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Library to control Kubernetes kubectl."""
19 from resources.libraries.python.constants import Constants
20 from resources.libraries.python.topology import NodeType
21 from resources.libraries.python.ssh import SSH
22 from resources.libraries.python.CpuUtils import CpuUtils
23 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
25 __all__ = ["KubernetesUtils"]
28 class KubernetesUtils(object):
29 """Kubernetes utilities class."""
32 """Initialize KubernetesUtils class."""
36 def setup_kubernetes_on_node(node):
37 """Set up Kubernetes on node.
39 :param node: DUT node.
41 :raises RuntimeError: If Kubernetes setup failed on node.
46 cmd = '{dir}/{lib}/k8s_setup.sh '.format(dir=Constants.REMOTE_FW_DIR,
47 lib=Constants.RESOURCES_LIB_SH)
48 (ret_code, _, _) = ssh.exec_command(cmd, timeout=120)
49 if int(ret_code) != 0:
50 raise RuntimeError('Failed to setup Kubernetes on {node}.'
51 .format(node=node['host']))
54 def setup_kubernetes_on_all_duts(nodes):
55 """Set up Kubernetes on all DUTs.
57 :param nodes: Topology nodes.
60 for node in nodes.values():
61 if node['type'] == NodeType.DUT:
62 KubernetesUtils.setup_kubernetes_on_node(node)
65 def apply_kubernetes_resource_on_node(node, yaml_file, **kwargs):
66 """Apply Kubernetes resource on node.
68 :param node: DUT node.
69 :param yaml_file: YAML configuration file.
70 :param kwargs: Key-value pairs to replace in YAML template.
74 :raises RuntimeError: If applying Kubernetes template failed.
79 stream = file('{tpl}/{yaml}'.format(tpl=Constants.RESOURCES_TPL_K8S,
82 for data in yaml.load_all(stream):
83 data = reduce(lambda a, kv: a.replace(*kv), kwargs.iteritems(),
84 yaml.dump(data, default_flow_style=False))
85 # Workaround to avoid using RAW string anotated with | in YAML as
86 # library + bash is misinterpreting spaces.
87 data = data.replace('.conf:\n', '.conf: |\n')
88 cmd = 'cat <<EOF | kubectl apply -f - \n{data}\nEOF'.format(
90 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
91 if int(ret_code) != 0:
92 raise RuntimeError('Failed to apply Kubernetes template {yaml} '
93 'on {node}.'.format(yaml=yaml_file,
97 def apply_kubernetes_resource_on_all_duts(nodes, yaml_file, **kwargs):
98 """Apply Kubernetes resource on all DUTs.
100 :param nodes: Topology nodes.
101 :param yaml_file: YAML configuration file.
102 :param kwargs: Key-value pairs to replace in YAML template.
107 for node in nodes.values():
108 if node['type'] == NodeType.DUT:
109 KubernetesUtils.apply_kubernetes_resource_on_node(node,
114 def create_kubernetes_cm_from_file_on_node(node, name, key, src_file):
115 """Create Kubernetes ConfigMap from file on node.
117 :param node: DUT node.
118 :param name: ConfigMap name.
119 :param key: Key (destination file).
120 :param src_file: Source file.
125 :raises RuntimeError: If creating Kubernetes ConfigMap failed.
130 cmd = 'kubectl create -n csit configmap {name} --from-file={key}='\
131 '{src_file}'.format(name=name, key=key, src_file=src_file)
132 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
133 if int(ret_code) != 0:
134 raise RuntimeError('Failed to create Kubernetes ConfigMap {name} '
135 'on {node}.'.format(name=name,
139 def create_kubernetes_cm_from_file_on_all_duts(nodes, name, key, src_file):
140 """Create Kubernetes ConfigMap from file on all DUTs.
142 :param nodes: Topology nodes.
143 :param name: ConfigMap name.
144 :param key: Key (destination file).
145 :param src_file: Source file.
151 for node in nodes.values():
152 if node['type'] == NodeType.DUT:
153 KubernetesUtils.create_kubernetes_cm_from_file_on_node(node,
159 def delete_kubernetes_resource_on_node(node, rtype='po,cm', name=None):
160 """Delete Kubernetes resource on node.
162 :param node: DUT node.
163 :param rtype: Kubernetes resource type.
164 :param name: Name of resource.
168 :raises RuntimeError: If deleting Kubernetes resource failed.
173 name = '{name}'.format(name=name) if name else '--all'
175 cmd = 'kubectl delete -n csit {rtype} {name}'\
176 .format(rtype=rtype, name=name)
177 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
178 if int(ret_code) != 0:
179 raise RuntimeError('Failed to delete Kubernetes resources in CSIT '
180 'namespace on {node}.'.format(node=node['host']))
182 cmd = 'kubectl get -n csit pods --no-headers'
184 (ret_code, stdout, _) = ssh.exec_command_sudo(cmd, timeout=120)
185 if int(ret_code) == 0:
187 for line in stdout.splitlines():
188 if 'No resources found.' not in line:
194 raise RuntimeError('Failed to delete Kubernetes resources in CSIT '
195 'namespace on {node}.'.format(node=node['host']))
198 def delete_kubernetes_resource_on_all_duts(nodes, rtype='po,cm', name=None):
199 """Delete all Kubernetes resource on all DUTs.
201 :param nodes: Topology nodes.
202 :param rtype: Kubernetes resource type.
203 :param name: Name of resource.
208 for node in nodes.values():
209 if node['type'] == NodeType.DUT:
210 KubernetesUtils.delete_kubernetes_resource_on_node(node, rtype,
214 def describe_kubernetes_resource_on_node(node, rtype='po,cm'):
215 """Describe Kubernetes resource on node.
217 :param node: DUT node.
218 :param rtype: Kubernetes resource type.
221 :raises RuntimeError: If describing Kubernetes resource failed.
226 cmd = 'kubectl describe -n csit {rtype}'.format(rtype=rtype)
227 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
228 if int(ret_code) != 0:
229 raise RuntimeError('Failed to describe Kubernetes resource on '
230 '{node}.'.format(node=node['host']))
233 def describe_kubernetes_resource_on_all_duts(nodes, rtype='po,cm'):
234 """Describe Kubernetes resource on all DUTs.
236 :param nodes: Topology nodes.
237 :param rtype: Kubernetes resource type.
241 for node in nodes.values():
242 if node['type'] == NodeType.DUT:
243 KubernetesUtils.describe_kubernetes_resource_on_node(node,
247 def reset_kubernetes_on_node(node):
248 """Reset Kubernetes on node.
250 :param node: DUT node.
252 :raises RuntimeError: If resetting Kubernetes failed.
257 cmd = 'kubeadm reset && rm -rf $HOME/.kube'
258 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
259 if int(ret_code) != 0:
260 raise RuntimeError('Failed to reset Kubernetes on {node}.'
261 .format(node=node['host']))
264 def reset_kubernetes_on_all_duts(nodes):
265 """Reset Kubernetes on all DUTs.
267 :param nodes: Topology nodes.
270 for node in nodes.values():
271 if node['type'] == NodeType.DUT:
272 KubernetesUtils.reset_kubernetes_on_node(node)
275 def wait_for_kubernetes_pods_on_node(node):
276 """Wait for Kubernetes PODs to become in 'Running' state on node.
278 :param node: DUT node.
280 :raises RuntimeError: If Kubernetes PODs are not ready.
285 cmd = 'kubectl get -n csit pods --no-headers'
287 (ret_code, stdout, _) = ssh.exec_command_sudo(cmd, timeout=120)
288 if int(ret_code) == 0:
290 for line in stdout.splitlines():
291 if 'Running' not in line:
297 raise RuntimeError('Kubernetes PODs are not ready on {node}.'
298 .format(node=node['host']))
301 def wait_for_kubernetes_pods_on_all_duts(nodes):
302 """Wait for Kubernetes PODs to become in Running state on all DUTs.
304 :param nodes: Topology nodes.
307 for node in nodes.values():
308 if node['type'] == NodeType.DUT:
309 KubernetesUtils.wait_for_kubernetes_pods_on_node(node)
312 def create_kubernetes_vswitch_startup_config(**kwargs):
313 """Create Kubernetes VSWITCH startup configuration.
315 :param kwargs: Key-value pairs used to create configuration.
319 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
320 cpu_node=kwargs['cpu_node'],
321 skip_cnt=kwargs['cpu_skip'],
322 cpu_cnt=kwargs['cpu_cnt'],
323 smt_used=kwargs['smt_used'])
325 # Create config instance
326 vpp_config = VppConfigGenerator()
327 vpp_config.set_node(kwargs['node'])
328 vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
329 vpp_config.add_unix_nodaemon()
330 vpp_config.add_dpdk_socketmem('1024,1024')
331 vpp_config.add_heapsize('3G')
332 vpp_config.add_ip6_hash_buckets('2000000')
333 vpp_config.add_ip6_heap_size('3G')
334 if kwargs['framesize'] < 1522:
335 vpp_config.add_dpdk_no_multi_seg()
336 vpp_config.add_dpdk_dev_default_rxq(kwargs['rxq'])
337 vpp_config.add_dpdk_dev(kwargs['if1'], kwargs['if2'])
338 # We will pop first core from list to be main core
339 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
340 # if this is not only core in list, the rest will be used as workers.
342 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
343 vpp_config.add_cpu_corelist_workers(corelist_workers)
344 vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)
347 def create_kubernetes_vnf_startup_config(**kwargs):
348 """Create Kubernetes VNF startup configuration.
350 :param kwargs: Key-value pairs used to create configuration.
354 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
355 cpu_node=kwargs['cpu_node'],
356 skip_cnt=kwargs['cpu_skip'],
357 cpu_cnt=kwargs['cpu_cnt'],
358 smt_used=kwargs['smt_used'])
360 # Create config instance
361 vpp_config = VppConfigGenerator()
362 vpp_config.set_node(kwargs['node'])
363 vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
364 vpp_config.add_unix_nodaemon()
365 # We will pop first core from list to be main core
366 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
367 # if this is not only core in list, the rest will be used as workers.
369 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
370 vpp_config.add_cpu_corelist_workers(corelist_workers)
371 vpp_config.add_plugin_disable('dpdk_plugin.so')
372 vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)