1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Library to control Kubernetes kubectl."""
16 from time import sleep
18 from resources.libraries.python.constants import Constants
19 from resources.libraries.python.topology import NodeType
20 from resources.libraries.python.ssh import SSH
21 from resources.libraries.python.CpuUtils import CpuUtils
22 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
24 __all__ = ["KubernetesUtils"]
26 # Maximum number of retries to check if PODs are running or deleted.
29 class KubernetesUtils(object):
30 """Kubernetes utilities class."""
33 """Initialize KubernetesUtils class."""
37 def setup_kubernetes_on_node(node):
38 """Set up Kubernetes on node.
40 :param node: DUT node.
42 :raises RuntimeError: If Kubernetes setup failed on node.
47 cmd = '{dir}/{lib}/k8s_setup.sh deploy_calico'\
48 .format(dir=Constants.REMOTE_FW_DIR,
49 lib=Constants.RESOURCES_LIB_SH)
50 (ret_code, _, _) = ssh.exec_command(cmd, timeout=240)
51 if int(ret_code) != 0:
52 raise RuntimeError('Failed to setup Kubernetes on {node}.'
53 .format(node=node['host']))
55 KubernetesUtils.wait_for_kubernetes_pods_on_node(node,
59 def setup_kubernetes_on_all_duts(nodes):
60 """Set up Kubernetes on all DUTs.
62 :param nodes: Topology nodes.
65 for node in nodes.values():
66 if node['type'] == NodeType.DUT:
67 KubernetesUtils.setup_kubernetes_on_node(node)
70 def destroy_kubernetes_on_node(node):
71 """Destroy Kubernetes on node.
73 :param node: DUT node.
75 :raises RuntimeError: If destroying Kubernetes failed.
80 cmd = '{dir}/{lib}/k8s_setup.sh destroy'\
81 .format(dir=Constants.REMOTE_FW_DIR,
82 lib=Constants.RESOURCES_LIB_SH)
83 (ret_code, _, _) = ssh.exec_command(cmd, timeout=120)
84 if int(ret_code) != 0:
85 raise RuntimeError('Failed to destroy Kubernetes on {node}.'
86 .format(node=node['host']))
89 def destroy_kubernetes_on_all_duts(nodes):
90 """Destroy Kubernetes on all DUTs.
92 :param nodes: Topology nodes.
95 for node in nodes.values():
96 if node['type'] == NodeType.DUT:
97 KubernetesUtils.destroy_kubernetes_on_node(node)
100 def apply_kubernetes_resource_on_node(node, yaml_file, **kwargs):
101 """Apply Kubernetes resource on node.
103 :param node: DUT node.
104 :param yaml_file: YAML configuration file.
105 :param kwargs: Key-value pairs to replace in YAML template.
109 :raises RuntimeError: If applying Kubernetes template failed.
114 fqn_file = '{tpl}/{yaml}'.format(tpl=Constants.RESOURCES_TPL_K8S,
116 with open(fqn_file, 'r') as src_file:
117 stream = src_file.read()
118 data = reduce(lambda a, kv: a.replace(*kv), kwargs.iteritems(),
120 cmd = 'cat <<EOF | kubectl apply -f - \n{data}\nEOF'.format(
122 (ret_code, _, _) = ssh.exec_command_sudo(cmd)
123 if int(ret_code) != 0:
124 raise RuntimeError('Failed to apply Kubernetes template {yaml} '
125 'on {node}.'.format(yaml=yaml_file,
129 def apply_kubernetes_resource_on_all_duts(nodes, yaml_file, **kwargs):
130 """Apply Kubernetes resource on all DUTs.
132 :param nodes: Topology nodes.
133 :param yaml_file: YAML configuration file.
134 :param kwargs: Key-value pairs to replace in YAML template.
139 for node in nodes.values():
140 if node['type'] == NodeType.DUT:
141 KubernetesUtils.apply_kubernetes_resource_on_node(node,
146 def create_kubernetes_cm_from_file_on_node(node, nspace, name, **kwargs):
147 """Create Kubernetes ConfigMap from file on node.
149 :param node: DUT node.
150 :param nspace: Kubernetes namespace.
151 :param name: ConfigMap name.
152 :param kwargs: Named parameters.
157 :raises RuntimeError: If creating Kubernetes ConfigMap failed.
162 nspace = '-n {nspace}'.format(nspace=nspace) if nspace else ''
164 from_file = '{0}'.format(' '.join('--from-file={0}={1} '\
165 .format(key, kwargs[key]) for key in kwargs))
167 cmd = 'kubectl create {nspace} configmap {name} {from_file}'\
168 .format(nspace=nspace, name=name, from_file=from_file)
169 (ret_code, _, _) = ssh.exec_command_sudo(cmd)
170 if int(ret_code) != 0:
171 raise RuntimeError('Failed to create Kubernetes ConfigMap '
172 'on {node}.'.format(node=node['host']))
175 def create_kubernetes_cm_from_file_on_all_duts(nodes, nspace, name,
177 """Create Kubernetes ConfigMap from file on all DUTs.
179 :param nodes: Topology nodes.
180 :param nspace: Kubernetes namespace.
181 :param name: ConfigMap name.
182 :param kwargs: Named parameters.
188 for node in nodes.values():
189 if node['type'] == NodeType.DUT:
190 KubernetesUtils.create_kubernetes_cm_from_file_on_node(node,
196 def delete_kubernetes_resource_on_node(node, nspace, name=None,
197 rtype='po,cm,deploy,rs,rc,svc'):
198 """Delete Kubernetes resource on node.
200 :param node: DUT node.
201 :param nspace: Kubernetes namespace.
202 :param rtype: Kubernetes resource type.
203 :param name: Name of resource (Default: all).
208 :raises RuntimeError: If retrieving or deleting Kubernetes resource
214 name = '{name}'.format(name=name) if name else '--all'
215 nspace = '-n {nspace}'.format(nspace=nspace) if nspace else ''
217 cmd = 'kubectl delete {nspace} {rtype} {name}'\
218 .format(nspace=nspace, rtype=rtype, name=name)
219 (ret_code, _, _) = ssh.exec_command_sudo(cmd)
220 if int(ret_code) != 0:
221 raise RuntimeError('Failed to delete Kubernetes resources '
222 'on {node}.'.format(node=node['host']))
224 cmd = 'kubectl get {nspace} pods -a --no-headers'\
225 .format(nspace=nspace)
226 for _ in range(MAX_RETRY):
227 (ret_code, stdout, stderr) = ssh.exec_command_sudo(cmd)
228 if int(ret_code) != 0:
229 raise RuntimeError('Failed to retrieve Kubernetes resources on '
230 '{node}.'.format(node=node['host']))
233 for line in stderr.splitlines():
234 if 'No resources found.' in line:
240 for line in stdout.splitlines():
242 state = line.split()[1].split('/')
243 ready = True if 'Running' in line and\
244 state == state[::-1] else False
247 except (ValueError, IndexError):
253 raise RuntimeError('Failed to delete Kubernetes resources on '
254 '{node}.'.format(node=node['host']))
257 def delete_kubernetes_resource_on_all_duts(nodes, nspace, name=None,
258 rtype='po,cm,deploy,rs,rc,svc'):
259 """Delete all Kubernetes resource on all DUTs.
261 :param nodes: Topology nodes.
262 :param nspace: Kubernetes namespace.
263 :param rtype: Kubernetes resource type.
264 :param name: Name of resource.
270 for node in nodes.values():
271 if node['type'] == NodeType.DUT:
272 KubernetesUtils.delete_kubernetes_resource_on_node(node, nspace,
276 def describe_kubernetes_resource_on_node(node, nspace):
277 """Describe all Kubernetes PODs in namespace on node.
279 :param node: DUT node.
280 :param nspace: Kubernetes namespace.
287 nspace = '-n {nspace}'.format(nspace=nspace) if nspace else ''
289 cmd = 'kubectl describe {nspace} all'.format(nspace=nspace)
290 ssh.exec_command_sudo(cmd)
293 def describe_kubernetes_resource_on_all_duts(nodes, nspace):
294 """Describe all Kubernetes PODs in namespace on all DUTs.
296 :param nodes: Topology nodes.
297 :param nspace: Kubernetes namespace.
301 for node in nodes.values():
302 if node['type'] == NodeType.DUT:
303 KubernetesUtils.describe_kubernetes_resource_on_node(node,
307 def get_kubernetes_logs_on_node(node, nspace):
308 """Get Kubernetes logs from all PODs in namespace on node.
310 :param node: DUT node.
311 :param nspace: Kubernetes namespace.
318 nspace = '-n {nspace}'.format(nspace=nspace) if nspace else ''
320 cmd = "for p in $(kubectl get pods {nspace} -o jsonpath="\
321 "'{{.items[*].metadata.name}}'); do echo $p; kubectl logs "\
322 "{nspace} $p; done".format(nspace=nspace)
323 ssh.exec_command(cmd)
325 cmd = "kubectl exec {nspace} etcdv3 -- etcdctl --endpoints "\
326 "\"localhost:22379\" get \"/\" --prefix=true".format(nspace=nspace)
327 ssh.exec_command(cmd)
330 def get_kubernetes_logs_on_all_duts(nodes, nspace):
331 """Get Kubernetes logs from all PODs in namespace on all DUTs.
333 :param nodes: Topology nodes.
334 :param nspace: Kubernetes namespace.
338 for node in nodes.values():
339 if node['type'] == NodeType.DUT:
340 KubernetesUtils.get_kubernetes_logs_on_node(node, nspace)
343 def wait_for_kubernetes_pods_on_node(node, nspace):
344 """Wait for Kubernetes PODs to become ready on node.
346 :param node: DUT node.
347 :param nspace: Kubernetes namespace.
350 :raises RuntimeError: If Kubernetes PODs are not in Running state.
355 nspace = '-n {nspace}'.format(nspace=nspace) if nspace \
356 else '--all-namespaces'
358 cmd = 'kubectl get {nspace} pods -a --no-headers' \
359 .format(nspace=nspace)
360 for _ in range(MAX_RETRY):
361 (ret_code, stdout, _) = ssh.exec_command_sudo(cmd)
362 if int(ret_code) == 0:
364 for line in stdout.splitlines():
366 state = line.split()[1].split('/')
367 ready = True if 'Running' in line and \
368 state == state[::-1] else False
371 except (ValueError, IndexError):
377 raise RuntimeError('Kubernetes PODs are not running on {node}.'
378 .format(node=node['host']))
381 def wait_for_kubernetes_pods_on_all_duts(nodes, nspace):
382 """Wait for Kubernetes to become ready on all DUTs.
384 :param nodes: Topology nodes.
385 :param nspace: Kubernetes namespace.
389 for node in nodes.values():
390 if node['type'] == NodeType.DUT:
391 KubernetesUtils.wait_for_kubernetes_pods_on_node(node, nspace)
394 def set_kubernetes_pods_affinity_on_node(node):
395 """Set affinity for all Kubernetes PODs except VPP on node.
397 :param node: DUT node.
403 cmd = '{dir}/{lib}/k8s_setup.sh affinity_non_vpp'\
404 .format(dir=Constants.REMOTE_FW_DIR,
405 lib=Constants.RESOURCES_LIB_SH)
406 ssh.exec_command(cmd)
409 def set_kubernetes_pods_affinity_on_all_duts(nodes):
410 """Set affinity for all Kubernetes PODs except VPP on all DUTs.
412 :param nodes: Topology nodes.
415 for node in nodes.values():
416 if node['type'] == NodeType.DUT:
417 KubernetesUtils.set_kubernetes_pods_affinity_on_node(node)
420 def create_kubernetes_vswitch_startup_config(**kwargs):
421 """Create Kubernetes VSWITCH startup configuration.
423 :param kwargs: Key-value pairs used to create configuration.
426 smt_used = CpuUtils.is_smt_enabled(kwargs['node']['cpuinfo'])
429 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
430 cpu_node=kwargs['cpu_node'],
432 cpu_cnt=kwargs['phy_cores'],
435 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
436 cpu_node=kwargs['cpu_node'],
441 # Create config instance
442 vpp_config = VppConfigGenerator()
443 vpp_config.set_node(kwargs['node'])
444 vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
445 vpp_config.add_unix_nodaemon()
446 vpp_config.add_dpdk_socketmem('1024,1024')
447 vpp_config.add_heapsize('4G')
448 vpp_config.add_ip_heap_size('4G')
449 vpp_config.add_ip6_heap_size('4G')
450 vpp_config.add_ip6_hash_buckets('2000000')
451 if not kwargs['jumbo']:
452 vpp_config.add_dpdk_no_multi_seg()
453 vpp_config.add_dpdk_no_tx_checksum_offload()
454 vpp_config.add_dpdk_dev_default_rxq(kwargs['rxq_count_int'])
455 vpp_config.add_dpdk_dev(kwargs['if1'], kwargs['if2'])
456 vpp_config.add_dpdk_num_mbufs(kwargs['num_mbufs_int'])
457 # We will pop first core from list to be main core
458 vpp_config.add_cpu_main_core(str(cpuset_main.pop(0)))
459 # if this is not only core in list, the rest will be used as workers.
461 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
462 vpp_config.add_cpu_corelist_workers(corelist_workers)
463 vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)
466 def create_kubernetes_vnf_startup_config(**kwargs):
467 """Create Kubernetes VNF startup configuration.
469 :param kwargs: Key-value pairs used to create configuration.
472 smt_used = CpuUtils.is_smt_enabled(kwargs['node']['cpuinfo'])
473 skip_cnt = kwargs['cpu_skip'] + (kwargs['i'] - 1) * \
474 (kwargs['phy_cores'] - 1)
476 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
477 cpu_node=kwargs['cpu_node'],
479 cpu_cnt=kwargs['phy_cores']-1,
482 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
483 cpu_node=kwargs['cpu_node'],
487 # Create config instance
488 vpp_config = VppConfigGenerator()
489 vpp_config.set_node(kwargs['node'])
490 vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
491 vpp_config.add_unix_nodaemon()
492 # We will pop first core from list to be main core
493 vpp_config.add_cpu_main_core(str(cpuset_main.pop(0)))
494 # if this is not only core in list, the rest will be used as workers.
496 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
497 vpp_config.add_cpu_corelist_workers(corelist_workers)
498 vpp_config.add_plugin('disable', 'dpdk_plugin.so')
499 vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)