1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Library to control Kubernetes kubectl."""
16 from time import sleep
18 from resources.libraries.python.Constants import Constants
19 from resources.libraries.python.topology import NodeType
20 from resources.libraries.python.ssh import SSH, exec_cmd_no_error
21 from resources.libraries.python.CpuUtils import CpuUtils
22 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
24 __all__ = ["KubernetesUtils"]
26 # Maximum number of retries to check if PODs are running or deleted.
29 class KubernetesUtils(object):
30 """Kubernetes utilities class."""
33 """Initialize KubernetesUtils class."""
37 def load_docker_image_on_node(node, image_path):
38 """Load Docker container image from file on node.
40 :param node: DUT node.
41 :param image_path: Container image path.
44 :raises RuntimeError: If loading image failed on node.
46 command = 'docker load -i {image_path}'.\
47 format(image_path=image_path)
48 message = 'Failed to load Docker image on {node}.'.\
49 format(node=node['host'])
50 exec_cmd_no_error(node, command, timeout=240, sudo=True,
53 command = "docker rmi $(sudo docker images -f 'dangling=true' -q)".\
54 format(image_path=image_path)
55 message = 'Failed to clean Docker images on {node}.'.\
56 format(node=node['host'])
58 exec_cmd_no_error(node, command, timeout=240, sudo=True,
64 def load_docker_image_on_all_duts(nodes, image_path):
65 """Load Docker container image from file on all DUTs.
67 :param nodes: Topology nodes.
68 :param image_path: Container image path.
72 for node in nodes.values():
73 if node['type'] == NodeType.DUT:
74 KubernetesUtils.load_docker_image_on_node(node, image_path)
77 def setup_kubernetes_on_node(node):
78 """Set up Kubernetes on node.
80 :param node: DUT node.
82 :raises RuntimeError: If Kubernetes setup failed on node.
87 cmd = '{dir}/{lib}/k8s_setup.sh deploy_calico'\
88 .format(dir=Constants.REMOTE_FW_DIR,
89 lib=Constants.RESOURCES_LIB_SH)
90 (ret_code, _, _) = ssh.exec_command(cmd, timeout=240)
91 if int(ret_code) != 0:
92 raise RuntimeError('Failed to setup Kubernetes on {node}.'
93 .format(node=node['host']))
95 KubernetesUtils.wait_for_kubernetes_pods_on_node(node,
99 def setup_kubernetes_on_all_duts(nodes):
100 """Set up Kubernetes on all DUTs.
102 :param nodes: Topology nodes.
105 for node in nodes.values():
106 if node['type'] == NodeType.DUT:
107 KubernetesUtils.setup_kubernetes_on_node(node)
110 def destroy_kubernetes_on_node(node):
111 """Destroy Kubernetes on node.
113 :param node: DUT node.
115 :raises RuntimeError: If destroying Kubernetes failed.
120 cmd = '{dir}/{lib}/k8s_setup.sh destroy'\
121 .format(dir=Constants.REMOTE_FW_DIR,
122 lib=Constants.RESOURCES_LIB_SH)
123 (ret_code, _, _) = ssh.exec_command(cmd, timeout=120)
124 if int(ret_code) != 0:
125 raise RuntimeError('Failed to destroy Kubernetes on {node}.'
126 .format(node=node['host']))
129 def destroy_kubernetes_on_all_duts(nodes):
130 """Destroy Kubernetes on all DUTs.
132 :param nodes: Topology nodes.
135 for node in nodes.values():
136 if node['type'] == NodeType.DUT:
137 KubernetesUtils.destroy_kubernetes_on_node(node)
140 def apply_kubernetes_resource_on_node(node, yaml_file, **kwargs):
141 """Apply Kubernetes resource on node.
143 :param node: DUT node.
144 :param yaml_file: YAML configuration file.
145 :param kwargs: Key-value pairs to replace in YAML template.
149 :raises RuntimeError: If applying Kubernetes template failed.
154 fqn_file = '{tpl}/{yaml}'.format(tpl=Constants.RESOURCES_TPL_K8S,
156 with open(fqn_file, 'r') as src_file:
157 stream = src_file.read()
158 data = reduce(lambda a, kv: a.replace(*kv), kwargs.iteritems(),
160 cmd = 'cat <<EOF | kubectl apply -f - \n{data}\nEOF'.format(
162 (ret_code, _, _) = ssh.exec_command_sudo(cmd)
163 if int(ret_code) != 0:
164 raise RuntimeError('Failed to apply Kubernetes template {yaml} '
165 'on {node}.'.format(yaml=yaml_file,
169 def apply_kubernetes_resource_on_all_duts(nodes, yaml_file, **kwargs):
170 """Apply Kubernetes resource on all DUTs.
172 :param nodes: Topology nodes.
173 :param yaml_file: YAML configuration file.
174 :param kwargs: Key-value pairs to replace in YAML template.
179 for node in nodes.values():
180 if node['type'] == NodeType.DUT:
181 KubernetesUtils.apply_kubernetes_resource_on_node(node,
186 def create_kubernetes_cm_from_file_on_node(node, nspace, name, **kwargs):
187 """Create Kubernetes ConfigMap from file on node.
189 :param node: DUT node.
190 :param nspace: Kubernetes namespace.
191 :param name: ConfigMap name.
192 :param kwargs: Named parameters.
197 :raises RuntimeError: If creating Kubernetes ConfigMap failed.
202 nspace = '-n {nspace}'.format(nspace=nspace) if nspace else ''
204 from_file = '{0}'.format(' '.join('--from-file={0}={1} '\
205 .format(key, kwargs[key]) for key in kwargs))
207 cmd = 'kubectl create {nspace} configmap {name} {from_file}'\
208 .format(nspace=nspace, name=name, from_file=from_file)
209 (ret_code, _, _) = ssh.exec_command_sudo(cmd)
210 if int(ret_code) != 0:
211 raise RuntimeError('Failed to create Kubernetes ConfigMap '
212 'on {node}.'.format(node=node['host']))
215 def create_kubernetes_cm_from_file_on_all_duts(nodes, nspace, name,
217 """Create Kubernetes ConfigMap from file on all DUTs.
219 :param nodes: Topology nodes.
220 :param nspace: Kubernetes namespace.
221 :param name: ConfigMap name.
222 :param kwargs: Named parameters.
228 for node in nodes.values():
229 if node['type'] == NodeType.DUT:
230 KubernetesUtils.create_kubernetes_cm_from_file_on_node(node,
236 def delete_kubernetes_resource_on_node(node, nspace, name=None,
237 rtype='po,cm,deploy,rs,rc,svc'):
238 """Delete Kubernetes resource on node.
240 :param node: DUT node.
241 :param nspace: Kubernetes namespace.
242 :param rtype: Kubernetes resource type.
243 :param name: Name of resource (Default: all).
248 :raises RuntimeError: If retrieving or deleting Kubernetes resource
254 name = '{name}'.format(name=name) if name else '--all'
255 nspace = '-n {nspace}'.format(nspace=nspace) if nspace else ''
257 cmd = 'kubectl delete {nspace} {rtype} {name}'\
258 .format(nspace=nspace, rtype=rtype, name=name)
259 (ret_code, _, _) = ssh.exec_command_sudo(cmd, timeout=120)
260 if int(ret_code) != 0:
261 raise RuntimeError('Failed to delete Kubernetes resources '
262 'on {node}.'.format(node=node['host']))
264 cmd = 'kubectl get {nspace} pods --no-headers'\
265 .format(nspace=nspace)
266 for _ in range(MAX_RETRY):
267 (ret_code, stdout, stderr) = ssh.exec_command_sudo(cmd)
268 if int(ret_code) != 0:
269 raise RuntimeError('Failed to retrieve Kubernetes resources on '
270 '{node}.'.format(node=node['host']))
273 for line in stderr.splitlines():
274 if 'No resources found.' in line:
280 for line in stdout.splitlines():
282 state = line.split()[1].split('/')
283 ready = True if 'Running' in line and\
284 state == state[::-1] else False
287 except (ValueError, IndexError):
293 raise RuntimeError('Failed to delete Kubernetes resources on '
294 '{node}.'.format(node=node['host']))
297 def delete_kubernetes_resource_on_all_duts(nodes, nspace, name=None,
298 rtype='po,cm,deploy,rs,rc,svc'):
299 """Delete all Kubernetes resource on all DUTs.
301 :param nodes: Topology nodes.
302 :param nspace: Kubernetes namespace.
303 :param rtype: Kubernetes resource type.
304 :param name: Name of resource.
310 for node in nodes.values():
311 if node['type'] == NodeType.DUT:
312 KubernetesUtils.delete_kubernetes_resource_on_node(node, nspace,
316 def describe_kubernetes_resource_on_node(node, nspace):
317 """Describe all Kubernetes PODs in namespace on node.
319 :param node: DUT node.
320 :param nspace: Kubernetes namespace.
327 nspace = '-n {nspace}'.format(nspace=nspace) if nspace else ''
329 cmd = 'kubectl describe {nspace} all'.format(nspace=nspace)
330 ssh.exec_command_sudo(cmd)
333 def describe_kubernetes_resource_on_all_duts(nodes, nspace):
334 """Describe all Kubernetes PODs in namespace on all DUTs.
336 :param nodes: Topology nodes.
337 :param nspace: Kubernetes namespace.
341 for node in nodes.values():
342 if node['type'] == NodeType.DUT:
343 KubernetesUtils.describe_kubernetes_resource_on_node(node,
347 def get_kubernetes_logs_on_node(node, nspace):
348 """Get Kubernetes logs from all PODs in namespace on node.
350 :param node: DUT node.
351 :param nspace: Kubernetes namespace.
358 nspace = '-n {nspace}'.format(nspace=nspace) if nspace else ''
360 cmd = "for p in $(kubectl get pods {nspace} -o jsonpath="\
361 "'{{.items[*].metadata.name}}'); do echo $p; kubectl logs "\
362 "{nspace} $p; done".format(nspace=nspace)
363 ssh.exec_command(cmd)
365 cmd = "kubectl exec {nspace} etcdv3 -- etcdctl --endpoints "\
366 "\"localhost:22379\" get \"/\" --prefix=true".format(nspace=nspace)
367 ssh.exec_command(cmd)
370 def get_kubernetes_logs_on_all_duts(nodes, nspace):
371 """Get Kubernetes logs from all PODs in namespace on all DUTs.
373 :param nodes: Topology nodes.
374 :param nspace: Kubernetes namespace.
378 for node in nodes.values():
379 if node['type'] == NodeType.DUT:
380 KubernetesUtils.get_kubernetes_logs_on_node(node, nspace)
383 def wait_for_kubernetes_pods_on_node(node, nspace):
384 """Wait for Kubernetes PODs to become ready on node.
386 :param node: DUT node.
387 :param nspace: Kubernetes namespace.
390 :raises RuntimeError: If Kubernetes PODs are not in Running state.
395 nspace = '-n {nspace}'.format(nspace=nspace) if nspace \
396 else '--all-namespaces'
398 cmd = 'kubectl get {nspace} pods --no-headers' \
399 .format(nspace=nspace)
400 for _ in range(MAX_RETRY):
401 (ret_code, stdout, _) = ssh.exec_command_sudo(cmd)
402 if int(ret_code) == 0:
404 for line in stdout.splitlines():
406 state = line.split()[1].split('/')
407 ready = True if 'Running' in line and \
408 state == state[::-1] else False
411 except (ValueError, IndexError):
417 raise RuntimeError('Kubernetes PODs are not running on {node}.'
418 .format(node=node['host']))
421 def wait_for_kubernetes_pods_on_all_duts(nodes, nspace):
422 """Wait for Kubernetes to become ready on all DUTs.
424 :param nodes: Topology nodes.
425 :param nspace: Kubernetes namespace.
429 for node in nodes.values():
430 if node['type'] == NodeType.DUT:
431 KubernetesUtils.wait_for_kubernetes_pods_on_node(node, nspace)
434 def set_kubernetes_pods_affinity_on_node(node):
435 """Set affinity for all Kubernetes PODs except VPP on node.
437 :param node: DUT node.
443 cmd = '{dir}/{lib}/k8s_setup.sh affinity_non_vpp'\
444 .format(dir=Constants.REMOTE_FW_DIR,
445 lib=Constants.RESOURCES_LIB_SH)
446 ssh.exec_command(cmd)
449 def set_kubernetes_pods_affinity_on_all_duts(nodes):
450 """Set affinity for all Kubernetes PODs except VPP on all DUTs.
452 :param nodes: Topology nodes.
455 for node in nodes.values():
456 if node['type'] == NodeType.DUT:
457 KubernetesUtils.set_kubernetes_pods_affinity_on_node(node)
460 def create_kubernetes_vswitch_startup_config(**kwargs):
461 """Create Kubernetes VSWITCH startup configuration.
463 :param kwargs: Key-value pairs used to create configuration.
466 smt_used = CpuUtils.is_smt_enabled(kwargs['node']['cpuinfo'])
469 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
470 cpu_node=kwargs['cpu_node'],
472 cpu_cnt=kwargs['phy_cores'],
475 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
476 cpu_node=kwargs['cpu_node'],
481 # Create config instance
482 vpp_config = VppConfigGenerator()
483 vpp_config.set_node(kwargs['node'])
484 vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
485 vpp_config.add_unix_nodaemon()
486 vpp_config.add_socksvr()
487 vpp_config.add_heapsize('4G')
488 vpp_config.add_ip_heap_size('4G')
489 vpp_config.add_ip6_heap_size('4G')
490 vpp_config.add_ip6_hash_buckets('2000000')
491 if not kwargs['jumbo']:
492 vpp_config.add_dpdk_no_multi_seg()
493 vpp_config.add_dpdk_no_tx_checksum_offload()
494 vpp_config.add_dpdk_dev_default_rxq(kwargs['rxq_count_int'])
495 vpp_config.add_dpdk_dev(kwargs['if1'], kwargs['if2'])
496 vpp_config.add_buffers_per_numa(kwargs['buffers_per_numa'])
497 # We will pop first core from list to be main core
498 vpp_config.add_cpu_main_core(str(cpuset_main.pop(0)))
499 # if this is not only core in list, the rest will be used as workers.
501 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
502 vpp_config.add_cpu_corelist_workers(corelist_workers)
503 vpp_config.write_config(filename=kwargs['filename'])
506 def create_kubernetes_vnf_startup_config(**kwargs):
507 """Create Kubernetes VNF startup configuration.
509 :param kwargs: Key-value pairs used to create configuration.
512 smt_used = CpuUtils.is_smt_enabled(kwargs['node']['cpuinfo'])
513 skip_cnt = kwargs['cpu_skip'] + (kwargs['i'] - 1) * \
514 (kwargs['phy_cores'] - 1)
516 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
517 cpu_node=kwargs['cpu_node'],
519 cpu_cnt=kwargs['phy_cores']-1,
522 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
523 cpu_node=kwargs['cpu_node'],
527 # Create config instance
528 vpp_config = VppConfigGenerator()
529 vpp_config.set_node(kwargs['node'])
530 vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
531 vpp_config.add_unix_nodaemon()
532 vpp_config.add_socksvr()
533 # We will pop first core from list to be main core
534 vpp_config.add_cpu_main_core(str(cpuset_main.pop(0)))
535 # if this is not only core in list, the rest will be used as workers.
537 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
538 vpp_config.add_cpu_corelist_workers(corelist_workers)
539 vpp_config.add_plugin('disable', 'dpdk_plugin.so')
540 vpp_config.write_config(filename=kwargs['filename'])