1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Library to control Kubernetes kubectl."""
16 from functools import reduce
18 from time import sleep
20 from resources.libraries.python.Constants import Constants
21 from resources.libraries.python.CpuUtils import CpuUtils
22 from resources.libraries.python.ssh import SSH, exec_cmd_no_error
23 from resources.libraries.python.topology import NodeType
24 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
26 __all__ = [u"KubernetesUtils"]
28 # Maximum number of retries to check if PODs are running or deleted.
32 class KubernetesUtils:
33 """Kubernetes utilities class."""
36 """Initialize KubernetesUtils class."""
39 def load_docker_image_on_node(node, image_path):
40 """Load Docker container image from file on node.
42 :param node: DUT node.
43 :param image_path: Container image path.
46 :raises RuntimeError: If loading image failed on node.
48 command = f"docker load -i {image_path}"
49 message = f"Failed to load Docker image on {node[u'host']}."
51 node, command, timeout=240, sudo=True, message=message
54 command = u"docker rmi $(sudo docker images -f 'dangling=true' -q)"
55 message = f"Failed to clean Docker images on {node[u'host']}."
58 node, command, timeout=240, sudo=True, message=message
64 def load_docker_image_on_all_duts(nodes, image_path):
65 """Load Docker container image from file on all DUTs.
67 :param nodes: Topology nodes.
68 :param image_path: Container image path.
72 for node in nodes.values():
73 if node[u"type"] == NodeType.DUT:
74 KubernetesUtils.load_docker_image_on_node(node, image_path)
77 def setup_kubernetes_on_node(node):
78 """Set up Kubernetes on node.
80 :param node: DUT node.
82 :raises RuntimeError: If Kubernetes setup failed on node.
87 cmd = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}/" \
88 f"k8s_setup.sh deploy_calico"
89 ret_code, _, _ = ssh.exec_command(cmd, timeout=240)
90 if int(ret_code) != 0:
92 "Failed to setup Kubernetes on {node[u'host']}."
95 KubernetesUtils.wait_for_kubernetes_pods_on_node(
96 node, nspace=u"kube-system"
100 def setup_kubernetes_on_all_duts(nodes):
101 """Set up Kubernetes on all DUTs.
103 :param nodes: Topology nodes.
106 for node in nodes.values():
107 if node[u"type"] == NodeType.DUT:
108 KubernetesUtils.setup_kubernetes_on_node(node)
111 def destroy_kubernetes_on_node(node):
112 """Destroy Kubernetes on node.
114 :param node: DUT node.
116 :raises RuntimeError: If destroying Kubernetes failed.
121 cmd = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}/" \
122 f"k8s_setup.sh destroy"
124 ret_code, _, _ = ssh.exec_command(cmd, timeout=120)
125 if int(ret_code) != 0:
127 f"Failed to destroy Kubernetes on {node[u'host']}."
131 def destroy_kubernetes_on_all_duts(nodes):
132 """Destroy Kubernetes on all DUTs.
134 :param nodes: Topology nodes.
137 for node in nodes.values():
138 if node[u"type"] == NodeType.DUT:
139 KubernetesUtils.destroy_kubernetes_on_node(node)
142 def apply_kubernetes_resource_on_node(node, yaml_file, **kwargs):
143 """Apply Kubernetes resource on node.
145 :param node: DUT node.
146 :param yaml_file: YAML configuration file.
147 :param kwargs: Key-value pairs to replace in YAML template.
151 :raises RuntimeError: If applying Kubernetes template failed.
156 fqn_file = f"{Constants.RESOURCES_TPL_K8S}/{yaml_file}"
157 with open(fqn_file, 'r') as src_file:
158 stream = src_file.read()
160 lambda a, kv: a.replace(*kv), list(kwargs.items()), stream
162 cmd = f"cat <<EOF | kubectl apply -f - \n{data}\nEOF"
164 ret_code, _, _ = ssh.exec_command_sudo(cmd)
165 if int(ret_code) != 0:
167 f"Failed to apply Kubernetes template {yaml_file} "
168 f"on {node[u'host']}."
172 def apply_kubernetes_resource_on_all_duts(nodes, yaml_file, **kwargs):
173 """Apply Kubernetes resource on all DUTs.
175 :param nodes: Topology nodes.
176 :param yaml_file: YAML configuration file.
177 :param kwargs: Key-value pairs to replace in YAML template.
182 for node in nodes.values():
183 if node[u"type"] == NodeType.DUT:
184 KubernetesUtils.apply_kubernetes_resource_on_node(
185 node, yaml_file, **kwargs
189 def create_kubernetes_cm_from_file_on_node(node, nspace, name, **kwargs):
190 """Create Kubernetes ConfigMap from file on node.
192 :param node: DUT node.
193 :param nspace: Kubernetes namespace.
194 :param name: ConfigMap name.
195 :param kwargs: Named parameters.
200 :raises RuntimeError: If creating Kubernetes ConfigMap failed.
205 nspace = f"-n {nspace}" if nspace else u""
206 from_file = u" ".join(
207 f"--from-file={key}={kwargs[key]} " for key in kwargs
209 cmd = f"kubectl create {nspace} configmap {name} {from_file}"
211 ret_code, _, _ = ssh.exec_command_sudo(cmd)
212 if int(ret_code) != 0:
214 f"Failed to create Kubernetes ConfigMap on {node[u'host']}."
218 def create_kubernetes_cm_from_file_on_all_duts(
219 nodes, nspace, name, **kwargs):
220 """Create Kubernetes ConfigMap from file on all DUTs.
222 :param nodes: Topology nodes.
223 :param nspace: Kubernetes namespace.
224 :param name: ConfigMap name.
225 :param kwargs: Named parameters.
231 for node in nodes.values():
232 if node[u"type"] == NodeType.DUT:
233 KubernetesUtils.create_kubernetes_cm_from_file_on_node(
234 node, nspace, name, **kwargs
238 def delete_kubernetes_resource_on_node(
239 node, nspace, name=None, rtype=u"po,cm,deploy,rs,rc,svc"):
240 """Delete Kubernetes resource on node.
242 :param node: DUT node.
243 :param nspace: Kubernetes namespace.
244 :param rtype: Kubernetes resource type.
245 :param name: Name of resource (Default: all).
250 :raises RuntimeError: If retrieving or deleting Kubernetes resource
256 name = f"{name}" if name else u"--all"
257 nspace = f"-n {nspace}" if nspace else u""
258 cmd = f"kubectl delete {nspace} {rtype} {name}"
260 ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=120)
261 if int(ret_code) != 0:
263 f"Failed to delete Kubernetes resources on {node[u'host']}."
266 cmd = f"kubectl get {nspace} pods --no-headers"
267 for _ in range(MAX_RETRY):
268 ret_code, stdout, stderr = ssh.exec_command_sudo(cmd)
269 if int(ret_code) != 0:
271 f"Failed to retrieve Kubernetes resources "
272 f"on {node[u'host']}."
276 for line in stderr.splitlines():
277 if u"No resources found." in line:
283 for line in stdout.splitlines():
285 state = line.split()[1].split(u"/")
287 u"Running" in line and state == state[::-1]
291 except (ValueError, IndexError):
298 f"Failed to delete Kubernetes resources on {node[u'host']}."
302 def delete_kubernetes_resource_on_all_duts(
303 nodes, nspace, name=None, rtype=u"po,cm,deploy,rs,rc,svc"):
304 """Delete all Kubernetes resource on all DUTs.
306 :param nodes: Topology nodes.
307 :param nspace: Kubernetes namespace.
308 :param rtype: Kubernetes resource type.
309 :param name: Name of resource.
315 for node in nodes.values():
316 if node[u"type"] == NodeType.DUT:
317 KubernetesUtils.delete_kubernetes_resource_on_node(
318 node, nspace, name, rtype
322 def describe_kubernetes_resource_on_node(node, nspace):
323 """Describe all Kubernetes PODs in namespace on node.
325 :param node: DUT node.
326 :param nspace: Kubernetes namespace.
333 nspace = f"-n {nspace}" if nspace else u""
334 cmd = f"kubectl describe {nspace} all"
336 ssh.exec_command_sudo(cmd)
339 def describe_kubernetes_resource_on_all_duts(nodes, nspace):
340 """Describe all Kubernetes PODs in namespace on all DUTs.
342 :param nodes: Topology nodes.
343 :param nspace: Kubernetes namespace.
347 for node in nodes.values():
348 if node[u"type"] == NodeType.DUT:
349 KubernetesUtils.describe_kubernetes_resource_on_node(
354 def get_kubernetes_logs_on_node(node, nspace):
355 """Get Kubernetes logs from all PODs in namespace on node.
357 :param node: DUT node.
358 :param nspace: Kubernetes namespace.
365 nspace = f"-n {nspace}" if nspace else u""
366 cmd = f"for p in $(kubectl get pods {nspace} " \
367 f"-o jsonpath='{{.items[*].metadata.name}}'); do echo $p; " \
368 f"kubectl logs {nspace} $p; done"
370 ssh.exec_command(cmd)
372 cmd = f"kubectl exec {nspace} etcdv3 -- etcdctl " \
373 f"--endpoints \"localhost:22379\" get \"/\" --prefix=true"
375 ssh.exec_command(cmd)
378 def get_kubernetes_logs_on_all_duts(nodes, nspace):
379 """Get Kubernetes logs from all PODs in namespace on all DUTs.
381 :param nodes: Topology nodes.
382 :param nspace: Kubernetes namespace.
386 for node in nodes.values():
387 if node[u"type"] == NodeType.DUT:
388 KubernetesUtils.get_kubernetes_logs_on_node(node, nspace)
391 def wait_for_kubernetes_pods_on_node(node, nspace):
392 """Wait for Kubernetes PODs to become ready on node.
394 :param node: DUT node.
395 :param nspace: Kubernetes namespace.
398 :raises RuntimeError: If Kubernetes PODs are not in Running state.
403 nspace = f"-n {nspace}" if nspace else u"--all-namespaces"
404 cmd = f"kubectl get {nspace} pods --no-headers"
406 for _ in range(MAX_RETRY):
407 ret_code, stdout, _ = ssh.exec_command_sudo(cmd)
408 if int(ret_code) == 0:
410 for line in stdout.splitlines():
412 state = line.split()[1].split(u"/")
414 u"Running" in line and state == state[::-1]
418 except (ValueError, IndexError):
425 f"Kubernetes PODs are not running on {node[u'host']}."
429 def wait_for_kubernetes_pods_on_all_duts(nodes, nspace):
430 """Wait for Kubernetes to become ready on all DUTs.
432 :param nodes: Topology nodes.
433 :param nspace: Kubernetes namespace.
437 for node in nodes.values():
438 if node[u"type"] == NodeType.DUT:
439 KubernetesUtils.wait_for_kubernetes_pods_on_node(node, nspace)
442 def set_kubernetes_pods_affinity_on_node(node):
443 """Set affinity for all Kubernetes PODs except VPP on node.
445 :param node: DUT node.
451 cmd = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}/" \
452 f"k8s_setup.sh affinity_non_vpp"
454 ssh.exec_command(cmd)
457 def set_kubernetes_pods_affinity_on_all_duts(nodes):
458 """Set affinity for all Kubernetes PODs except VPP on all DUTs.
460 :param nodes: Topology nodes.
463 for node in nodes.values():
464 if node[u"type"] == NodeType.DUT:
465 KubernetesUtils.set_kubernetes_pods_affinity_on_node(node)
468 def create_kubernetes_vswitch_startup_config(**kwargs):
469 """Create Kubernetes VSWITCH startup configuration.
471 :param kwargs: Key-value pairs used to create configuration.
474 smt_used = CpuUtils.is_smt_enabled(kwargs[u"node"][u"cpuinfo"])
476 cpuset_cpus = CpuUtils.cpu_slice_of_list_per_node(
477 node=kwargs[u"node"], cpu_node=kwargs[u"cpu_node"], skip_cnt=2,
478 cpu_cnt=kwargs[u"phy_cores"], smt_used=smt_used
480 cpuset_main = CpuUtils.cpu_slice_of_list_per_node(
481 node=kwargs[u"node"], cpu_node=kwargs[u"cpu_node"], skip_cnt=1,
482 cpu_cnt=1, smt_used=smt_used
485 # Create config instance
486 vpp_config = VppConfigGenerator()
487 vpp_config.set_node(kwargs[u"node"])
488 vpp_config.add_unix_cli_listen(value=u"0.0.0.0:5002")
489 vpp_config.add_unix_nodaemon()
490 vpp_config.add_socksvr()
491 vpp_config.add_heapsize(u"4G")
492 vpp_config.add_ip_heap_size(u"4G")
493 vpp_config.add_ip6_heap_size(u"4G")
494 vpp_config.add_ip6_hash_buckets(u"2000000")
495 if not kwargs[u"jumbo"]:
496 vpp_config.add_dpdk_no_multi_seg()
497 vpp_config.add_dpdk_no_tx_checksum_offload()
498 vpp_config.add_dpdk_dev_default_rxq(kwargs[u"rxq_count_int"])
499 vpp_config.add_dpdk_dev(kwargs[u"if1"], kwargs[u"if2"])
500 vpp_config.add_buffers_per_numa(kwargs[u"buffers_per_numa"])
501 # We will pop first core from list to be main core
502 vpp_config.add_cpu_main_core(str(cpuset_main.pop(0)))
503 # if this is not only core in list, the rest will be used as workers.
505 corelist_workers = u",".join(str(cpu) for cpu in cpuset_cpus)
506 vpp_config.add_cpu_corelist_workers(corelist_workers)
507 vpp_config.write_config(filename=kwargs[u"filename"])
510 def create_kubernetes_vnf_startup_config(**kwargs):
511 """Create Kubernetes VNF startup configuration.
513 :param kwargs: Key-value pairs used to create configuration.
516 smt_used = CpuUtils.is_smt_enabled(kwargs[u"node"][u"cpuinfo"])
517 skip_cnt = kwargs[u"cpu_skip"] + (kwargs[u"i"] - 1) * \
518 (kwargs[u"phy_cores"] - 1)
519 cpuset_cpus = CpuUtils.cpu_slice_of_list_per_node(
520 node=kwargs[u"node"], cpu_node=kwargs[u"cpu_node"],
521 skip_cnt=skip_cnt, cpu_cnt=kwargs[u"phy_cores"]-1, smt_used=smt_used
523 cpuset_main = CpuUtils.cpu_slice_of_list_per_node(
524 node=kwargs[u"node"], cpu_node=kwargs[u"cpu_node"], skip_cnt=1,
525 cpu_cnt=1, smt_used=smt_used
527 # Create config instance
528 vpp_config = VppConfigGenerator()
529 vpp_config.set_node(kwargs[u"node"])
530 vpp_config.add_unix_cli_listen(value=u"0.0.0.0:5002")
531 vpp_config.add_unix_nodaemon()
532 vpp_config.add_socksvr()
533 # We will pop first core from list to be main core
534 vpp_config.add_cpu_main_core(str(cpuset_main.pop(0)))
535 # if this is not only core in list, the rest will be used as workers.
537 corelist_workers = u",".join(str(cpu) for cpu in cpuset_cpus)
538 vpp_config.add_cpu_corelist_workers(corelist_workers)
539 vpp_config.add_plugin(u"disable", [u"dpdk_plugin.so"])
540 vpp_config.write_config(filename=kwargs[u"filename"])