X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2FContainerUtils.py;h=228648921aae9444fe56ff369e7977cb5397b44c;hp=478a9781a023823920370adbf9d414c6d45b7b75;hb=46efda9bd3a32cfe9944cb421ae3e2a3978bf9fc;hpb=b4e5c717f5e2c39ded81f0c6f7b0f9f61945befd diff --git a/resources/libraries/python/ContainerUtils.py b/resources/libraries/python/ContainerUtils.py index 478a9781a0..228648921a 100644 --- a/resources/libraries/python/ContainerUtils.py +++ b/resources/libraries/python/ContainerUtils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 Cisco and/or its affiliates. +# Copyright (c) 2019 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -16,11 +16,11 @@ """Library to manipulate Containers.""" +from string import Template from collections import OrderedDict, Counter from resources.libraries.python.ssh import SSH -from resources.libraries.python.constants import Constants -from resources.libraries.python.CpuUtils import CpuUtils +from resources.libraries.python.Constants import Constants from resources.libraries.python.topology import Topology from resources.libraries.python.VppConfigGenerator import VppConfigGenerator @@ -78,24 +78,6 @@ class ContainerManager(object): setattr(self.engine.container, 'env', 'MICROSERVICE_LABEL={label}'.format(label=kwargs['name'])) - # Set cpuset.cpus cgroup - skip_cnt = kwargs['cpu_skip'] - smt_used = CpuUtils.is_smt_enabled(kwargs['node']['cpuinfo']) - if not kwargs['cpu_shared']: - skip_cnt += kwargs['i'] * kwargs['cpu_count'] - self.engine.container.cpuset_cpus = \ - CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'], - cpu_node=kwargs['cpuset_mems'], - skip_cnt=skip_cnt, - cpu_cnt=1, - smt_used=False) \ - + \ - CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'], - cpu_node=kwargs['cpuset_mems'], - skip_cnt=skip_cnt+1, - cpu_cnt=kwargs['cpu_count']-1, - smt_used=smt_used) - # Store container instance self.containers[kwargs['name']] = self.engine.container @@ -154,84 +136,197 @@ class ContainerManager(object): self.engine.container = self.containers[container] self.engine.execute(command) - def install_vpp_in_all_containers(self): - """Install VPP into all containers.""" + def start_vpp_in_all_containers(self): + """Start VPP in all containers.""" for container in self.containers: self.engine.container = self.containers[container] # We need to install supervisor client/server system to control VPP # as a service self.engine.install_supervisor() - self.engine.install_vpp() - self.engine.restart_vpp() + self.engine.start_vpp() def restart_vpp_in_all_containers(self): - """Restart VPP on all containers.""" + """Restart VPP in all containers.""" for container in self.containers: self.engine.container = self.containers[container] self.engine.restart_vpp() - def configure_vpp_in_all_containers(self, chain_topology, - dut1_if=None, dut2_if=None): + def configure_vpp_in_all_containers(self, chain_topology, **kwargs): """Configure VPP in all containers. :param chain_topology: Topology used for chaining containers can be chain or cross_horiz. Chain topology is using 1 memif pair per container. Cross_horiz topology is using 1 memif and 1 physical interface in container (only single container can be configured). - :param dut1_if: Interface on DUT1 directly connected to DUT2. - :param dut2_if: Interface on DUT2 directly connected to DUT1. - :type container_topology: str - :type dut1_if: str - :type dut2_if: str + :param kwargs: Named parameters. + :type chain_topology: str + :param kwargs: dict """ # Count number of DUTs based on node's host information dut_cnt = len(Counter([self.containers[container].node['host'] for container in self.containers])) mod = len(self.containers)/dut_cnt - container_vat_template = 'memif_create_{topology}.vat'.format( - topology=chain_topology) - - if chain_topology == 'chain': - for i, container in enumerate(self.containers): - mid1 = i % mod + 1 - mid2 = i % mod + 1 - sid1 = i % mod * 2 + 1 - sid2 = i % mod * 2 + 2 - self.engine.container = self.containers[container] - self.engine.create_vpp_startup_config() - self.engine.create_vpp_exec_config(container_vat_template, \ - mid1=mid1, mid2=mid2, sid1=sid1, sid2=sid2, \ - socket1='memif-{c.name}-{sid}'. \ - format(c=self.engine.container, sid=sid1), \ - socket2='memif-{c.name}-{sid}'. \ - format(c=self.engine.container, sid=sid2)) - elif chain_topology == 'cross_horiz': - if mod > 1: - raise RuntimeError('Container chain topology {topology} ' - 'supports only single container.'. - format(topology=chain_topology)) - for i, container in enumerate(self.containers): - mid1 = i % mod + 1 - sid1 = i % mod * 2 + 1 - self.engine.container = self.containers[container] - if 'DUT1' in self.engine.container.name: - if_pci = Topology.get_interface_pci_addr( \ - self.engine.container.node, dut1_if) - if_name = Topology.get_interface_name( \ - self.engine.container.node, dut1_if) - if 'DUT2' in self.engine.container.name: - if_pci = Topology.get_interface_pci_addr( \ - self.engine.container.node, dut2_if) - if_name = Topology.get_interface_name( \ - self.engine.container.node, dut2_if) - self.engine.create_vpp_startup_config_dpdk_dev(if_pci) - self.engine.create_vpp_exec_config(container_vat_template, \ - mid1=mid1, sid1=sid1, if_name=if_name, \ - socket1='memif-{c.name}-{sid}'. \ - format(c=self.engine.container, sid=sid1)) - else: - raise RuntimeError('Container topology {topology} not implemented'. - format(topology=chain_topology)) + + for i, container in enumerate(self.containers): + mid1 = i % mod + 1 + mid2 = i % mod + 1 + sid1 = i % mod * 2 + 1 + sid2 = i % mod * 2 + 2 + self.engine.container = self.containers[container] + guest_dir = self.engine.container.mnt[0].split(':')[1] + + if chain_topology == 'chain': + self._configure_vpp_chain_l2xc(mid1=mid1, mid2=mid2, + sid1=sid1, sid2=sid2, + guest_dir=guest_dir, + **kwargs) + elif chain_topology == 'cross_horiz': + self._configure_vpp_cross_horiz(mid1=mid1, mid2=mid2, + sid1=sid1, sid2=sid2, + guest_dir=guest_dir, + **kwargs) + elif chain_topology == 'chain_functional': + self._configure_vpp_chain_functional(mid1=mid1, mid2=mid2, + sid1=sid1, sid2=sid2, + guest_dir=guest_dir, + **kwargs) + elif chain_topology == 'chain_ip4': + self._configure_vpp_chain_ip4(mid1=mid1, mid2=mid2, + sid1=sid1, sid2=sid2, + guest_dir=guest_dir, + **kwargs) + elif chain_topology == 'pipeline_ip4': + self._configure_vpp_pipeline_ip4(mid1=mid1, mid2=mid2, + sid1=sid1, sid2=sid2, + guest_dir=guest_dir, + **kwargs) + else: + raise RuntimeError('Container topology {name} not implemented'. + format(name=chain_topology)) + + def _configure_vpp_chain_l2xc(self, **kwargs): + """Configure VPP in chain topology with l2xc. + + :param kwargs: Named parameters. + :param kwargs: dict + """ + self.engine.create_vpp_startup_config() + self.engine.create_vpp_exec_config( + 'memif_create_chain_l2xc.exec', + mid1=kwargs['mid1'], mid2=kwargs['mid2'], + sid1=kwargs['sid1'], sid2=kwargs['sid2'], + socket1='{guest_dir}/memif-{c.name}-{sid1}'. + format(c=self.engine.container, **kwargs), + socket2='{guest_dir}/memif-{c.name}-{sid2}'. + format(c=self.engine.container, **kwargs)) + + def _configure_vpp_cross_horiz(self, **kwargs): + """Configure VPP in cross horizontal topology (single memif). + + :param kwargs: Named parameters. + :param kwargs: dict + """ + if 'DUT1' in self.engine.container.name: + if_pci = Topology.get_interface_pci_addr( + self.engine.container.node, kwargs['dut1_if']) + if_name = Topology.get_interface_name( + self.engine.container.node, kwargs['dut1_if']) + if 'DUT2' in self.engine.container.name: + if_pci = Topology.get_interface_pci_addr( + self.engine.container.node, kwargs['dut2_if']) + if_name = Topology.get_interface_name( + self.engine.container.node, kwargs['dut2_if']) + self.engine.create_vpp_startup_config_dpdk_dev(if_pci) + self.engine.create_vpp_exec_config( + 'memif_create_cross_horizon.exec', + mid1=kwargs['mid1'], sid1=kwargs['sid1'], if_name=if_name, + socket1='{guest_dir}/memif-{c.name}-{sid1}'. + format(c=self.engine.container, **kwargs)) + + def _configure_vpp_chain_functional(self, **kwargs): + """Configure VPP in chain topology with l2xc (functional). + + :param kwargs: Named parameters. + :param kwargs: dict + """ + self.engine.create_vpp_startup_config_func_dev() + self.engine.create_vpp_exec_config( + 'memif_create_chain_functional.exec', + mid1=kwargs['mid1'], mid2=kwargs['mid2'], + sid1=kwargs['sid1'], sid2=kwargs['sid2'], + socket1='{guest_dir}/memif-{c.name}-{sid1}'. + format(c=self.engine.container, **kwargs), + socket2='{guest_dir}/memif-{c.name}-{sid2}'. + format(c=self.engine.container, **kwargs), + rx_mode='interrupt') + + def _configure_vpp_chain_ip4(self, **kwargs): + """Configure VPP in chain topology with ip4. + + :param kwargs: Named parameters. + :param kwargs: dict + """ + self.engine.create_vpp_startup_config() + + vif1_mac = kwargs['tg_if1_mac'] \ + if (kwargs['mid1'] - 1) % kwargs['nodes'] + 1 == 1 \ + else '52:54:00:00:{0:02X}:02'.format(kwargs['mid1'] - 1) + vif2_mac = kwargs['tg_if2_mac'] \ + if (kwargs['mid2'] - 1) % kwargs['nodes'] + 1 == kwargs['nodes'] \ + else '52:54:00:00:{0:02X}:01'.format(kwargs['mid2'] + 1) + self.engine.create_vpp_exec_config( + 'memif_create_chain_ip4.exec', + mid1=kwargs['mid1'], mid2=kwargs['mid2'], + sid1=kwargs['sid1'], sid2=kwargs['sid2'], + socket1='{guest_dir}/memif-{c.name}-{sid1}'. + format(c=self.engine.container, **kwargs), + socket2='{guest_dir}/memif-{c.name}-{sid2}'. + format(c=self.engine.container, **kwargs), + mac1='52:54:00:00:{0:02X}:01'.format(kwargs['mid1']), + mac2='52:54:00:00:{0:02X}:02'.format(kwargs['mid2']), + vif1_mac=vif1_mac, vif2_mac=vif2_mac) + + def _configure_vpp_pipeline_ip4(self, **kwargs): + """Configure VPP in pipeline topology with ip4. + + :param kwargs: Named parameters. + :param kwargs: dict + """ + self.engine.create_vpp_startup_config() + node = (kwargs['mid1'] - 1) % kwargs['nodes'] + 1 + mid1 = kwargs['mid1'] + mid2 = kwargs['mid2'] + role1 = 'master' + role2 = 'master' \ + if node == kwargs['nodes'] or node == kwargs['nodes'] and node == 1\ + else 'slave' + kwargs['mid2'] = kwargs['mid2'] \ + if node == kwargs['nodes'] or node == kwargs['nodes'] and node == 1\ + else kwargs['mid2'] + 1 + vif1_mac = kwargs['tg_if1_mac'] \ + if (kwargs['mid1'] - 1) % kwargs['nodes'] + 1 == 1 \ + else '52:54:00:00:{0:02X}:02'.format(kwargs['mid1'] - 1) + vif2_mac = kwargs['tg_if2_mac'] \ + if (kwargs['mid2'] - 1) % kwargs['nodes'] + 1 == kwargs['nodes'] \ + else '52:54:00:00:{0:02X}:01'.format(kwargs['mid2'] + 1) + socket1 = '{guest_dir}/memif-{c.name}-{sid1}'.\ + format(c=self.engine.container, **kwargs) \ + if node == 1 else '{guest_dir}/memif-pipe-{mid1}'.\ + format(c=self.engine.container, **kwargs) + socket2 = '{guest_dir}/memif-{c.name}-{sid2}'.\ + format(c=self.engine.container, **kwargs) \ + if node == 1 and kwargs['nodes'] == 1 or node == kwargs['nodes'] \ + else '{guest_dir}/memif-pipe-{mid2}'.\ + format(c=self.engine.container, **kwargs) + + self.engine.create_vpp_exec_config( + 'memif_create_pipeline_ip4.exec', + mid1=kwargs['mid1'], mid2=kwargs['mid2'], + sid1=kwargs['sid1'], sid2=kwargs['sid2'], + socket1=socket1, socket2=socket2, role1=role1, role2=role2, + mac1='52:54:00:00:{0:02X}:01'.format(mid1), + mac2='52:54:00:00:{0:02X}:02'.format(mid2), + vif1_mac=vif1_mac, vif2_mac=vif2_mac) def stop_all_containers(self): """Stop all containers.""" @@ -299,10 +394,11 @@ class ContainerEngine(object): def install_supervisor(self): """Install supervisord inside a container.""" - self.execute('sleep 3') - self.execute('apt-get update') - self.execute('apt-get install -y supervisor') - self.execute('echo "{config}" > {config_file}'. + if isinstance(self, LXC): + self.execute('sleep 3; apt-get update') + self.execute('apt-get install -y supervisor') + self.execute('echo "{config}" > {config_file} && ' + 'supervisord -c {config_file}'. format( config='[unix_http_server]\n' 'file = /tmp/supervisor.sock\n\n' @@ -319,38 +415,20 @@ class ContainerEngine(object): 'loglevel=debug\n' 'nodaemon=false\n\n', config_file=SUPERVISOR_CONF)) - self.execute('supervisord -c {config_file}'. - format(config_file=SUPERVISOR_CONF)) - - def install_vpp(self): - """Install VPP inside a container.""" - self.execute('ln -s /dev/null /etc/sysctl.d/80-vpp.conf') - self.execute('apt-get update') - if self.container.install_dkms: - self.execute( - 'apt-get install -y dkms && ' - 'dpkg -i --force-all ' - '{guest_dir}/openvpp-testing/download_dir/*.deb'. - format(guest_dir=self.container.mnt[0].split(':')[1])) - else: - self.execute( - 'for i in $(ls -I \"*dkms*\" ' - '{guest_dir}/openvpp-testing/download_dir/); do ' - 'dpkg -i --force-all ' - '{guest_dir}/openvpp-testing/download_dir/$i; done'. - format(guest_dir=self.container.mnt[0].split(':')[1])) - self.execute('apt-get -f install -y') - self.execute('apt-get install -y ca-certificates') + + def start_vpp(self): + """Start VPP inside a container.""" self.execute('echo "{config}" >> {config_file}'. format( config='[program:vpp]\n' 'command=/usr/bin/vpp -c /etc/vpp/startup.conf\n' + 'autostart=false\n' 'autorestart=false\n' 'redirect_stderr=true\n' 'priority=1', config_file=SUPERVISOR_CONF)) self.execute('supervisorctl reload') - self.execute('supervisorctl restart vpp') + self.execute('supervisorctl start vpp') def restart_vpp(self): """Restart VPP service inside a container.""" @@ -371,9 +449,9 @@ class ContainerEngine(object): vpp_config.add_unix_cli_listen() vpp_config.add_unix_nodaemon() vpp_config.add_unix_exec('/tmp/running.exec') - # We will pop first core from list to be main core + # We will pop the first core from the list to be a main core vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0))) - # if this is not only core in list, the rest will be used as workers. + # If more cores in the list, the rest will be used as workers. if cpuset_cpus: corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus) vpp_config.add_cpu_corelist_workers(corelist_workers) @@ -410,23 +488,40 @@ class ContainerEngine(object): self.execute('echo "{config}" | tee /etc/vpp/startup.conf' .format(config=vpp_config.get_config_str())) - def create_vpp_exec_config(self, vat_template_file, **kwargs): + def create_vpp_startup_config_func_dev(self): + """Create startup configuration of VPP on container for functional + vpp_device tests. + """ + # Create config instance + vpp_config = VppConfigGenerator() + vpp_config.set_node(self.container.node) + vpp_config.add_unix_cli_listen() + vpp_config.add_unix_nodaemon() + vpp_config.add_unix_exec('/tmp/running.exec') + vpp_config.add_plugin('disable', 'dpdk_plugin.so') + + # Apply configuration + self.execute('mkdir -p /etc/vpp/') + self.execute('echo "{config}" | tee /etc/vpp/startup.conf' + .format(config=vpp_config.get_config_str())) + + def create_vpp_exec_config(self, template_file, **kwargs): """Create VPP exec configuration on container. - :param vat_template_file: File name of a VAT template script. - :param kwargs: Parameters for VAT script. - :type vat_template_file: str + :param template_file: File name of a template script. + :param kwargs: Parameters for script. + :type template_file: str :type kwargs: dict """ - vat_file_path = '{p}/{f}'.format(p=Constants.RESOURCES_TPL_VAT, - f=vat_template_file) + running = '/tmp/running.exec' + + template = '{res}/{tpl}'.format( + res=Constants.RESOURCES_TPL_CONTAINER, tpl=template_file) - with open(vat_file_path, 'r') as template_file: - cmd_template = template_file.readlines() - for line_tmpl in cmd_template: - vat_cmd = line_tmpl.format(**kwargs) - self.execute('echo "{c}" >> /tmp/running.exec' - .format(c=vat_cmd.replace('\n', ''))) + with open(template, 'r') as src_file: + src = Template(src_file.read()) + self.execute('echo "{out}" > {running}'.format( + out=src.safe_substitute(**kwargs), running=running)) def is_container_running(self): """Check if container is running.""" @@ -497,7 +592,7 @@ class LXC(ContainerEngine): return image = self.container.image if self.container.image else\ - "-d ubuntu -r xenial -a amd64" + "-d ubuntu -r bionic -a amd64" cmd = 'lxc-create -t download --name {c.name} -- {image} '\ '--no-validate'.format(c=self.container, image=image) @@ -516,11 +611,14 @@ class LXC(ContainerEngine): if self.container.mnt: for mount in self.container.mnt: host_dir, guest_dir = mount.split(':') + options = 'bind,create=dir' \ + if guest_dir.endswith('/') else 'bind,create=file' entry = 'lxc.mount.entry = {host_dir} '\ '/var/lib/lxc/{c.name}/rootfs{guest_dir} none ' \ - 'bind,create=dir 0 0'.format(c=self.container, - host_dir=host_dir, - guest_dir=guest_dir) + '{options} 0 0'.format(c=self.container, + host_dir=host_dir, + guest_dir=guest_dir, + options=options) ret, _, _ = self.container.ssh.exec_command_sudo( "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'". format(e=entry, c=self.container)) @@ -685,16 +783,18 @@ class Docker(ContainerEngine): else: return - image = self.container.image if self.container.image else\ - "ubuntu:xenial-20180412" + if not self.container.image: + setattr(self.container, 'image', + Constants.DOCKER_SUT_IMAGE_UBUNTU) - cmd = 'docker pull {image}'.format(image=image) + cmd = 'docker pull {image}'.format(image=self.container.image) ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800) if int(ret) != 0: raise RuntimeError('Failed to create container {c.name}.' .format(c=self.container)) - self._configure_cgroup('docker') + if self.container.cpuset_cpus: + self._configure_cgroup('docker') def create(self): """Create/deploy container. @@ -748,7 +848,7 @@ class Docker(ContainerEngine): :param command: Command to run inside container. :type command: str - :raises RuntimeError: If runnig the command in a container failed. + :raises RuntimeError: If running the command in a container failed. """ cmd = "docker exec --interactive {c.name} /bin/sh -c '{command}; "\ "exit $?'".format(c=self.container, command=command)