X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2FContainerUtils.py;h=4d5f8ee4d6a5f9294789195c7297d938c2d55969;hp=fb2695fe44f5d13635279b709b9a42d0169674c4;hb=fd6fedc983a7d1796eb2531782be8a37b6d0921d;hpb=df5ed0d9317e2a1d4a7248459cc6aec39f80a052 diff --git a/resources/libraries/python/ContainerUtils.py b/resources/libraries/python/ContainerUtils.py index fb2695fe44..4d5f8ee4d6 100644 --- a/resources/libraries/python/ContainerUtils.py +++ b/resources/libraries/python/ContainerUtils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -12,7 +12,7 @@ # limitations under the License. # Bug workaround in pylint for abstract classes. -#pylint: disable=W0223 +# pylint: disable=W0223 """Library to manipulate Containers.""" @@ -42,8 +42,8 @@ class ContainerManager(object): try: self.engine = globals()[engine]() except KeyError: - raise NotImplementedError('{e} is not implemented.' - .format(e=engine)) + raise NotImplementedError('{engine} is not implemented.'. + format(engine=engine)) self.containers = OrderedDict() def get_container_by_name(self, name): @@ -58,8 +58,8 @@ class ContainerManager(object): try: return self.containers[name] except KeyError: - raise RuntimeError('Failed to get container with name: {n}' - .format(n=name)) + raise RuntimeError('Failed to get container with name: {name}'. + format(name=name)) def construct_container(self, **kwargs): """Construct container object on node with specified parameters. @@ -75,7 +75,7 @@ class ContainerManager(object): # Set additional environmental variables setattr(self.engine.container, 'env', - 'MICROSERVICE_LABEL={n}'.format(n=kwargs['name'])) + 'MICROSERVICE_LABEL={label}'.format(label=kwargs['name'])) # Set cpuset.cpus cgroup skip_cnt = kwargs['cpu_skip'] @@ -93,11 +93,12 @@ class ContainerManager(object): def construct_containers(self, **kwargs): """Construct 1..N container(s) on node with specified name. + Ordinal number is automatically added to the name of container as suffix. - :param kwargs: Name of container. - :param kwargs: str + :param kwargs: Named parameters. + :param kwargs: dict """ name = kwargs['name'] for i in range(kwargs['count']): @@ -155,6 +156,12 @@ class ContainerManager(object): self.engine.install_vpp() self.engine.restart_vpp() + def restart_vpp_in_all_containers(self): + """Restart VPP on all containers.""" + for container in self.containers: + self.engine.container = self.containers[container] + self.engine.restart_vpp() + def configure_vpp_in_all_containers(self, vat_template_file): """Configure VPP in all containers. @@ -165,18 +172,23 @@ class ContainerManager(object): dut_cnt = len(Counter([self.containers[container].node['host'] for container in self.containers])) container_cnt = len(self.containers) - mod = dut_cnt/container_cnt + mod = container_cnt/dut_cnt for i, container in enumerate(self.containers): + mid1 = i % mod + 1 + mid2 = i % mod + 1 + sid1 = i % mod * 2 + 1 + sid2 = i % mod * 2 + 2 self.engine.container = self.containers[container] self.engine.create_vpp_startup_config() - self.engine.create_vpp_exec_config(vat_template_file, - memif_id1=i % mod * 2 + 1, - memif_id2=i % mod * 2 + 2, - socket1='memif-{c.name}-1' - .format(c=self.engine.container), - socket2='memif-{c.name}-2' - .format(c=self.engine.container)) + self.engine.create_vpp_exec_config(vat_template_file, mid1=mid1, + mid2=mid2, sid1=sid1, sid2=sid2, + socket1='memif-{c.name}-{sid}' + .format(c=self.engine.container, + sid=sid1), + socket2='memif-{c.name}-{sid}' + .format(c=self.engine.container, + sid=sid2)) def stop_all_containers(self): """Stop all containers.""" @@ -247,9 +259,9 @@ class ContainerEngine(object): self.execute('sleep 3') self.execute('apt-get update') self.execute('apt-get install -y supervisor') - self.execute('echo "{0}" > {1}' - .format( - '[unix_http_server]\n' + self.execute('echo "{config}" > {config_file}'. + format( + config='[unix_http_server]\n' 'file = /tmp/supervisor.sock\n\n' '[rpcinterface:supervisor]\n' 'supervisor.rpcinterface_factory = ' @@ -263,41 +275,40 @@ class ContainerEngine(object): 'logfile=/tmp/supervisord.log\n' 'loglevel=debug\n' 'nodaemon=false\n\n', - SUPERVISOR_CONF)) - self.execute('supervisord -c {0}'.format(SUPERVISOR_CONF)) + config_file=SUPERVISOR_CONF)) + self.execute('supervisord -c {config_file}'. + format(config_file=SUPERVISOR_CONF)) - def install_vpp(self, install_dkms=False): - """Install VPP inside a container. - - :param install_dkms: If install dkms package. This will impact install - time. Dkms is required for installation of vpp-dpdk-dkms. Default is - false. - :type install_dkms: bool - """ + def install_vpp(self): + """Install VPP inside a container.""" self.execute('ln -s /dev/null /etc/sysctl.d/80-vpp.conf') self.execute('apt-get update') - if install_dkms: - self.execute('apt-get install -y dkms && ' - 'dpkg -i --force-all {0}/install_dir/*.deb' - .format(self.container.guest_dir)) + if self.container.install_dkms: + self.execute( + 'apt-get install -y dkms && ' + 'dpkg -i --force-all {guest_dir}/install_dir/*.deb'. + format(guest_dir=self.container.mnt[0].split(':')[1])) else: - self.execute('for i in $(ls -I \"*dkms*\" {0}/install_dir/); ' - 'do dpkg -i --force-all {0}/install_dir/$i; done' - .format(self.container.guest_dir)) + self.execute( + 'for i in $(ls -I \"*dkms*\" {guest_dir}/install_dir/); do ' + 'dpkg -i --force-all {guest_dir}/install_dir/$i; done'. + format(guest_dir=self.container.mnt[0].split(':')[1])) self.execute('apt-get -f install -y') - self.execute('echo "{0}" >> {1}' - .format( - '[program:vpp]\n' + self.execute('apt-get install -y ca-certificates') + self.execute('echo "{config}" >> {config_file}'. + format( + config='[program:vpp]\n' 'command=/usr/bin/vpp -c /etc/vpp/startup.conf\n' 'autorestart=false\n' 'redirect_stderr=true\n' 'priority=1', - SUPERVISOR_CONF)) + config_file=SUPERVISOR_CONF)) self.execute('supervisorctl reload') def restart_vpp(self): """Restart VPP service inside a container.""" self.execute('supervisorctl restart vpp') + self.execute('cat /tmp/supervisord.log') def create_vpp_startup_config(self, config_filename='/etc/vpp/startup.conf'): @@ -311,7 +322,6 @@ class ContainerEngine(object): # Create config instance vpp_config = VppConfigGenerator() vpp_config.set_node(self.container.node) - vpp_config.set_config_filename(config_filename) vpp_config.add_unix_cli_listen() vpp_config.add_unix_nodaemon() vpp_config.add_unix_exec('/tmp/running.exec') @@ -321,20 +331,20 @@ class ContainerEngine(object): if cpuset_cpus: corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus) vpp_config.add_cpu_corelist_workers(corelist_workers) - vpp_config.add_plugin_disable('dpdk_plugin.so') + vpp_config.add_plugin('disable', 'dpdk_plugin.so') self.execute('mkdir -p /etc/vpp/') self.execute('echo "{c}" | tee {f}' .format(c=vpp_config.get_config_str(), - f=vpp_config.get_config_filename())) + f=config_filename)) - def create_vpp_exec_config(self, vat_template_file, **args): + def create_vpp_exec_config(self, vat_template_file, **kwargs): """Create VPP exec configuration on container. :param vat_template_file: File name of a VAT template script. - :param args: Parameters for VAT script. + :param kwargs: Parameters for VAT script. :type vat_template_file: str - :type args: dict + :type kwargs: dict """ vat_file_path = '{p}/{f}'.format(p=Constants.RESOURCES_TPL_VAT, f=vat_template_file) @@ -342,7 +352,7 @@ class ContainerEngine(object): with open(vat_file_path, 'r') as template_file: cmd_template = template_file.readlines() for line_tmpl in cmd_template: - vat_cmd = line_tmpl.format(**args) + vat_cmd = line_tmpl.format(**kwargs) self.execute('echo "{c}" >> /tmp/running.exec' .format(c=vat_cmd.replace('\n', ''))) @@ -354,6 +364,45 @@ class ContainerEngine(object): """Check if container is present.""" raise NotImplementedError + def _configure_cgroup(self, name): + """Configure the control group associated with a container. + + By default the cpuset cgroup is using exclusive CPU/MEM. When Docker/LXC + container is initialized a new cgroup /docker or /lxc is created under + cpuset parent tree. This newly created cgroup is inheriting parent + setting for cpu/mem exclusive parameter and thus cannot be overriden + within /docker or /lxc cgroup. This function is supposed to set cgroups + to allow coexistence of both engines. + + :param name: Name of cgroup. + :type name: str + :raises RuntimeError: If applying cgroup settings via cgset failed. + """ + ret, _, _ = self.container.ssh.exec_command_sudo( + 'cgset -r cpuset.cpu_exclusive=0 /') + if int(ret) != 0: + raise RuntimeError('Failed to apply cgroup settings.') + + ret, _, _ = self.container.ssh.exec_command_sudo( + 'cgset -r cpuset.mem_exclusive=0 /') + if int(ret) != 0: + raise RuntimeError('Failed to apply cgroup settings.') + + ret, _, _ = self.container.ssh.exec_command_sudo( + 'cgcreate -g cpuset:/{name}'.format(name=name)) + if int(ret) != 0: + raise RuntimeError('Failed to copy cgroup settings from root.') + + ret, _, _ = self.container.ssh.exec_command_sudo( + 'cgset -r cpuset.cpu_exclusive=0 /{name}'.format(name=name)) + if int(ret) != 0: + raise RuntimeError('Failed to apply cgroup settings.') + + ret, _, _ = self.container.ssh.exec_command_sudo( + 'cgset -r cpuset.mem_exclusive=0 /{name}'.format(name=name)) + if int(ret) != 0: + raise RuntimeError('Failed to apply cgroup settings.') + class LXC(ContainerEngine): """LXC implementation.""" @@ -363,14 +412,13 @@ class LXC(ContainerEngine): super(LXC, self).__init__() def acquire(self, force=True): - """Acquire a privileged system object where configuration is stored and - where user information can be stored. + """Acquire a privileged system object where configuration is stored. :param force: If a container exists, destroy it and create a new - container. + container. :type force: bool :raises RuntimeError: If creating the container or writing the container - config fails. + config fails. """ if self.is_container_present(): if force: @@ -388,40 +436,58 @@ class LXC(ContainerEngine): if int(ret) != 0: raise RuntimeError('Failed to create container.') - if self.container.host_dir and self.container.guest_dir: - entry = 'lxc.mount.entry = '\ - '{c.host_dir} /var/lib/lxc/{c.name}/rootfs{c.guest_dir} ' \ - 'none bind,create=dir 0 0'.format(c=self.container) - ret, _, _ = self.container.ssh.exec_command_sudo( - "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'" - .format(e=entry, c=self.container)) - if int(ret) != 0: - raise RuntimeError('Failed to write {c.name} config.' - .format(c=self.container)) + self._configure_cgroup('lxc') def create(self): """Create/deploy an application inside a container on system. :raises RuntimeError: If creating the container fails. """ + if self.container.mnt: + for mount in self.container.mnt: + host_dir, guest_dir = mount.split(':') + entry = 'lxc.mount.entry = {host_dir} '\ + '/var/lib/lxc/{c.name}/rootfs{guest_dir} none ' \ + 'bind,create=dir 0 0'.format(c=self.container, + host_dir=host_dir, + guest_dir=guest_dir) + ret, _, _ = self.container.ssh.exec_command_sudo( + "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'". + format(e=entry, c=self.container)) + if int(ret) != 0: + raise RuntimeError('Failed to write {c.name} config.' + .format(c=self.container)) + cpuset_cpus = '{0}'.format( ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\ if self.container.cpuset_cpus else '' - cmd = 'lxc-start --name {c.name} --daemon'.format(c=self.container) - - ret, _, _ = self.container.ssh.exec_command_sudo(cmd) + ret, _, _ = self.container.ssh.exec_command_sudo( + 'lxc-start --name {c.name} --daemon'. + format(c=self.container)) if int(ret) != 0: - raise RuntimeError('Failed to start container {c.name}.' - .format(c=self.container)) + raise RuntimeError('Failed to start container {c.name}.'. + format(c=self.container)) self._lxc_wait('RUNNING') - self._lxc_cgroup(state_object='cpuset.cpus', - value=cpuset_cpus) + + # Workaround for LXC to be able to allocate all cpus including isolated. + ret, _, _ = self.container.ssh.exec_command_sudo( + 'cgset --copy-from / lxc/') + if int(ret) != 0: + raise RuntimeError('Failed to copy cgroup to LXC') + + ret, _, _ = self.container.ssh.exec_command_sudo( + 'lxc-cgroup --name {c.name} cpuset.cpus {cpus}'. + format(c=self.container, cpus=cpuset_cpus)) + if int(ret) != 0: + raise RuntimeError('Failed to set cpuset.cpus to container ' + '{c.name}.'.format(c=self.container)) def execute(self, command): - """Start a process inside a running container. Runs the specified - command inside the container specified by name. The container has to - be running already. + """Start a process inside a running container. + + Runs the specified command inside the container specified by name. The + container has to be running already. :param command: Command to run inside container. :type command: str @@ -431,8 +497,8 @@ class LXC(ContainerEngine): ' '.join('--set-var %s' % env for env in self.container.env))\ if self.container.env else '' - cmd = "lxc-attach {env} --name {c.name} -- /bin/sh -c '{command}'"\ - .format(env=env, c=self.container, command=command) + cmd = "lxc-attach {env} --name {c.name} -- /bin/sh -c '{command}; "\ + "exit $?'".format(env=env, c=self.container, command=command) ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180) if int(ret) != 0: @@ -530,33 +596,6 @@ class LXC(ContainerEngine): raise RuntimeError('Failed to wait for state "{s}" of container ' '{c.name}.'.format(s=state, c=self.container)) - def _lxc_cgroup(self, state_object, value=''): - """Manage the control group associated with a container. - - :param state_object: Specify the state object name. - :param value: Specify the value to assign to the state object. If empty, - then action is GET, otherwise is action SET. - :type state_object: str - :type value: str - :raises RuntimeError: If getting/setting state of a container failed. - """ - cmd = 'lxc-cgroup --name {c.name} {s} {v}'\ - .format(c=self.container, s=state_object, v=value) - - ret, _, _ = self.container.ssh.exec_command_sudo( - 'cgset --copy-from / lxc') - if int(ret) != 0: - raise RuntimeError('Failed to copy cgroup settings from root.') - - ret, _, _ = self.container.ssh.exec_command_sudo(cmd) - if int(ret) != 0: - if value: - raise RuntimeError('Failed to set {s} of container {c.name}.' - .format(s=state_object, c=self.container)) - else: - raise RuntimeError('Failed to get {s} of container {c.name}.' - .format(s=state_object, c=self.container)) - class Docker(ContainerEngine): """Docker implementation.""" @@ -584,6 +623,7 @@ class Docker(ContainerEngine): if int(ret) != 0: raise RuntimeError('Failed to create container {c.name}.' .format(c=self.container)) + self._configure_cgroup('docker') def create(self): """Create/deploy container. @@ -596,6 +636,8 @@ class Docker(ContainerEngine): cpuset_mems = '--cpuset-mems={0}'.format(self.container.cpuset_mems)\ if self.container.cpuset_mems is not None else '' + # Temporary workaround - disabling due to bug in memif + cpuset_mems = '' env = '{0}'.format( ' '.join('--env %s' % env for env in self.container.env))\ @@ -608,12 +650,13 @@ class Docker(ContainerEngine): ' '.join('--publish %s' % var for var in self.container.publish))\ if self.container.publish else '' - volume = '--volume {c.host_dir}:{c.guest_dir}'.format(c=self.container)\ - if self.container.host_dir and self.container.guest_dir else '' + volume = '{0}'.format( + ' '.join('--volume %s' % mnt for mnt in self.container.mnt))\ + if self.container.mnt else '' cmd = 'docker run '\ '--privileged --detach --interactive --tty --rm '\ - '--cgroup-parent lxc {cpuset_cpus} {cpuset_mems} {publish} '\ + '--cgroup-parent docker {cpuset_cpus} {cpuset_mems} {publish} '\ '{env} {volume} --name {container.name} {container.image} '\ '{command}'.format(cpuset_cpus=cpuset_cpus, cpuset_mems=cpuset_mems, container=self.container, command=command, @@ -627,16 +670,17 @@ class Docker(ContainerEngine): self.info() def execute(self, command): - """Start a process inside a running container. Runs the specified - command inside the container specified by name. The container has to - be running already. + """Start a process inside a running container. + + Runs the specified command inside the container specified by name. The + container has to be running already. :param command: Command to run inside container. :type command: str :raises RuntimeError: If runnig the command in a container failed. """ - cmd = "docker exec --interactive {c.name} /bin/sh -c '{command}'"\ - .format(c=self.container, command=command) + cmd = "docker exec --interactive {c.name} /bin/sh -c '{command}; "\ + "exit $?'".format(c=self.container, command=command) ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180) if int(ret) != 0: @@ -731,12 +775,26 @@ class Container(object): pass def __getattr__(self, attr): + """Get attribute custom implementation. + + :param attr: Attribute to get. + :type attr: str + :returns: Attribute value or None. + :rtype: any + """ try: return self.__dict__[attr] except KeyError: return None def __setattr__(self, attr, value): + """Set attribute custom implementation. + + :param attr: Attribute to set. + :param value: Value to set. + :type attr: str + :type value: any + """ try: # Check if attribute exists self.__dict__[attr]