-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
# limitations under the License.
# Bug workaround in pylint for abstract classes.
-#pylint: disable=W0223
+# pylint: disable=W0223
"""Library to manipulate Containers."""
try:
self.engine = globals()[engine]()
except KeyError:
- raise NotImplementedError('{e} is not implemented.'
- .format(e=engine))
+ raise NotImplementedError('{engine} is not implemented.'.
+ format(engine=engine))
self.containers = OrderedDict()
def get_container_by_name(self, name):
try:
return self.containers[name]
except KeyError:
- raise RuntimeError('Failed to get container with name: {n}'
- .format(n=name))
+ raise RuntimeError('Failed to get container with name: {name}'.
+ format(name=name))
def construct_container(self, **kwargs):
"""Construct container object on node with specified parameters.
# Set additional environmental variables
setattr(self.engine.container, 'env',
- 'MICROSERVICE_LABEL={n}'.format(n=kwargs['name']))
+ 'MICROSERVICE_LABEL={label}'.format(label=kwargs['name']))
# Set cpuset.cpus cgroup
skip_cnt = kwargs['cpu_skip']
def construct_containers(self, **kwargs):
"""Construct 1..N container(s) on node with specified name.
+
Ordinal number is automatically added to the name of container as
suffix.
- :param kwargs: Name of container.
- :param kwargs: str
+ :param kwargs: Named parameters.
+ :param kwargs: dict
"""
name = kwargs['name']
for i in range(kwargs['count']):
self.engine.install_vpp()
self.engine.restart_vpp()
+ def restart_vpp_in_all_containers(self):
+ """Restart VPP on all containers."""
+ for container in self.containers:
+ self.engine.container = self.containers[container]
+ self.engine.restart_vpp()
+
def configure_vpp_in_all_containers(self, vat_template_file):
"""Configure VPP in all containers.
dut_cnt = len(Counter([self.containers[container].node['host']
for container in self.containers]))
container_cnt = len(self.containers)
- mod = dut_cnt/container_cnt
+ mod = container_cnt/dut_cnt
for i, container in enumerate(self.containers):
+ mid1 = i % mod + 1
+ mid2 = i % mod + 1
+ sid1 = i % mod * 2 + 1
+ sid2 = i % mod * 2 + 2
self.engine.container = self.containers[container]
self.engine.create_vpp_startup_config()
- self.engine.create_vpp_exec_config(vat_template_file,
- memif_id1=i % mod * 2 + 1,
- memif_id2=i % mod * 2 + 2,
- socket1='memif-{c.name}-1'
- .format(c=self.engine.container),
- socket2='memif-{c.name}-2'
- .format(c=self.engine.container))
+ self.engine.create_vpp_exec_config(vat_template_file, mid1=mid1,
+ mid2=mid2, sid1=sid1, sid2=sid2,
+ socket1='memif-{c.name}-{sid}'
+ .format(c=self.engine.container,
+ sid=sid1),
+ socket2='memif-{c.name}-{sid}'
+ .format(c=self.engine.container,
+ sid=sid2))
def stop_all_containers(self):
"""Stop all containers."""
self.execute('sleep 3')
self.execute('apt-get update')
self.execute('apt-get install -y supervisor')
- self.execute('echo "{0}" > {1}'
- .format(
- '[unix_http_server]\n'
+ self.execute('echo "{config}" > {config_file}'.
+ format(
+ config='[unix_http_server]\n'
'file = /tmp/supervisor.sock\n\n'
'[rpcinterface:supervisor]\n'
'supervisor.rpcinterface_factory = '
'logfile=/tmp/supervisord.log\n'
'loglevel=debug\n'
'nodaemon=false\n\n',
- SUPERVISOR_CONF))
- self.execute('supervisord -c {0}'.format(SUPERVISOR_CONF))
+ config_file=SUPERVISOR_CONF))
+ self.execute('supervisord -c {config_file}'.
+ format(config_file=SUPERVISOR_CONF))
- def install_vpp(self, install_dkms=False):
- """Install VPP inside a container.
-
- :param install_dkms: If install dkms package. This will impact install
- time. Dkms is required for installation of vpp-dpdk-dkms. Default is
- false.
- :type install_dkms: bool
- """
+ def install_vpp(self):
+ """Install VPP inside a container."""
self.execute('ln -s /dev/null /etc/sysctl.d/80-vpp.conf')
self.execute('apt-get update')
- if install_dkms:
- self.execute('apt-get install -y dkms && '
- 'dpkg -i --force-all {0}/install_dir/*.deb'
- .format(self.container.guest_dir))
+ if self.container.install_dkms:
+ self.execute(
+ 'apt-get install -y dkms && '
+ 'dpkg -i --force-all {guest_dir}/install_dir/*.deb'.
+ format(guest_dir=self.container.mnt[0].split(':')[1]))
else:
- self.execute('for i in $(ls -I \"*dkms*\" {0}/install_dir/); '
- 'do dpkg -i --force-all {0}/install_dir/$i; done'
- .format(self.container.guest_dir))
+ self.execute(
+ 'for i in $(ls -I \"*dkms*\" {guest_dir}/install_dir/); do '
+ 'dpkg -i --force-all {guest_dir}/install_dir/$i; done'.
+ format(guest_dir=self.container.mnt[0].split(':')[1]))
self.execute('apt-get -f install -y')
- self.execute('echo "{0}" >> {1}'
- .format(
- '[program:vpp]\n'
+ self.execute('apt-get install -y ca-certificates')
+ self.execute('echo "{config}" >> {config_file}'.
+ format(
+ config='[program:vpp]\n'
'command=/usr/bin/vpp -c /etc/vpp/startup.conf\n'
'autorestart=false\n'
'redirect_stderr=true\n'
'priority=1',
- SUPERVISOR_CONF))
+ config_file=SUPERVISOR_CONF))
self.execute('supervisorctl reload')
def restart_vpp(self):
"""Restart VPP service inside a container."""
self.execute('supervisorctl restart vpp')
+ self.execute('cat /tmp/supervisord.log')
def create_vpp_startup_config(self,
config_filename='/etc/vpp/startup.conf'):
# Create config instance
vpp_config = VppConfigGenerator()
vpp_config.set_node(self.container.node)
- vpp_config.set_config_filename(config_filename)
vpp_config.add_unix_cli_listen()
vpp_config.add_unix_nodaemon()
vpp_config.add_unix_exec('/tmp/running.exec')
if cpuset_cpus:
corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
vpp_config.add_cpu_corelist_workers(corelist_workers)
- vpp_config.add_plugin_disable('dpdk_plugin.so')
+ vpp_config.add_plugin('disable', 'dpdk_plugin.so')
self.execute('mkdir -p /etc/vpp/')
self.execute('echo "{c}" | tee {f}'
.format(c=vpp_config.get_config_str(),
- f=vpp_config.get_config_filename()))
+ f=config_filename))
- def create_vpp_exec_config(self, vat_template_file, **args):
+ def create_vpp_exec_config(self, vat_template_file, **kwargs):
"""Create VPP exec configuration on container.
:param vat_template_file: File name of a VAT template script.
- :param args: Parameters for VAT script.
+ :param kwargs: Parameters for VAT script.
:type vat_template_file: str
- :type args: dict
+ :type kwargs: dict
"""
vat_file_path = '{p}/{f}'.format(p=Constants.RESOURCES_TPL_VAT,
f=vat_template_file)
with open(vat_file_path, 'r') as template_file:
cmd_template = template_file.readlines()
for line_tmpl in cmd_template:
- vat_cmd = line_tmpl.format(**args)
+ vat_cmd = line_tmpl.format(**kwargs)
self.execute('echo "{c}" >> /tmp/running.exec'
.format(c=vat_cmd.replace('\n', '')))
"""Check if container is present."""
raise NotImplementedError
+ def _configure_cgroup(self, name):
+ """Configure the control group associated with a container.
+
+ By default the cpuset cgroup is using exclusive CPU/MEM. When Docker/LXC
+ container is initialized a new cgroup /docker or /lxc is created under
+ cpuset parent tree. This newly created cgroup is inheriting parent
+ setting for cpu/mem exclusive parameter and thus cannot be overriden
+ within /docker or /lxc cgroup. This function is supposed to set cgroups
+ to allow coexistence of both engines.
+
+ :param name: Name of cgroup.
+ :type name: str
+ :raises RuntimeError: If applying cgroup settings via cgset failed.
+ """
+ ret, _, _ = self.container.ssh.exec_command_sudo(
+ 'cgset -r cpuset.cpu_exclusive=0 /')
+ if int(ret) != 0:
+ raise RuntimeError('Failed to apply cgroup settings.')
+
+ ret, _, _ = self.container.ssh.exec_command_sudo(
+ 'cgset -r cpuset.mem_exclusive=0 /')
+ if int(ret) != 0:
+ raise RuntimeError('Failed to apply cgroup settings.')
+
+ ret, _, _ = self.container.ssh.exec_command_sudo(
+ 'cgcreate -g cpuset:/{name}'.format(name=name))
+ if int(ret) != 0:
+ raise RuntimeError('Failed to copy cgroup settings from root.')
+
+ ret, _, _ = self.container.ssh.exec_command_sudo(
+ 'cgset -r cpuset.cpu_exclusive=0 /{name}'.format(name=name))
+ if int(ret) != 0:
+ raise RuntimeError('Failed to apply cgroup settings.')
+
+ ret, _, _ = self.container.ssh.exec_command_sudo(
+ 'cgset -r cpuset.mem_exclusive=0 /{name}'.format(name=name))
+ if int(ret) != 0:
+ raise RuntimeError('Failed to apply cgroup settings.')
+
class LXC(ContainerEngine):
"""LXC implementation."""
super(LXC, self).__init__()
def acquire(self, force=True):
- """Acquire a privileged system object where configuration is stored and
- where user information can be stored.
+ """Acquire a privileged system object where configuration is stored.
:param force: If a container exists, destroy it and create a new
- container.
+ container.
:type force: bool
:raises RuntimeError: If creating the container or writing the container
- config fails.
+ config fails.
"""
if self.is_container_present():
if force:
if int(ret) != 0:
raise RuntimeError('Failed to create container.')
- if self.container.host_dir and self.container.guest_dir:
- entry = 'lxc.mount.entry = '\
- '{c.host_dir} /var/lib/lxc/{c.name}/rootfs{c.guest_dir} ' \
- 'none bind,create=dir 0 0'.format(c=self.container)
- ret, _, _ = self.container.ssh.exec_command_sudo(
- "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'"
- .format(e=entry, c=self.container))
- if int(ret) != 0:
- raise RuntimeError('Failed to write {c.name} config.'
- .format(c=self.container))
+ self._configure_cgroup('lxc')
def create(self):
"""Create/deploy an application inside a container on system.
:raises RuntimeError: If creating the container fails.
"""
+ if self.container.mnt:
+ for mount in self.container.mnt:
+ host_dir, guest_dir = mount.split(':')
+ entry = 'lxc.mount.entry = {host_dir} '\
+ '/var/lib/lxc/{c.name}/rootfs{guest_dir} none ' \
+ 'bind,create=dir 0 0'.format(c=self.container,
+ host_dir=host_dir,
+ guest_dir=guest_dir)
+ ret, _, _ = self.container.ssh.exec_command_sudo(
+ "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'".
+ format(e=entry, c=self.container))
+ if int(ret) != 0:
+ raise RuntimeError('Failed to write {c.name} config.'
+ .format(c=self.container))
+
cpuset_cpus = '{0}'.format(
','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
if self.container.cpuset_cpus else ''
- cmd = 'lxc-start --name {c.name} --daemon'.format(c=self.container)
-
- ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
+ ret, _, _ = self.container.ssh.exec_command_sudo(
+ 'lxc-start --name {c.name} --daemon'.
+ format(c=self.container))
if int(ret) != 0:
- raise RuntimeError('Failed to start container {c.name}.'
- .format(c=self.container))
+ raise RuntimeError('Failed to start container {c.name}.'.
+ format(c=self.container))
self._lxc_wait('RUNNING')
- self._lxc_cgroup(state_object='cpuset.cpus',
- value=cpuset_cpus)
+
+ # Workaround for LXC to be able to allocate all cpus including isolated.
+ ret, _, _ = self.container.ssh.exec_command_sudo(
+ 'cgset --copy-from / lxc/')
+ if int(ret) != 0:
+ raise RuntimeError('Failed to copy cgroup to LXC')
+
+ ret, _, _ = self.container.ssh.exec_command_sudo(
+ 'lxc-cgroup --name {c.name} cpuset.cpus {cpus}'.
+ format(c=self.container, cpus=cpuset_cpus))
+ if int(ret) != 0:
+ raise RuntimeError('Failed to set cpuset.cpus to container '
+ '{c.name}.'.format(c=self.container))
def execute(self, command):
- """Start a process inside a running container. Runs the specified
- command inside the container specified by name. The container has to
- be running already.
+ """Start a process inside a running container.
+
+ Runs the specified command inside the container specified by name. The
+ container has to be running already.
:param command: Command to run inside container.
:type command: str
' '.join('--set-var %s' % env for env in self.container.env))\
if self.container.env else ''
- cmd = "lxc-attach {env} --name {c.name} -- /bin/sh -c '{command}'"\
- .format(env=env, c=self.container, command=command)
+ cmd = "lxc-attach {env} --name {c.name} -- /bin/sh -c '{command}; "\
+ "exit $?'".format(env=env, c=self.container, command=command)
ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
if int(ret) != 0:
raise RuntimeError('Failed to wait for state "{s}" of container '
'{c.name}.'.format(s=state, c=self.container))
- def _lxc_cgroup(self, state_object, value=''):
- """Manage the control group associated with a container.
-
- :param state_object: Specify the state object name.
- :param value: Specify the value to assign to the state object. If empty,
- then action is GET, otherwise is action SET.
- :type state_object: str
- :type value: str
- :raises RuntimeError: If getting/setting state of a container failed.
- """
- cmd = 'lxc-cgroup --name {c.name} {s} {v}'\
- .format(c=self.container, s=state_object, v=value)
-
- ret, _, _ = self.container.ssh.exec_command_sudo(
- 'cgset --copy-from / lxc')
- if int(ret) != 0:
- raise RuntimeError('Failed to copy cgroup settings from root.')
-
- ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
- if int(ret) != 0:
- if value:
- raise RuntimeError('Failed to set {s} of container {c.name}.'
- .format(s=state_object, c=self.container))
- else:
- raise RuntimeError('Failed to get {s} of container {c.name}.'
- .format(s=state_object, c=self.container))
-
class Docker(ContainerEngine):
"""Docker implementation."""
if int(ret) != 0:
raise RuntimeError('Failed to create container {c.name}.'
.format(c=self.container))
+ self._configure_cgroup('docker')
def create(self):
"""Create/deploy container.
cpuset_mems = '--cpuset-mems={0}'.format(self.container.cpuset_mems)\
if self.container.cpuset_mems is not None else ''
+ # Temporary workaround - disabling due to bug in memif
+ cpuset_mems = ''
env = '{0}'.format(
' '.join('--env %s' % env for env in self.container.env))\
' '.join('--publish %s' % var for var in self.container.publish))\
if self.container.publish else ''
- volume = '--volume {c.host_dir}:{c.guest_dir}'.format(c=self.container)\
- if self.container.host_dir and self.container.guest_dir else ''
+ volume = '{0}'.format(
+ ' '.join('--volume %s' % mnt for mnt in self.container.mnt))\
+ if self.container.mnt else ''
cmd = 'docker run '\
'--privileged --detach --interactive --tty --rm '\
- '--cgroup-parent lxc {cpuset_cpus} {cpuset_mems} {publish} '\
+ '--cgroup-parent docker {cpuset_cpus} {cpuset_mems} {publish} '\
'{env} {volume} --name {container.name} {container.image} '\
'{command}'.format(cpuset_cpus=cpuset_cpus, cpuset_mems=cpuset_mems,
container=self.container, command=command,
self.info()
def execute(self, command):
- """Start a process inside a running container. Runs the specified
- command inside the container specified by name. The container has to
- be running already.
+ """Start a process inside a running container.
+
+ Runs the specified command inside the container specified by name. The
+ container has to be running already.
:param command: Command to run inside container.
:type command: str
:raises RuntimeError: If runnig the command in a container failed.
"""
- cmd = "docker exec --interactive {c.name} /bin/sh -c '{command}'"\
- .format(c=self.container, command=command)
+ cmd = "docker exec --interactive {c.name} /bin/sh -c '{command}; "\
+ "exit $?'".format(c=self.container, command=command)
ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
if int(ret) != 0:
pass
def __getattr__(self, attr):
+ """Get attribute custom implementation.
+
+ :param attr: Attribute to get.
+ :type attr: str
+ :returns: Attribute value or None.
+ :rtype: any
+ """
try:
return self.__dict__[attr]
except KeyError:
return None
def __setattr__(self, attr, value):
+ """Set attribute custom implementation.
+
+ :param attr: Attribute to set.
+ :param value: Value to set.
+ :type attr: str
+ :type value: any
+ """
try:
# Check if attribute exists
self.__dict__[attr]
Code Review / csit.git / blobdiff