1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 # Bug workaround in pylint for abstract classes.
15 # pylint: disable=W0223
17 """Library to manipulate Containers."""
19 from collections import OrderedDict, Counter
21 from resources.libraries.python.ssh import SSH
22 from resources.libraries.python.constants import Constants
23 from resources.libraries.python.CpuUtils import CpuUtils
24 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
27 __all__ = ["ContainerManager", "ContainerEngine", "LXC", "Docker", "Container"]
29 SUPERVISOR_CONF = '/etc/supervisord.conf'
32 class ContainerManager(object):
33 """Container lifecycle management class."""
35 def __init__(self, engine):
36 """Initialize Container Manager class.
38 :param engine: Container technology used (LXC/Docker/...).
40 :raises NotImplementedError: If container technology is not implemented.
43 self.engine = globals()[engine]()
45 raise NotImplementedError('{e} is not implemented.'
47 self.containers = OrderedDict()
49 def get_container_by_name(self, name):
50 """Get container instance.
52 :param name: Container name.
54 :returns: Container instance.
56 :raises RuntimeError: If failed to get container with name.
59 return self.containers[name]
61 raise RuntimeError('Failed to get container with name: {n}'
64 def construct_container(self, **kwargs):
65 """Construct container object on node with specified parameters.
67 :param kwargs: Key-value pairs used to construct container.
71 self.engine.initialize()
74 setattr(self.engine.container, key, kwargs[key])
76 # Set additional environmental variables
77 setattr(self.engine.container, 'env',
78 'MICROSERVICE_LABEL={n}'.format(n=kwargs['name']))
80 # Set cpuset.cpus cgroup
81 skip_cnt = kwargs['cpu_skip']
82 if not kwargs['cpu_shared']:
83 skip_cnt += kwargs['i'] * kwargs['cpu_count']
84 self.engine.container.cpuset_cpus = \
85 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
86 cpu_node=kwargs['cpuset_mems'],
88 cpu_cnt=kwargs['cpu_count'],
89 smt_used=kwargs['smt_used'])
91 # Store container instance
92 self.containers[kwargs['name']] = self.engine.container
94 def construct_containers(self, **kwargs):
95 """Construct 1..N container(s) on node with specified name.
97 Ordinal number is automatically added to the name of container as
100 :param kwargs: Named parameters.
103 name = kwargs['name']
104 for i in range(kwargs['count']):
105 # Name will contain ordinal suffix
106 kwargs['name'] = ''.join([name, str(i+1)])
108 self.construct_container(i=i, **kwargs)
110 def acquire_all_containers(self):
111 """Acquire all containers."""
112 for container in self.containers:
113 self.engine.container = self.containers[container]
114 self.engine.acquire()
116 def build_all_containers(self):
117 """Build all containers."""
118 for container in self.containers:
119 self.engine.container = self.containers[container]
122 def create_all_containers(self):
123 """Create all containers."""
124 for container in self.containers:
125 self.engine.container = self.containers[container]
128 def execute_on_container(self, name, command):
129 """Execute command on container with name.
131 :param name: Container name.
132 :param command: Command to execute.
136 self.engine.container = self.get_container_by_name(name)
137 self.engine.execute(command)
139 def execute_on_all_containers(self, command):
140 """Execute command on all containers.
142 :param command: Command to execute.
145 for container in self.containers:
146 self.engine.container = self.containers[container]
147 self.engine.execute(command)
149 def install_vpp_in_all_containers(self):
150 """Install VPP into all containers."""
151 for container in self.containers:
152 self.engine.container = self.containers[container]
153 # We need to install supervisor client/server system to control VPP
155 self.engine.install_supervisor()
156 self.engine.install_vpp()
157 self.engine.restart_vpp()
159 def restart_vpp_in_all_containers(self):
160 """Restart VPP on all containers."""
161 for container in self.containers:
162 self.engine.container = self.containers[container]
163 self.engine.restart_vpp()
165 def configure_vpp_in_all_containers(self, vat_template_file):
166 """Configure VPP in all containers.
168 :param vat_template_file: Template file name of a VAT script.
169 :type vat_template_file: str
171 # Count number of DUTs based on node's host information
172 dut_cnt = len(Counter([self.containers[container].node['host']
173 for container in self.containers]))
174 container_cnt = len(self.containers)
175 mod = container_cnt/dut_cnt
177 for i, container in enumerate(self.containers):
180 sid1 = i % mod * 2 + 1
181 sid2 = i % mod * 2 + 2
182 self.engine.container = self.containers[container]
183 self.engine.create_vpp_startup_config()
184 self.engine.create_vpp_exec_config(vat_template_file, mid1=mid1,
185 mid2=mid2, sid1=sid1, sid2=sid2,
186 socket1='memif-{c.name}-{sid}'
187 .format(c=self.engine.container,
189 socket2='memif-{c.name}-{sid}'
190 .format(c=self.engine.container,
193 def stop_all_containers(self):
194 """Stop all containers."""
195 for container in self.containers:
196 self.engine.container = self.containers[container]
199 def destroy_all_containers(self):
200 """Destroy all containers."""
201 for container in self.containers:
202 self.engine.container = self.containers[container]
203 self.engine.destroy()
206 class ContainerEngine(object):
207 """Abstract class for container engine."""
210 """Init ContainerEngine object."""
211 self.container = None
213 def initialize(self):
214 """Initialize container object."""
215 self.container = Container()
217 def acquire(self, force):
218 """Acquire/download container.
220 :param force: Destroy a container if exists and create.
223 raise NotImplementedError
226 """Build container (compile)."""
227 raise NotImplementedError
230 """Create/deploy container."""
231 raise NotImplementedError
233 def execute(self, command):
234 """Execute process inside container.
236 :param command: Command to run inside container.
239 raise NotImplementedError
242 """Stop container."""
243 raise NotImplementedError
246 """Destroy/remove container."""
247 raise NotImplementedError
250 """Info about container."""
251 raise NotImplementedError
253 def system_info(self):
255 raise NotImplementedError
257 def install_supervisor(self):
258 """Install supervisord inside a container."""
259 self.execute('sleep 3')
260 self.execute('apt-get update')
261 self.execute('apt-get install -y supervisor')
262 self.execute('echo "{0}" > {1}'
264 '[unix_http_server]\n'
265 'file = /tmp/supervisor.sock\n\n'
266 '[rpcinterface:supervisor]\n'
267 'supervisor.rpcinterface_factory = '
268 'supervisor.rpcinterface:make_main_rpcinterface\n\n'
270 'serverurl = unix:///tmp/supervisor.sock\n\n'
272 'pidfile = /tmp/supervisord.pid\n'
273 'identifier = supervisor\n'
275 'logfile=/tmp/supervisord.log\n'
277 'nodaemon=false\n\n',
279 self.execute('supervisord -c {0}'.format(SUPERVISOR_CONF))
281 def install_vpp(self, install_dkms=False):
282 """Install VPP inside a container.
284 :param install_dkms: If install dkms package. This will impact
285 install time. Dkms is required for installation of vpp-dpdk-dkms.
287 :type install_dkms: bool
289 self.execute('ln -s /dev/null /etc/sysctl.d/80-vpp.conf')
290 self.execute('apt-get update')
292 self.execute('apt-get install -y dkms && '
293 'dpkg -i --force-all {0}/install_dir/*.deb'
294 .format(self.container.guest_dir))
296 self.execute('for i in $(ls -I \"*dkms*\" {0}/install_dir/); '
297 'do dpkg -i --force-all {0}/install_dir/$i; done'
298 .format(self.container.guest_dir))
299 self.execute('apt-get -f install -y')
300 self.execute('apt-get install -y ca-certificates')
301 self.execute('echo "{0}" >> {1}'
304 'command=/usr/bin/vpp -c /etc/vpp/startup.conf\n'
305 'autorestart=false\n'
306 'redirect_stderr=true\n'
309 self.execute('supervisorctl reload')
311 def restart_vpp(self):
312 """Restart VPP service inside a container."""
313 self.execute('supervisorctl restart vpp')
314 self.execute('cat /tmp/supervisord.log')
316 def create_vpp_startup_config(self,
317 config_filename='/etc/vpp/startup.conf'):
318 """Create base startup configuration of VPP on container.
320 :param config_filename: Startup configuration file name.
321 :type config_filename: str
323 cpuset_cpus = self.container.cpuset_cpus
325 # Create config instance
326 vpp_config = VppConfigGenerator()
327 vpp_config.set_node(self.container.node)
328 vpp_config.add_unix_cli_listen()
329 vpp_config.add_unix_nodaemon()
330 vpp_config.add_unix_exec('/tmp/running.exec')
331 # We will pop first core from list to be main core
332 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
333 # if this is not only core in list, the rest will be used as workers.
335 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
336 vpp_config.add_cpu_corelist_workers(corelist_workers)
337 vpp_config.add_plugin('disable', 'dpdk_plugin.so')
339 self.execute('mkdir -p /etc/vpp/')
340 self.execute('echo "{c}" | tee {f}'
341 .format(c=vpp_config.get_config_str(),
344 def create_vpp_exec_config(self, vat_template_file, **kwargs):
345 """Create VPP exec configuration on container.
347 :param vat_template_file: File name of a VAT template script.
348 :param kwargs: Parameters for VAT script.
349 :type vat_template_file: str
352 vat_file_path = '{p}/{f}'.format(p=Constants.RESOURCES_TPL_VAT,
355 with open(vat_file_path, 'r') as template_file:
356 cmd_template = template_file.readlines()
357 for line_tmpl in cmd_template:
358 vat_cmd = line_tmpl.format(**kwargs)
359 self.execute('echo "{c}" >> /tmp/running.exec'
360 .format(c=vat_cmd.replace('\n', '')))
362 def is_container_running(self):
363 """Check if container is running."""
364 raise NotImplementedError
366 def is_container_present(self):
367 """Check if container is present."""
368 raise NotImplementedError
370 def _configure_cgroup(self, name):
371 """Configure the control group associated with a container.
373 By default the cpuset cgroup is using exclusive CPU/MEM. When Docker
374 container is initialized a new cgroup /docker or /lxc is created under
375 cpuset parent tree. This newly created cgroup is inheriting parent
376 setting for cpu/mem exclusive parameter and thus cannot be overriden
377 within /docker or /lxc cgroup. This patch is supposed to set cpu/mem
378 exclusive parameter for both parent and subgroup.
380 :param name: Name of cgroup.
382 :raises RuntimeError: If applying cgroup settings via cgset failed.
384 ret, _, _ = self.container.ssh.exec_command_sudo(
385 'cgset -r cpuset.cpu_exclusive=0 /')
387 raise RuntimeError('Failed to apply cgroup settings.')
389 ret, _, _ = self.container.ssh.exec_command_sudo(
390 'cgset -r cpuset.mem_exclusive=0 /')
392 raise RuntimeError('Failed to apply cgroup settings.')
394 ret, _, _ = self.container.ssh.exec_command_sudo(
395 'cgcreate -g cpuset:/{name}'.format(name=name))
397 raise RuntimeError('Failed to copy cgroup settings from root.')
399 ret, _, _ = self.container.ssh.exec_command_sudo(
400 'cgset -r cpuset.cpu_exclusive=0 /{name}'.format(name=name))
402 raise RuntimeError('Failed to apply cgroup settings.')
404 ret, _, _ = self.container.ssh.exec_command_sudo(
405 'cgset -r cpuset.mem_exclusive=0 /{name}'.format(name=name))
407 raise RuntimeError('Failed to apply cgroup settings.')
410 class LXC(ContainerEngine):
411 """LXC implementation."""
414 """Initialize LXC object."""
415 super(LXC, self).__init__()
417 def acquire(self, force=True):
418 """Acquire a privileged system object where configuration is stored.
420 :param force: If a container exists, destroy it and create a new
423 :raises RuntimeError: If creating the container or writing the container
426 if self.is_container_present():
432 image = self.container.image if self.container.image else\
433 "-d ubuntu -r xenial -a amd64"
435 cmd = 'lxc-create -t download --name {c.name} -- {image} '\
436 '--no-validate'.format(c=self.container, image=image)
438 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
440 raise RuntimeError('Failed to create container.')
442 if self.container.host_dir and self.container.guest_dir:
443 entry = 'lxc.mount.entry = '\
444 '{c.host_dir} /var/lib/lxc/{c.name}/rootfs{c.guest_dir} ' \
445 'none bind,create=dir 0 0'.format(c=self.container)
446 ret, _, _ = self.container.ssh.exec_command_sudo(
447 "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'"
448 .format(e=entry, c=self.container))
450 raise RuntimeError('Failed to write {c.name} config.'
451 .format(c=self.container))
452 self._configure_cgroup('lxc')
455 """Create/deploy an application inside a container on system.
457 :raises RuntimeError: If creating the container fails.
459 cpuset_cpus = '{0}'.format(
460 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
461 if self.container.cpuset_cpus else ''
463 cmd = 'lxc-start --name {c.name} --daemon'.format(c=self.container)
465 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
467 raise RuntimeError('Failed to start container {c.name}.'
468 .format(c=self.container))
469 self._lxc_wait('RUNNING')
471 # Workaround for LXC to be able to allocate all cpus including isolated.
472 cmd = 'cgset --copy-from / lxc/'
473 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
475 raise RuntimeError('Failed to copy cgroup to LXC')
477 cmd = 'lxc-cgroup --name {c.name} cpuset.cpus {cpus}'\
478 .format(c=self.container, cpus=cpuset_cpus)
479 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
481 raise RuntimeError('Failed to set cpuset.cpus to container '
482 '{c.name}.'.format(c=self.container))
484 def execute(self, command):
485 """Start a process inside a running container.
487 Runs the specified command inside the container specified by name. The
488 container has to be running already.
490 :param command: Command to run inside container.
492 :raises RuntimeError: If running the command failed.
494 env = '--keep-env {0}'.format(
495 ' '.join('--set-var %s' % env for env in self.container.env))\
496 if self.container.env else ''
498 cmd = "lxc-attach {env} --name {c.name} -- /bin/sh -c '{command}'"\
499 .format(env=env, c=self.container, command=command)
501 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
503 raise RuntimeError('Failed to run command inside container '
504 '{c.name}.'.format(c=self.container))
509 :raises RuntimeError: If stopping the container failed.
511 cmd = 'lxc-stop --name {c.name}'.format(c=self.container)
513 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
515 raise RuntimeError('Failed to stop container {c.name}.'
516 .format(c=self.container))
517 self._lxc_wait('STOPPED|FROZEN')
520 """Destroy a container.
522 :raises RuntimeError: If destroying container failed.
524 cmd = 'lxc-destroy --force --name {c.name}'.format(c=self.container)
526 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
528 raise RuntimeError('Failed to destroy container {c.name}.'
529 .format(c=self.container))
532 """Query and shows information about a container.
534 :raises RuntimeError: If getting info about a container failed.
536 cmd = 'lxc-info --name {c.name}'.format(c=self.container)
538 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
540 raise RuntimeError('Failed to get info about container {c.name}.'
541 .format(c=self.container))
543 def system_info(self):
544 """Check the current kernel for LXC support.
546 :raises RuntimeError: If checking LXC support failed.
548 cmd = 'lxc-checkconfig'
550 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
552 raise RuntimeError('Failed to check LXC support.')
554 def is_container_running(self):
555 """Check if container is running on node.
557 :returns: True if container is running.
559 :raises RuntimeError: If getting info about a container failed.
561 cmd = 'lxc-info --no-humanize --state --name {c.name}'\
562 .format(c=self.container)
564 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
566 raise RuntimeError('Failed to get info about container {c.name}.'
567 .format(c=self.container))
568 return True if 'RUNNING' in stdout else False
570 def is_container_present(self):
571 """Check if container is existing on node.
573 :returns: True if container is present.
575 :raises RuntimeError: If getting info about a container failed.
577 cmd = 'lxc-info --no-humanize --name {c.name}'.format(c=self.container)
579 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
580 return False if int(ret) else True
582 def _lxc_wait(self, state):
583 """Wait for a specific container state.
585 :param state: Specify the container state(s) to wait for.
587 :raises RuntimeError: If waiting for state of a container failed.
589 cmd = 'lxc-wait --name {c.name} --state "{s}"'\
590 .format(c=self.container, s=state)
592 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
594 raise RuntimeError('Failed to wait for state "{s}" of container '
595 '{c.name}.'.format(s=state, c=self.container))
598 class Docker(ContainerEngine):
599 """Docker implementation."""
602 """Initialize Docker object."""
603 super(Docker, self).__init__()
605 def acquire(self, force=True):
606 """Pull an image or a repository from a registry.
608 :param force: Destroy a container if exists.
610 :raises RuntimeError: If pulling a container failed.
612 if self.is_container_present():
618 cmd = 'docker pull {c.image}'.format(c=self.container)
620 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
622 raise RuntimeError('Failed to create container {c.name}.'
623 .format(c=self.container))
624 self._configure_cgroup('docker')
627 """Create/deploy container.
629 :raises RuntimeError: If creating a container failed.
631 cpuset_cpus = '--cpuset-cpus={0}'.format(
632 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
633 if self.container.cpuset_cpus else ''
635 cpuset_mems = '--cpuset-mems={0}'.format(self.container.cpuset_mems)\
636 if self.container.cpuset_mems is not None else ''
637 # Temporary workaround - disabling due to bug in memif
641 ' '.join('--env %s' % env for env in self.container.env))\
642 if self.container.env else ''
644 command = '{0}'.format(self.container.command)\
645 if self.container.command else ''
647 publish = '{0}'.format(
648 ' '.join('--publish %s' % var for var in self.container.publish))\
649 if self.container.publish else ''
651 volume = '--volume {c.host_dir}:{c.guest_dir}'.format(c=self.container)\
652 if self.container.host_dir and self.container.guest_dir else ''
655 '--privileged --detach --interactive --tty --rm '\
656 '--cgroup-parent docker {cpuset_cpus} {cpuset_mems} {publish} '\
657 '{env} {volume} --name {container.name} {container.image} '\
658 '{command}'.format(cpuset_cpus=cpuset_cpus, cpuset_mems=cpuset_mems,
659 container=self.container, command=command,
660 env=env, publish=publish, volume=volume)
662 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
664 raise RuntimeError('Failed to create container {c.name}'
665 .format(c=self.container))
669 def execute(self, command):
670 """Start a process inside a running container.
672 Runs the specified command inside the container specified by name. The
673 container has to be running already.
675 :param command: Command to run inside container.
677 :raises RuntimeError: If runnig the command in a container failed.
679 cmd = "docker exec --interactive {c.name} /bin/sh -c '{command}'"\
680 .format(c=self.container, command=command)
682 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
684 raise RuntimeError('Failed to execute command in container '
685 '{c.name}.'.format(c=self.container))
688 """Stop running container.
690 :raises RuntimeError: If stopping a container failed.
692 cmd = 'docker stop {c.name}'.format(c=self.container)
694 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
696 raise RuntimeError('Failed to stop container {c.name}.'
697 .format(c=self.container))
700 """Remove a container.
702 :raises RuntimeError: If removing a container failed.
704 cmd = 'docker rm --force {c.name}'.format(c=self.container)
706 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
708 raise RuntimeError('Failed to destroy container {c.name}.'
709 .format(c=self.container))
712 """Return low-level information on Docker objects.
714 :raises RuntimeError: If getting info about a container failed.
716 cmd = 'docker inspect {c.name}'.format(c=self.container)
718 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
720 raise RuntimeError('Failed to get info about container {c.name}.'
721 .format(c=self.container))
723 def system_info(self):
724 """Display the docker system-wide information.
726 :raises RuntimeError: If displaying system information failed.
728 cmd = 'docker system info'
730 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
732 raise RuntimeError('Failed to get system info.')
734 def is_container_present(self):
735 """Check if container is present on node.
737 :returns: True if container is present.
739 :raises RuntimeError: If getting info about a container failed.
741 cmd = 'docker ps --all --quiet --filter name={c.name}'\
742 .format(c=self.container)
744 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
746 raise RuntimeError('Failed to get info about container {c.name}.'
747 .format(c=self.container))
748 return True if stdout else False
750 def is_container_running(self):
751 """Check if container is running on node.
753 :returns: True if container is running.
755 :raises RuntimeError: If getting info about a container failed.
757 cmd = 'docker ps --quiet --filter name={c.name}'\
758 .format(c=self.container)
760 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
762 raise RuntimeError('Failed to get info about container {c.name}.'
763 .format(c=self.container))
764 return True if stdout else False
767 class Container(object):
768 """Container class."""
771 """Initialize Container object."""
774 def __getattr__(self, attr):
775 """Get attribute custom implementation.
777 :param attr: Attribute to get.
779 :returns: Attribute value or None.
783 return self.__dict__[attr]
787 def __setattr__(self, attr, value):
788 """Set attribute custom implementation.
790 :param attr: Attribute to set.
791 :param value: Value to set.
796 # Check if attribute exists
799 # Creating new attribute
801 self.__dict__['ssh'] = SSH()
802 self.__dict__['ssh'].connect(value)
803 self.__dict__[attr] = value
805 # Updating attribute base of type
806 if isinstance(self.__dict__[attr], list):
807 self.__dict__[attr].append(value)
809 self.__dict__[attr] = value