1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 # Bug workaround in pylint for abstract classes.
15 # pylint: disable=W0223
17 """Library to manipulate Containers."""
19 from collections import OrderedDict, Counter
21 from resources.libraries.python.ssh import SSH
22 from resources.libraries.python.constants import Constants
23 from resources.libraries.python.CpuUtils import CpuUtils
24 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
27 __all__ = ["ContainerManager", "ContainerEngine", "LXC", "Docker", "Container"]
29 SUPERVISOR_CONF = '/etc/supervisord.conf'
32 class ContainerManager(object):
33 """Container lifecycle management class."""
35 def __init__(self, engine):
36 """Initialize Container Manager class.
38 :param engine: Container technology used (LXC/Docker/...).
40 :raises NotImplementedError: If container technology is not implemented.
43 self.engine = globals()[engine]()
45 raise NotImplementedError('{e} is not implemented.'
47 self.containers = OrderedDict()
49 def get_container_by_name(self, name):
50 """Get container instance.
52 :param name: Container name.
54 :returns: Container instance.
56 :raises RuntimeError: If failed to get container with name.
59 return self.containers[name]
61 raise RuntimeError('Failed to get container with name: {n}'
64 def construct_container(self, **kwargs):
65 """Construct container object on node with specified parameters.
67 :param kwargs: Key-value pairs used to construct container.
71 self.engine.initialize()
74 setattr(self.engine.container, key, kwargs[key])
76 # Set additional environmental variables
77 setattr(self.engine.container, 'env',
78 'MICROSERVICE_LABEL={n}'.format(n=kwargs['name']))
80 # Set cpuset.cpus cgroup
81 skip_cnt = kwargs['cpu_skip']
82 if not kwargs['cpu_shared']:
83 skip_cnt += kwargs['i'] * kwargs['cpu_count']
84 self.engine.container.cpuset_cpus = \
85 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
86 cpu_node=kwargs['cpuset_mems'],
88 cpu_cnt=kwargs['cpu_count'],
89 smt_used=kwargs['smt_used'])
91 # Store container instance
92 self.containers[kwargs['name']] = self.engine.container
94 def construct_containers(self, **kwargs):
95 """Construct 1..N container(s) on node with specified name.
97 Ordinal number is automatically added to the name of container as
100 :param kwargs: Named parameters.
103 name = kwargs['name']
104 for i in range(kwargs['count']):
105 # Name will contain ordinal suffix
106 kwargs['name'] = ''.join([name, str(i+1)])
108 self.construct_container(i=i, **kwargs)
110 def acquire_all_containers(self):
111 """Acquire all containers."""
112 for container in self.containers:
113 self.engine.container = self.containers[container]
114 self.engine.acquire()
116 def build_all_containers(self):
117 """Build all containers."""
118 for container in self.containers:
119 self.engine.container = self.containers[container]
122 def create_all_containers(self):
123 """Create all containers."""
124 for container in self.containers:
125 self.engine.container = self.containers[container]
128 def execute_on_container(self, name, command):
129 """Execute command on container with name.
131 :param name: Container name.
132 :param command: Command to execute.
136 self.engine.container = self.get_container_by_name(name)
137 self.engine.execute(command)
139 def execute_on_all_containers(self, command):
140 """Execute command on all containers.
142 :param command: Command to execute.
145 for container in self.containers:
146 self.engine.container = self.containers[container]
147 self.engine.execute(command)
149 def install_vpp_in_all_containers(self):
150 """Install VPP into all containers."""
151 for container in self.containers:
152 self.engine.container = self.containers[container]
153 # We need to install supervisor client/server system to control VPP
155 self.engine.install_supervisor()
156 self.engine.install_vpp()
157 self.engine.restart_vpp()
159 def configure_vpp_in_all_containers(self, vat_template_file):
160 """Configure VPP in all containers.
162 :param vat_template_file: Template file name of a VAT script.
163 :type vat_template_file: str
165 # Count number of DUTs based on node's host information
166 dut_cnt = len(Counter([self.containers[container].node['host']
167 for container in self.containers]))
168 container_cnt = len(self.containers)
169 mod = dut_cnt/container_cnt
171 for i, container in enumerate(self.containers):
172 self.engine.container = self.containers[container]
173 self.engine.create_vpp_startup_config()
174 self.engine.create_vpp_exec_config(vat_template_file,
175 memif_id1=i % mod * 2 + 1,
176 memif_id2=i % mod * 2 + 2,
177 socket1='memif-{c.name}-1'
178 .format(c=self.engine.container),
179 socket2='memif-{c.name}-2'
180 .format(c=self.engine.container))
182 def stop_all_containers(self):
183 """Stop all containers."""
184 for container in self.containers:
185 self.engine.container = self.containers[container]
188 def destroy_all_containers(self):
189 """Destroy all containers."""
190 for container in self.containers:
191 self.engine.container = self.containers[container]
192 self.engine.destroy()
195 class ContainerEngine(object):
196 """Abstract class for container engine."""
199 """Init ContainerEngine object."""
200 self.container = None
202 def initialize(self):
203 """Initialize container object."""
204 self.container = Container()
206 def acquire(self, force):
207 """Acquire/download container.
209 :param force: Destroy a container if exists and create.
212 raise NotImplementedError
215 """Build container (compile)."""
216 raise NotImplementedError
219 """Create/deploy container."""
220 raise NotImplementedError
222 def execute(self, command):
223 """Execute process inside container.
225 :param command: Command to run inside container.
228 raise NotImplementedError
231 """Stop container."""
232 raise NotImplementedError
235 """Destroy/remove container."""
236 raise NotImplementedError
239 """Info about container."""
240 raise NotImplementedError
242 def system_info(self):
244 raise NotImplementedError
246 def install_supervisor(self):
247 """Install supervisord inside a container."""
248 self.execute('sleep 3')
249 self.execute('apt-get update')
250 self.execute('apt-get install -y supervisor')
251 self.execute('echo "{0}" > {1}'
253 '[unix_http_server]\n'
254 'file = /tmp/supervisor.sock\n\n'
255 '[rpcinterface:supervisor]\n'
256 'supervisor.rpcinterface_factory = '
257 'supervisor.rpcinterface:make_main_rpcinterface\n\n'
259 'serverurl = unix:///tmp/supervisor.sock\n\n'
261 'pidfile = /tmp/supervisord.pid\n'
262 'identifier = supervisor\n'
264 'logfile=/tmp/supervisord.log\n'
266 'nodaemon=false\n\n',
268 self.execute('supervisord -c {0}'.format(SUPERVISOR_CONF))
270 def install_vpp(self, install_dkms=False):
271 """Install VPP inside a container.
273 :param install_dkms: If install dkms package. This will impact install
274 time. Dkms is required for installation of vpp-dpdk-dkms. Default is
276 :type install_dkms: bool
278 self.execute('ln -s /dev/null /etc/sysctl.d/80-vpp.conf')
279 self.execute('apt-get update')
281 self.execute('apt-get install -y dkms && '
282 'dpkg -i --force-all {0}/install_dir/*.deb'
283 .format(self.container.guest_dir))
285 self.execute('for i in $(ls -I \"*dkms*\" {0}/install_dir/); '
286 'do dpkg -i --force-all {0}/install_dir/$i; done'
287 .format(self.container.guest_dir))
288 self.execute('apt-get -f install -y')
289 self.execute('echo "{0}" >> {1}'
292 'command=/usr/bin/vpp -c /etc/vpp/startup.conf\n'
293 'autorestart=false\n'
294 'redirect_stderr=true\n'
297 self.execute('supervisorctl reload')
299 def restart_vpp(self):
300 """Restart VPP service inside a container."""
301 self.execute('supervisorctl restart vpp')
303 def create_vpp_startup_config(self,
304 config_filename='/etc/vpp/startup.conf'):
305 """Create base startup configuration of VPP on container.
307 :param config_filename: Startup configuration file name.
308 :type config_filename: str
310 cpuset_cpus = self.container.cpuset_cpus
312 # Create config instance
313 vpp_config = VppConfigGenerator()
314 vpp_config.set_node(self.container.node)
315 vpp_config.add_unix_cli_listen()
316 vpp_config.add_unix_nodaemon()
317 vpp_config.add_unix_exec('/tmp/running.exec')
318 # We will pop first core from list to be main core
319 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
320 # if this is not only core in list, the rest will be used as workers.
322 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
323 vpp_config.add_cpu_corelist_workers(corelist_workers)
324 vpp_config.add_plugin_disable('dpdk_plugin.so')
326 self.execute('mkdir -p /etc/vpp/')
327 self.execute('echo "{c}" | tee {f}'
328 .format(c=vpp_config.get_config_str(),
331 def create_vpp_exec_config(self, vat_template_file, **kwargs):
332 """Create VPP exec configuration on container.
334 :param vat_template_file: File name of a VAT template script.
335 :param kwargs: Parameters for VAT script.
336 :type vat_template_file: str
339 vat_file_path = '{p}/{f}'.format(p=Constants.RESOURCES_TPL_VAT,
342 with open(vat_file_path, 'r') as template_file:
343 cmd_template = template_file.readlines()
344 for line_tmpl in cmd_template:
345 vat_cmd = line_tmpl.format(**kwargs)
346 self.execute('echo "{c}" >> /tmp/running.exec'
347 .format(c=vat_cmd.replace('\n', '')))
349 def is_container_running(self):
350 """Check if container is running."""
351 raise NotImplementedError
353 def is_container_present(self):
354 """Check if container is present."""
355 raise NotImplementedError
357 def _configure_cgroup(self, name):
358 """Configure the control group associated with a container.
360 By default the cpuset cgroup is using exclusive CPU/MEM. When Docker
361 container is initialized a new cgroup /docker or /lxc is created under
362 cpuset parent tree. This newly created cgroup is inheriting parent
363 setting for cpu/mem exclusive parameter and thus cannot be overriden
364 within /docker or /lxc cgroup. This patch is supposed to set cpu/mem
365 exclusive parameter for both parent and subgroup.
367 :param name: Name of cgroup.
369 :raises RuntimeError: If applying cgroup settings via cgset failed.
371 ret, _, _ = self.container.ssh.exec_command_sudo(
372 'cgset -r cpuset.cpu_exclusive=0 /')
374 raise RuntimeError('Failed to apply cgroup settings.')
376 ret, _, _ = self.container.ssh.exec_command_sudo(
377 'cgset -r cpuset.mem_exclusive=0 /')
379 raise RuntimeError('Failed to apply cgroup settings.')
381 ret, _, _ = self.container.ssh.exec_command_sudo(
382 'cgcreate -g cpuset:/{name}'.format(name=name))
384 raise RuntimeError('Failed to copy cgroup settings from root.')
386 ret, _, _ = self.container.ssh.exec_command_sudo(
387 'cgset -r cpuset.cpu_exclusive=0 /{name}'.format(name=name))
389 raise RuntimeError('Failed to apply cgroup settings.')
391 ret, _, _ = self.container.ssh.exec_command_sudo(
392 'cgset -r cpuset.mem_exclusive=0 /{name}'.format(name=name))
394 raise RuntimeError('Failed to apply cgroup settings.')
397 class LXC(ContainerEngine):
398 """LXC implementation."""
401 """Initialize LXC object."""
402 super(LXC, self).__init__()
404 def acquire(self, force=True):
405 """Acquire a privileged system object where configuration is stored.
407 :param force: If a container exists, destroy it and create a new
410 :raises RuntimeError: If creating the container or writing the container
413 if self.is_container_present():
419 image = self.container.image if self.container.image else\
420 "-d ubuntu -r xenial -a amd64"
422 cmd = 'lxc-create -t download --name {c.name} -- {image} '\
423 '--no-validate'.format(c=self.container, image=image)
425 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
427 raise RuntimeError('Failed to create container.')
429 if self.container.host_dir and self.container.guest_dir:
430 entry = 'lxc.mount.entry = '\
431 '{c.host_dir} /var/lib/lxc/{c.name}/rootfs{c.guest_dir} ' \
432 'none bind,create=dir 0 0'.format(c=self.container)
433 ret, _, _ = self.container.ssh.exec_command_sudo(
434 "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'"
435 .format(e=entry, c=self.container))
437 raise RuntimeError('Failed to write {c.name} config.'
438 .format(c=self.container))
439 self._configure_cgroup('lxc')
442 """Create/deploy an application inside a container on system.
444 :raises RuntimeError: If creating the container fails.
446 cpuset_cpus = '{0}'.format(
447 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
448 if self.container.cpuset_cpus else ''
450 cmd = 'lxc-start --name {c.name} --daemon'.format(c=self.container)
452 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
454 raise RuntimeError('Failed to start container {c.name}.'
455 .format(c=self.container))
456 self._lxc_wait('RUNNING')
458 # Workaround for LXC to be able to allocate all cpus including isolated.
459 cmd = 'cgset --copy-from / lxc/'
460 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
462 raise RuntimeError('Failed to copy cgroup to LXC')
464 cmd = 'lxc-cgroup --name {c.name} cpuset.cpus {cpus}'\
465 .format(c=self.container, cpus=cpuset_cpus)
466 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
468 raise RuntimeError('Failed to set cpuset.cpus to container '
469 '{c.name}.'.format(c=self.container))
471 def execute(self, command):
472 """Start a process inside a running container.
474 Runs the specified command inside the container specified by name. The
475 container has to be running already.
477 :param command: Command to run inside container.
479 :raises RuntimeError: If running the command failed.
481 env = '--keep-env {0}'.format(
482 ' '.join('--set-var %s' % env for env in self.container.env))\
483 if self.container.env else ''
485 cmd = "lxc-attach {env} --name {c.name} -- /bin/sh -c '{command}'"\
486 .format(env=env, c=self.container, command=command)
488 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
490 raise RuntimeError('Failed to run command inside container '
491 '{c.name}.'.format(c=self.container))
496 :raises RuntimeError: If stopping the container failed.
498 cmd = 'lxc-stop --name {c.name}'.format(c=self.container)
500 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
502 raise RuntimeError('Failed to stop container {c.name}.'
503 .format(c=self.container))
504 self._lxc_wait('STOPPED|FROZEN')
507 """Destroy a container.
509 :raises RuntimeError: If destroying container failed.
511 cmd = 'lxc-destroy --force --name {c.name}'.format(c=self.container)
513 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
515 raise RuntimeError('Failed to destroy container {c.name}.'
516 .format(c=self.container))
519 """Query and shows information about a container.
521 :raises RuntimeError: If getting info about a container failed.
523 cmd = 'lxc-info --name {c.name}'.format(c=self.container)
525 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
527 raise RuntimeError('Failed to get info about container {c.name}.'
528 .format(c=self.container))
530 def system_info(self):
531 """Check the current kernel for LXC support.
533 :raises RuntimeError: If checking LXC support failed.
535 cmd = 'lxc-checkconfig'
537 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
539 raise RuntimeError('Failed to check LXC support.')
541 def is_container_running(self):
542 """Check if container is running on node.
544 :returns: True if container is running.
546 :raises RuntimeError: If getting info about a container failed.
548 cmd = 'lxc-info --no-humanize --state --name {c.name}'\
549 .format(c=self.container)
551 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
553 raise RuntimeError('Failed to get info about container {c.name}.'
554 .format(c=self.container))
555 return True if 'RUNNING' in stdout else False
557 def is_container_present(self):
558 """Check if container is existing on node.
560 :returns: True if container is present.
562 :raises RuntimeError: If getting info about a container failed.
564 cmd = 'lxc-info --no-humanize --name {c.name}'.format(c=self.container)
566 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
567 return False if int(ret) else True
569 def _lxc_wait(self, state):
570 """Wait for a specific container state.
572 :param state: Specify the container state(s) to wait for.
574 :raises RuntimeError: If waiting for state of a container failed.
576 cmd = 'lxc-wait --name {c.name} --state "{s}"'\
577 .format(c=self.container, s=state)
579 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
581 raise RuntimeError('Failed to wait for state "{s}" of container '
582 '{c.name}.'.format(s=state, c=self.container))
585 class Docker(ContainerEngine):
586 """Docker implementation."""
589 """Initialize Docker object."""
590 super(Docker, self).__init__()
592 def acquire(self, force=True):
593 """Pull an image or a repository from a registry.
595 :param force: Destroy a container if exists.
597 :raises RuntimeError: If pulling a container failed.
599 if self.is_container_present():
605 cmd = 'docker pull {c.image}'.format(c=self.container)
607 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
609 raise RuntimeError('Failed to create container {c.name}.'
610 .format(c=self.container))
611 self._configure_cgroup('docker')
614 """Create/deploy container.
616 :raises RuntimeError: If creating a container failed.
618 cpuset_cpus = '--cpuset-cpus={0}'.format(
619 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
620 if self.container.cpuset_cpus else ''
622 cpuset_mems = '--cpuset-mems={0}'.format(self.container.cpuset_mems)\
623 if self.container.cpuset_mems is not None else ''
626 ' '.join('--env %s' % env for env in self.container.env))\
627 if self.container.env else ''
629 command = '{0}'.format(self.container.command)\
630 if self.container.command else ''
632 publish = '{0}'.format(
633 ' '.join('--publish %s' % var for var in self.container.publish))\
634 if self.container.publish else ''
636 volume = '--volume {c.host_dir}:{c.guest_dir}'.format(c=self.container)\
637 if self.container.host_dir and self.container.guest_dir else ''
640 '--privileged --detach --interactive --tty --rm '\
641 '--cgroup-parent docker {cpuset_cpus} {cpuset_mems} {publish} '\
642 '{env} {volume} --name {container.name} {container.image} '\
643 '{command}'.format(cpuset_cpus=cpuset_cpus, cpuset_mems=cpuset_mems,
644 container=self.container, command=command,
645 env=env, publish=publish, volume=volume)
647 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
649 raise RuntimeError('Failed to create container {c.name}'
650 .format(c=self.container))
654 def execute(self, command):
655 """Start a process inside a running container.
657 Runs the specified command inside the container specified by name. The
658 container has to be running already.
660 :param command: Command to run inside container.
662 :raises RuntimeError: If runnig the command in a container failed.
664 cmd = "docker exec --interactive {c.name} /bin/sh -c '{command}'"\
665 .format(c=self.container, command=command)
667 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
669 raise RuntimeError('Failed to execute command in container '
670 '{c.name}.'.format(c=self.container))
673 """Stop running container.
675 :raises RuntimeError: If stopping a container failed.
677 cmd = 'docker stop {c.name}'.format(c=self.container)
679 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
681 raise RuntimeError('Failed to stop container {c.name}.'
682 .format(c=self.container))
685 """Remove a container.
687 :raises RuntimeError: If removing a container failed.
689 cmd = 'docker rm --force {c.name}'.format(c=self.container)
691 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
693 raise RuntimeError('Failed to destroy container {c.name}.'
694 .format(c=self.container))
697 """Return low-level information on Docker objects.
699 :raises RuntimeError: If getting info about a container failed.
701 cmd = 'docker inspect {c.name}'.format(c=self.container)
703 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
705 raise RuntimeError('Failed to get info about container {c.name}.'
706 .format(c=self.container))
708 def system_info(self):
709 """Display the docker system-wide information.
711 :raises RuntimeError: If displaying system information failed.
713 cmd = 'docker system info'
715 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
717 raise RuntimeError('Failed to get system info.')
719 def is_container_present(self):
720 """Check if container is present on node.
722 :returns: True if container is present.
724 :raises RuntimeError: If getting info about a container failed.
726 cmd = 'docker ps --all --quiet --filter name={c.name}'\
727 .format(c=self.container)
729 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
731 raise RuntimeError('Failed to get info about container {c.name}.'
732 .format(c=self.container))
733 return True if stdout else False
735 def is_container_running(self):
736 """Check if container is running on node.
738 :returns: True if container is running.
740 :raises RuntimeError: If getting info about a container failed.
742 cmd = 'docker ps --quiet --filter name={c.name}'\
743 .format(c=self.container)
745 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
747 raise RuntimeError('Failed to get info about container {c.name}.'
748 .format(c=self.container))
749 return True if stdout else False
752 class Container(object):
753 """Container class."""
756 """Initialize Container object."""
759 def __getattr__(self, attr):
760 """Get attribute custom implementation.
762 :param attr: Attribute to get.
764 :returns: Attribute value or None.
768 return self.__dict__[attr]
772 def __setattr__(self, attr, value):
773 """Set attribute custom implementation.
775 :param attr: Attribute to set.
776 :param value: Value to set.
781 # Check if attribute exists
784 # Creating new attribute
786 self.__dict__['ssh'] = SSH()
787 self.__dict__['ssh'].connect(value)
788 self.__dict__[attr] = value
790 # Updating attribute base of type
791 if isinstance(self.__dict__[attr], list):
792 self.__dict__[attr].append(value)
794 self.__dict__[attr] = value