1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 # Bug workaround in pylint for abstract classes.
15 # pylint: disable=W0223
17 """Library to manipulate Containers."""
19 from collections import OrderedDict, Counter
21 from resources.libraries.python.ssh import SSH
22 from resources.libraries.python.constants import Constants
23 from resources.libraries.python.CpuUtils import CpuUtils
24 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
27 __all__ = ["ContainerManager", "ContainerEngine", "LXC", "Docker", "Container"]
29 SUPERVISOR_CONF = '/etc/supervisord.conf'
32 class ContainerManager(object):
33 """Container lifecycle management class."""
35 def __init__(self, engine):
36 """Initialize Container Manager class.
38 :param engine: Container technology used (LXC/Docker/...).
40 :raises NotImplementedError: If container technology is not implemented.
43 self.engine = globals()[engine]()
45 raise NotImplementedError('{engine} is not implemented.'.
46 format(engine=engine))
47 self.containers = OrderedDict()
49 def get_container_by_name(self, name):
50 """Get container instance.
52 :param name: Container name.
54 :returns: Container instance.
56 :raises RuntimeError: If failed to get container with name.
59 return self.containers[name]
61 raise RuntimeError('Failed to get container with name: {name}'.
64 def construct_container(self, **kwargs):
65 """Construct container object on node with specified parameters.
67 :param kwargs: Key-value pairs used to construct container.
71 self.engine.initialize()
74 setattr(self.engine.container, key, kwargs[key])
76 # Set additional environmental variables
77 setattr(self.engine.container, 'env',
78 'MICROSERVICE_LABEL={label}'.format(label=kwargs['name']))
80 # Set cpuset.cpus cgroup
81 skip_cnt = kwargs['cpu_skip']
82 if not kwargs['cpu_shared']:
83 skip_cnt += kwargs['i'] * kwargs['cpu_count']
84 self.engine.container.cpuset_cpus = \
85 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
86 cpu_node=kwargs['cpuset_mems'],
88 cpu_cnt=kwargs['cpu_count'],
89 smt_used=kwargs['smt_used'])
91 # Store container instance
92 self.containers[kwargs['name']] = self.engine.container
94 def construct_containers(self, **kwargs):
95 """Construct 1..N container(s) on node with specified name.
97 Ordinal number is automatically added to the name of container as
100 :param kwargs: Named parameters.
103 name = kwargs['name']
104 for i in range(kwargs['count']):
105 # Name will contain ordinal suffix
106 kwargs['name'] = ''.join([name, str(i+1)])
108 self.construct_container(i=i, **kwargs)
110 def acquire_all_containers(self):
111 """Acquire all containers."""
112 for container in self.containers:
113 self.engine.container = self.containers[container]
114 self.engine.acquire()
116 def build_all_containers(self):
117 """Build all containers."""
118 for container in self.containers:
119 self.engine.container = self.containers[container]
122 def create_all_containers(self):
123 """Create all containers."""
124 for container in self.containers:
125 self.engine.container = self.containers[container]
128 def execute_on_container(self, name, command):
129 """Execute command on container with name.
131 :param name: Container name.
132 :param command: Command to execute.
136 self.engine.container = self.get_container_by_name(name)
137 self.engine.execute(command)
139 def execute_on_all_containers(self, command):
140 """Execute command on all containers.
142 :param command: Command to execute.
145 for container in self.containers:
146 self.engine.container = self.containers[container]
147 self.engine.execute(command)
149 def install_vpp_in_all_containers(self):
150 """Install VPP into all containers."""
151 for container in self.containers:
152 self.engine.container = self.containers[container]
153 # We need to install supervisor client/server system to control VPP
155 self.engine.install_supervisor()
156 self.engine.install_vpp()
157 self.engine.restart_vpp()
159 def restart_vpp_in_all_containers(self):
160 """Restart VPP on all containers."""
161 for container in self.containers:
162 self.engine.container = self.containers[container]
163 self.engine.restart_vpp()
165 def configure_vpp_in_all_containers(self, vat_template_file):
166 """Configure VPP in all containers.
168 :param vat_template_file: Template file name of a VAT script.
169 :type vat_template_file: str
171 # Count number of DUTs based on node's host information
172 dut_cnt = len(Counter([self.containers[container].node['host']
173 for container in self.containers]))
174 container_cnt = len(self.containers)
175 mod = container_cnt/dut_cnt
177 for i, container in enumerate(self.containers):
180 sid1 = i % mod * 2 + 1
181 sid2 = i % mod * 2 + 2
182 self.engine.container = self.containers[container]
183 self.engine.create_vpp_startup_config()
184 self.engine.create_vpp_exec_config(vat_template_file, mid1=mid1,
185 mid2=mid2, sid1=sid1, sid2=sid2,
186 socket1='memif-{c.name}-{sid}'
187 .format(c=self.engine.container,
189 socket2='memif-{c.name}-{sid}'
190 .format(c=self.engine.container,
193 def stop_all_containers(self):
194 """Stop all containers."""
195 for container in self.containers:
196 self.engine.container = self.containers[container]
199 def destroy_all_containers(self):
200 """Destroy all containers."""
201 for container in self.containers:
202 self.engine.container = self.containers[container]
203 self.engine.destroy()
206 class ContainerEngine(object):
207 """Abstract class for container engine."""
210 """Init ContainerEngine object."""
211 self.container = None
213 def initialize(self):
214 """Initialize container object."""
215 self.container = Container()
217 def acquire(self, force):
218 """Acquire/download container.
220 :param force: Destroy a container if exists and create.
223 raise NotImplementedError
226 """Build container (compile)."""
227 raise NotImplementedError
230 """Create/deploy container."""
231 raise NotImplementedError
233 def execute(self, command):
234 """Execute process inside container.
236 :param command: Command to run inside container.
239 raise NotImplementedError
242 """Stop container."""
243 raise NotImplementedError
246 """Destroy/remove container."""
247 raise NotImplementedError
250 """Info about container."""
251 raise NotImplementedError
253 def system_info(self):
255 raise NotImplementedError
257 def install_supervisor(self):
258 """Install supervisord inside a container."""
259 self.execute('sleep 3')
260 self.execute('apt-get update')
261 self.execute('apt-get install -y supervisor')
262 self.execute('echo "{config}" > {config_file}'.
264 config='[unix_http_server]\n'
265 'file = /tmp/supervisor.sock\n\n'
266 '[rpcinterface:supervisor]\n'
267 'supervisor.rpcinterface_factory = '
268 'supervisor.rpcinterface:make_main_rpcinterface\n\n'
270 'serverurl = unix:///tmp/supervisor.sock\n\n'
272 'pidfile = /tmp/supervisord.pid\n'
273 'identifier = supervisor\n'
275 'logfile=/tmp/supervisord.log\n'
277 'nodaemon=false\n\n',
278 config_file=SUPERVISOR_CONF))
279 self.execute('supervisord -c {config_file}'.
280 format(config_file=SUPERVISOR_CONF))
282 def install_vpp(self, install_dkms=False):
283 """Install VPP inside a container.
285 :param install_dkms: If install dkms package. This will impact
286 install time. Dkms is required for installation of vpp-dpdk-dkms.
288 :type install_dkms: bool
290 self.execute('ln -s /dev/null /etc/sysctl.d/80-vpp.conf')
291 self.execute('apt-get update')
294 'apt-get install -y dkms && '
295 'dpkg -i --force-all {guest_dir}/install_dir/*.deb'.
296 format(guest_dir=self.container.mnt[0].split(':')[1]))
299 'for i in $(ls -I \"*dkms*\" {guest_dir}/install_dir/); do '
300 'dpkg -i --force-all {guest_dir}/install_dir/$i; done'.
301 format(guest_dir=self.container.mnt[0].split(':')[1]))
302 self.execute('apt-get -f install -y')
303 self.execute('apt-get install -y ca-certificates')
304 self.execute('echo "{config}" >> {config_file}'.
306 config='[program:vpp]\n'
307 'command=/usr/bin/vpp -c /etc/vpp/startup.conf\n'
308 'autorestart=false\n'
309 'redirect_stderr=true\n'
311 config_file=SUPERVISOR_CONF))
312 self.execute('supervisorctl reload')
314 def restart_vpp(self):
315 """Restart VPP service inside a container."""
316 self.execute('supervisorctl restart vpp')
317 self.execute('cat /tmp/supervisord.log')
319 def create_vpp_startup_config(self,
320 config_filename='/etc/vpp/startup.conf'):
321 """Create base startup configuration of VPP on container.
323 :param config_filename: Startup configuration file name.
324 :type config_filename: str
326 cpuset_cpus = self.container.cpuset_cpus
328 # Create config instance
329 vpp_config = VppConfigGenerator()
330 vpp_config.set_node(self.container.node)
331 vpp_config.add_unix_cli_listen()
332 vpp_config.add_unix_nodaemon()
333 vpp_config.add_unix_exec('/tmp/running.exec')
334 # We will pop first core from list to be main core
335 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
336 # if this is not only core in list, the rest will be used as workers.
338 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
339 vpp_config.add_cpu_corelist_workers(corelist_workers)
340 vpp_config.add_plugin('disable', 'dpdk_plugin.so')
342 self.execute('mkdir -p /etc/vpp/')
343 self.execute('echo "{c}" | tee {f}'
344 .format(c=vpp_config.get_config_str(),
347 def create_vpp_exec_config(self, vat_template_file, **kwargs):
348 """Create VPP exec configuration on container.
350 :param vat_template_file: File name of a VAT template script.
351 :param kwargs: Parameters for VAT script.
352 :type vat_template_file: str
355 vat_file_path = '{p}/{f}'.format(p=Constants.RESOURCES_TPL_VAT,
358 with open(vat_file_path, 'r') as template_file:
359 cmd_template = template_file.readlines()
360 for line_tmpl in cmd_template:
361 vat_cmd = line_tmpl.format(**kwargs)
362 self.execute('echo "{c}" >> /tmp/running.exec'
363 .format(c=vat_cmd.replace('\n', '')))
365 def is_container_running(self):
366 """Check if container is running."""
367 raise NotImplementedError
369 def is_container_present(self):
370 """Check if container is present."""
371 raise NotImplementedError
373 def _configure_cgroup(self, name):
374 """Configure the control group associated with a container.
376 By default the cpuset cgroup is using exclusive CPU/MEM. When Docker/LXC
377 container is initialized a new cgroup /docker or /lxc is created under
378 cpuset parent tree. This newly created cgroup is inheriting parent
379 setting for cpu/mem exclusive parameter and thus cannot be overriden
380 within /docker or /lxc cgroup. This function is supposed to set cgroups
381 to allow coexistence of both engines.
383 :param name: Name of cgroup.
385 :raises RuntimeError: If applying cgroup settings via cgset failed.
387 ret, _, _ = self.container.ssh.exec_command_sudo(
388 'cgset -r cpuset.cpu_exclusive=0 /')
390 raise RuntimeError('Failed to apply cgroup settings.')
392 ret, _, _ = self.container.ssh.exec_command_sudo(
393 'cgset -r cpuset.mem_exclusive=0 /')
395 raise RuntimeError('Failed to apply cgroup settings.')
397 ret, _, _ = self.container.ssh.exec_command_sudo(
398 'cgcreate -g cpuset:/{name}'.format(name=name))
400 raise RuntimeError('Failed to copy cgroup settings from root.')
402 ret, _, _ = self.container.ssh.exec_command_sudo(
403 'cgset -r cpuset.cpu_exclusive=0 /{name}'.format(name=name))
405 raise RuntimeError('Failed to apply cgroup settings.')
407 ret, _, _ = self.container.ssh.exec_command_sudo(
408 'cgset -r cpuset.mem_exclusive=0 /{name}'.format(name=name))
410 raise RuntimeError('Failed to apply cgroup settings.')
413 class LXC(ContainerEngine):
414 """LXC implementation."""
417 """Initialize LXC object."""
418 super(LXC, self).__init__()
420 def acquire(self, force=True):
421 """Acquire a privileged system object where configuration is stored.
423 :param force: If a container exists, destroy it and create a new
426 :raises RuntimeError: If creating the container or writing the container
429 if self.is_container_present():
435 image = self.container.image if self.container.image else\
436 "-d ubuntu -r xenial -a amd64"
438 cmd = 'lxc-create -t download --name {c.name} -- {image} '\
439 '--no-validate'.format(c=self.container, image=image)
441 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
443 raise RuntimeError('Failed to create container.')
445 self._configure_cgroup('lxc')
448 """Create/deploy an application inside a container on system.
450 :raises RuntimeError: If creating the container fails.
452 if self.container.mnt:
453 for mount in self.container.mnt:
454 host_dir, guest_dir = mount.split(':')
455 entry = 'lxc.mount.entry = {host_dir} '\
456 '/var/lib/lxc/{c.name}/rootfs{guest_dir} none ' \
457 'bind,create=dir 0 0'.format(c=self.container,
460 ret, _, _ = self.container.ssh.exec_command_sudo(
461 "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'".
462 format(e=entry, c=self.container))
464 raise RuntimeError('Failed to write {c.name} config.'
465 .format(c=self.container))
467 cpuset_cpus = '{0}'.format(
468 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
469 if self.container.cpuset_cpus else ''
471 ret, _, _ = self.container.ssh.exec_command_sudo(
472 'lxc-start --name {c.name} --daemon'.
473 format(c=self.container))
475 raise RuntimeError('Failed to start container {c.name}.'.
476 format(c=self.container))
477 self._lxc_wait('RUNNING')
479 # Workaround for LXC to be able to allocate all cpus including isolated.
480 ret, _, _ = self.container.ssh.exec_command_sudo(
481 'cgset --copy-from / lxc/')
483 raise RuntimeError('Failed to copy cgroup to LXC')
485 ret, _, _ = self.container.ssh.exec_command_sudo(
486 'lxc-cgroup --name {c.name} cpuset.cpus {cpus}'.
487 format(c=self.container, cpus=cpuset_cpus))
489 raise RuntimeError('Failed to set cpuset.cpus to container '
490 '{c.name}.'.format(c=self.container))
492 def execute(self, command):
493 """Start a process inside a running container.
495 Runs the specified command inside the container specified by name. The
496 container has to be running already.
498 :param command: Command to run inside container.
500 :raises RuntimeError: If running the command failed.
502 env = '--keep-env {0}'.format(
503 ' '.join('--set-var %s' % env for env in self.container.env))\
504 if self.container.env else ''
506 cmd = "lxc-attach {env} --name {c.name} -- /bin/sh -c '{command}; "\
507 "exit $?'".format(env=env, c=self.container, command=command)
509 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
511 raise RuntimeError('Failed to run command inside container '
512 '{c.name}.'.format(c=self.container))
517 :raises RuntimeError: If stopping the container failed.
519 cmd = 'lxc-stop --name {c.name}'.format(c=self.container)
521 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
523 raise RuntimeError('Failed to stop container {c.name}.'
524 .format(c=self.container))
525 self._lxc_wait('STOPPED|FROZEN')
528 """Destroy a container.
530 :raises RuntimeError: If destroying container failed.
532 cmd = 'lxc-destroy --force --name {c.name}'.format(c=self.container)
534 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
536 raise RuntimeError('Failed to destroy container {c.name}.'
537 .format(c=self.container))
540 """Query and shows information about a container.
542 :raises RuntimeError: If getting info about a container failed.
544 cmd = 'lxc-info --name {c.name}'.format(c=self.container)
546 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
548 raise RuntimeError('Failed to get info about container {c.name}.'
549 .format(c=self.container))
551 def system_info(self):
552 """Check the current kernel for LXC support.
554 :raises RuntimeError: If checking LXC support failed.
556 cmd = 'lxc-checkconfig'
558 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
560 raise RuntimeError('Failed to check LXC support.')
562 def is_container_running(self):
563 """Check if container is running on node.
565 :returns: True if container is running.
567 :raises RuntimeError: If getting info about a container failed.
569 cmd = 'lxc-info --no-humanize --state --name {c.name}'\
570 .format(c=self.container)
572 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
574 raise RuntimeError('Failed to get info about container {c.name}.'
575 .format(c=self.container))
576 return True if 'RUNNING' in stdout else False
578 def is_container_present(self):
579 """Check if container is existing on node.
581 :returns: True if container is present.
583 :raises RuntimeError: If getting info about a container failed.
585 cmd = 'lxc-info --no-humanize --name {c.name}'.format(c=self.container)
587 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
588 return False if int(ret) else True
590 def _lxc_wait(self, state):
591 """Wait for a specific container state.
593 :param state: Specify the container state(s) to wait for.
595 :raises RuntimeError: If waiting for state of a container failed.
597 cmd = 'lxc-wait --name {c.name} --state "{s}"'\
598 .format(c=self.container, s=state)
600 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
602 raise RuntimeError('Failed to wait for state "{s}" of container '
603 '{c.name}.'.format(s=state, c=self.container))
606 class Docker(ContainerEngine):
607 """Docker implementation."""
610 """Initialize Docker object."""
611 super(Docker, self).__init__()
613 def acquire(self, force=True):
614 """Pull an image or a repository from a registry.
616 :param force: Destroy a container if exists.
618 :raises RuntimeError: If pulling a container failed.
620 if self.is_container_present():
626 cmd = 'docker pull {c.image}'.format(c=self.container)
628 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
630 raise RuntimeError('Failed to create container {c.name}.'
631 .format(c=self.container))
632 self._configure_cgroup('docker')
635 """Create/deploy container.
637 :raises RuntimeError: If creating a container failed.
639 cpuset_cpus = '--cpuset-cpus={0}'.format(
640 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
641 if self.container.cpuset_cpus else ''
643 cpuset_mems = '--cpuset-mems={0}'.format(self.container.cpuset_mems)\
644 if self.container.cpuset_mems is not None else ''
645 # Temporary workaround - disabling due to bug in memif
649 ' '.join('--env %s' % env for env in self.container.env))\
650 if self.container.env else ''
652 command = '{0}'.format(self.container.command)\
653 if self.container.command else ''
655 publish = '{0}'.format(
656 ' '.join('--publish %s' % var for var in self.container.publish))\
657 if self.container.publish else ''
659 volume = '{0}'.format(
660 ' '.join('--volume %s' % mnt for mnt in self.container.mnt))\
661 if self.container.mnt else ''
664 '--privileged --detach --interactive --tty --rm '\
665 '--cgroup-parent docker {cpuset_cpus} {cpuset_mems} {publish} '\
666 '{env} {volume} --name {container.name} {container.image} '\
667 '{command}'.format(cpuset_cpus=cpuset_cpus, cpuset_mems=cpuset_mems,
668 container=self.container, command=command,
669 env=env, publish=publish, volume=volume)
671 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
673 raise RuntimeError('Failed to create container {c.name}'
674 .format(c=self.container))
678 def execute(self, command):
679 """Start a process inside a running container.
681 Runs the specified command inside the container specified by name. The
682 container has to be running already.
684 :param command: Command to run inside container.
686 :raises RuntimeError: If runnig the command in a container failed.
688 cmd = "docker exec --interactive {c.name} /bin/sh -c '{command}; "\
689 "exit $?'".format(c=self.container, command=command)
691 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
693 raise RuntimeError('Failed to execute command in container '
694 '{c.name}.'.format(c=self.container))
697 """Stop running container.
699 :raises RuntimeError: If stopping a container failed.
701 cmd = 'docker stop {c.name}'.format(c=self.container)
703 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
705 raise RuntimeError('Failed to stop container {c.name}.'
706 .format(c=self.container))
709 """Remove a container.
711 :raises RuntimeError: If removing a container failed.
713 cmd = 'docker rm --force {c.name}'.format(c=self.container)
715 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
717 raise RuntimeError('Failed to destroy container {c.name}.'
718 .format(c=self.container))
721 """Return low-level information on Docker objects.
723 :raises RuntimeError: If getting info about a container failed.
725 cmd = 'docker inspect {c.name}'.format(c=self.container)
727 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
729 raise RuntimeError('Failed to get info about container {c.name}.'
730 .format(c=self.container))
732 def system_info(self):
733 """Display the docker system-wide information.
735 :raises RuntimeError: If displaying system information failed.
737 cmd = 'docker system info'
739 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
741 raise RuntimeError('Failed to get system info.')
743 def is_container_present(self):
744 """Check if container is present on node.
746 :returns: True if container is present.
748 :raises RuntimeError: If getting info about a container failed.
750 cmd = 'docker ps --all --quiet --filter name={c.name}'\
751 .format(c=self.container)
753 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
755 raise RuntimeError('Failed to get info about container {c.name}.'
756 .format(c=self.container))
757 return True if stdout else False
759 def is_container_running(self):
760 """Check if container is running on node.
762 :returns: True if container is running.
764 :raises RuntimeError: If getting info about a container failed.
766 cmd = 'docker ps --quiet --filter name={c.name}'\
767 .format(c=self.container)
769 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
771 raise RuntimeError('Failed to get info about container {c.name}.'
772 .format(c=self.container))
773 return True if stdout else False
776 class Container(object):
777 """Container class."""
780 """Initialize Container object."""
783 def __getattr__(self, attr):
784 """Get attribute custom implementation.
786 :param attr: Attribute to get.
788 :returns: Attribute value or None.
792 return self.__dict__[attr]
796 def __setattr__(self, attr, value):
797 """Set attribute custom implementation.
799 :param attr: Attribute to set.
800 :param value: Value to set.
805 # Check if attribute exists
808 # Creating new attribute
810 self.__dict__['ssh'] = SSH()
811 self.__dict__['ssh'].connect(value)
812 self.__dict__[attr] = value
814 # Updating attribute base of type
815 if isinstance(self.__dict__[attr], list):
816 self.__dict__[attr].append(value)
818 self.__dict__[attr] = value