1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 # Bug workaround in pylint for abstract classes.
15 # pylint: disable=W0223
17 """Library to manipulate Containers."""
19 from collections import OrderedDict, Counter
21 from resources.libraries.python.ssh import SSH
22 from resources.libraries.python.constants import Constants
23 from resources.libraries.python.CpuUtils import CpuUtils
24 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
27 __all__ = ["ContainerManager", "ContainerEngine", "LXC", "Docker", "Container"]
29 SUPERVISOR_CONF = '/etc/supervisord.conf'
32 class ContainerManager(object):
33 """Container lifecycle management class."""
35 def __init__(self, engine):
36 """Initialize Container Manager class.
38 :param engine: Container technology used (LXC/Docker/...).
40 :raises NotImplementedError: If container technology is not implemented.
43 self.engine = globals()[engine]()
45 raise NotImplementedError('{engine} is not implemented.'.
46 format(engine=engine))
47 self.containers = OrderedDict()
49 def get_container_by_name(self, name):
50 """Get container instance.
52 :param name: Container name.
54 :returns: Container instance.
56 :raises RuntimeError: If failed to get container with name.
59 return self.containers[name]
61 raise RuntimeError('Failed to get container with name: {name}'.
64 def construct_container(self, **kwargs):
65 """Construct container object on node with specified parameters.
67 :param kwargs: Key-value pairs used to construct container.
71 self.engine.initialize()
74 setattr(self.engine.container, key, kwargs[key])
76 # Set additional environmental variables
77 setattr(self.engine.container, 'env',
78 'MICROSERVICE_LABEL={label}'.format(label=kwargs['name']))
80 # Set cpuset.cpus cgroup
81 skip_cnt = kwargs['cpu_skip']
82 if not kwargs['cpu_shared']:
83 skip_cnt += kwargs['i'] * kwargs['cpu_count']
84 self.engine.container.cpuset_cpus = \
85 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
86 cpu_node=kwargs['cpuset_mems'],
88 cpu_cnt=kwargs['cpu_count'],
89 smt_used=kwargs['smt_used'])
91 # Store container instance
92 self.containers[kwargs['name']] = self.engine.container
94 def construct_containers(self, **kwargs):
95 """Construct 1..N container(s) on node with specified name.
97 Ordinal number is automatically added to the name of container as
100 :param kwargs: Named parameters.
103 name = kwargs['name']
104 for i in range(kwargs['count']):
105 # Name will contain ordinal suffix
106 kwargs['name'] = ''.join([name, str(i+1)])
108 self.construct_container(i=i, **kwargs)
110 def acquire_all_containers(self):
111 """Acquire all containers."""
112 for container in self.containers:
113 self.engine.container = self.containers[container]
114 self.engine.acquire()
116 def build_all_containers(self):
117 """Build all containers."""
118 for container in self.containers:
119 self.engine.container = self.containers[container]
122 def create_all_containers(self):
123 """Create all containers."""
124 for container in self.containers:
125 self.engine.container = self.containers[container]
128 def execute_on_container(self, name, command):
129 """Execute command on container with name.
131 :param name: Container name.
132 :param command: Command to execute.
136 self.engine.container = self.get_container_by_name(name)
137 self.engine.execute(command)
139 def execute_on_all_containers(self, command):
140 """Execute command on all containers.
142 :param command: Command to execute.
145 for container in self.containers:
146 self.engine.container = self.containers[container]
147 self.engine.execute(command)
149 def install_vpp_in_all_containers(self):
150 """Install VPP into all containers."""
151 for container in self.containers:
152 self.engine.container = self.containers[container]
153 # We need to install supervisor client/server system to control VPP
155 self.engine.install_supervisor()
156 self.engine.install_vpp()
157 self.engine.restart_vpp()
159 def restart_vpp_in_all_containers(self):
160 """Restart VPP on all containers."""
161 for container in self.containers:
162 self.engine.container = self.containers[container]
163 self.engine.restart_vpp()
165 def configure_vpp_in_all_containers(self, vat_template_file):
166 """Configure VPP in all containers.
168 :param vat_template_file: Template file name of a VAT script.
169 :type vat_template_file: str
171 # Count number of DUTs based on node's host information
172 dut_cnt = len(Counter([self.containers[container].node['host']
173 for container in self.containers]))
174 container_cnt = len(self.containers)
175 mod = container_cnt/dut_cnt
177 for i, container in enumerate(self.containers):
180 sid1 = i % mod * 2 + 1
181 sid2 = i % mod * 2 + 2
182 self.engine.container = self.containers[container]
183 self.engine.create_vpp_startup_config()
184 self.engine.create_vpp_exec_config(vat_template_file, mid1=mid1,
185 mid2=mid2, sid1=sid1, sid2=sid2,
186 socket1='memif-{c.name}-{sid}'
187 .format(c=self.engine.container,
189 socket2='memif-{c.name}-{sid}'
190 .format(c=self.engine.container,
193 def stop_all_containers(self):
194 """Stop all containers."""
195 for container in self.containers:
196 self.engine.container = self.containers[container]
199 def destroy_all_containers(self):
200 """Destroy all containers."""
201 for container in self.containers:
202 self.engine.container = self.containers[container]
203 self.engine.destroy()
206 class ContainerEngine(object):
207 """Abstract class for container engine."""
210 """Init ContainerEngine object."""
211 self.container = None
213 def initialize(self):
214 """Initialize container object."""
215 self.container = Container()
217 def acquire(self, force):
218 """Acquire/download container.
220 :param force: Destroy a container if exists and create.
223 raise NotImplementedError
226 """Build container (compile)."""
227 raise NotImplementedError
230 """Create/deploy container."""
231 raise NotImplementedError
233 def execute(self, command):
234 """Execute process inside container.
236 :param command: Command to run inside container.
239 raise NotImplementedError
242 """Stop container."""
243 raise NotImplementedError
246 """Destroy/remove container."""
247 raise NotImplementedError
250 """Info about container."""
251 raise NotImplementedError
253 def system_info(self):
255 raise NotImplementedError
257 def install_supervisor(self):
258 """Install supervisord inside a container."""
259 self.execute('sleep 3')
260 self.execute('apt-get update')
261 self.execute('apt-get install -y supervisor')
262 self.execute('echo "{config}" > {config_file}'.
264 config='[unix_http_server]\n'
265 'file = /tmp/supervisor.sock\n\n'
266 '[rpcinterface:supervisor]\n'
267 'supervisor.rpcinterface_factory = '
268 'supervisor.rpcinterface:make_main_rpcinterface\n\n'
270 'serverurl = unix:///tmp/supervisor.sock\n\n'
272 'pidfile = /tmp/supervisord.pid\n'
273 'identifier = supervisor\n'
275 'logfile=/tmp/supervisord.log\n'
277 'nodaemon=false\n\n',
278 config_file=SUPERVISOR_CONF))
279 self.execute('supervisord -c {config_file}'.
280 format(config_file=SUPERVISOR_CONF))
282 def install_vpp(self):
283 """Install VPP inside a container."""
284 self.execute('ln -s /dev/null /etc/sysctl.d/80-vpp.conf')
285 self.execute('apt-get update')
286 if self.container.install_dkms:
288 'apt-get install -y dkms && '
289 'dpkg -i --force-all {guest_dir}/install_dir/*.deb'.
290 format(guest_dir=self.container.mnt[0].split(':')[1]))
293 'for i in $(ls -I \"*dkms*\" {guest_dir}/install_dir/); do '
294 'dpkg -i --force-all {guest_dir}/install_dir/$i; done'.
295 format(guest_dir=self.container.mnt[0].split(':')[1]))
296 self.execute('apt-get -f install -y')
297 self.execute('apt-get install -y ca-certificates')
298 self.execute('echo "{config}" >> {config_file}'.
300 config='[program:vpp]\n'
301 'command=/usr/bin/vpp -c /etc/vpp/startup.conf\n'
302 'autorestart=false\n'
303 'redirect_stderr=true\n'
305 config_file=SUPERVISOR_CONF))
306 self.execute('supervisorctl reload')
308 def restart_vpp(self):
309 """Restart VPP service inside a container."""
310 self.execute('supervisorctl restart vpp')
311 self.execute('cat /tmp/supervisord.log')
313 def create_vpp_startup_config(self,
314 config_filename='/etc/vpp/startup.conf'):
315 """Create base startup configuration of VPP on container.
317 :param config_filename: Startup configuration file name.
318 :type config_filename: str
320 cpuset_cpus = self.container.cpuset_cpus
322 # Create config instance
323 vpp_config = VppConfigGenerator()
324 vpp_config.set_node(self.container.node)
325 vpp_config.add_unix_cli_listen()
326 vpp_config.add_unix_nodaemon()
327 vpp_config.add_unix_exec('/tmp/running.exec')
328 # We will pop first core from list to be main core
329 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
330 # if this is not only core in list, the rest will be used as workers.
332 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
333 vpp_config.add_cpu_corelist_workers(corelist_workers)
334 vpp_config.add_plugin('disable', 'dpdk_plugin.so')
336 self.execute('mkdir -p /etc/vpp/')
337 self.execute('echo "{c}" | tee {f}'
338 .format(c=vpp_config.get_config_str(),
341 def create_vpp_exec_config(self, vat_template_file, **kwargs):
342 """Create VPP exec configuration on container.
344 :param vat_template_file: File name of a VAT template script.
345 :param kwargs: Parameters for VAT script.
346 :type vat_template_file: str
349 vat_file_path = '{p}/{f}'.format(p=Constants.RESOURCES_TPL_VAT,
352 with open(vat_file_path, 'r') as template_file:
353 cmd_template = template_file.readlines()
354 for line_tmpl in cmd_template:
355 vat_cmd = line_tmpl.format(**kwargs)
356 self.execute('echo "{c}" >> /tmp/running.exec'
357 .format(c=vat_cmd.replace('\n', '')))
359 def is_container_running(self):
360 """Check if container is running."""
361 raise NotImplementedError
363 def is_container_present(self):
364 """Check if container is present."""
365 raise NotImplementedError
367 def _configure_cgroup(self, name):
368 """Configure the control group associated with a container.
370 By default the cpuset cgroup is using exclusive CPU/MEM. When Docker/LXC
371 container is initialized a new cgroup /docker or /lxc is created under
372 cpuset parent tree. This newly created cgroup is inheriting parent
373 setting for cpu/mem exclusive parameter and thus cannot be overriden
374 within /docker or /lxc cgroup. This function is supposed to set cgroups
375 to allow coexistence of both engines.
377 :param name: Name of cgroup.
379 :raises RuntimeError: If applying cgroup settings via cgset failed.
381 ret, _, _ = self.container.ssh.exec_command_sudo(
382 'cgset -r cpuset.cpu_exclusive=0 /')
384 raise RuntimeError('Failed to apply cgroup settings.')
386 ret, _, _ = self.container.ssh.exec_command_sudo(
387 'cgset -r cpuset.mem_exclusive=0 /')
389 raise RuntimeError('Failed to apply cgroup settings.')
391 ret, _, _ = self.container.ssh.exec_command_sudo(
392 'cgcreate -g cpuset:/{name}'.format(name=name))
394 raise RuntimeError('Failed to copy cgroup settings from root.')
396 ret, _, _ = self.container.ssh.exec_command_sudo(
397 'cgset -r cpuset.cpu_exclusive=0 /{name}'.format(name=name))
399 raise RuntimeError('Failed to apply cgroup settings.')
401 ret, _, _ = self.container.ssh.exec_command_sudo(
402 'cgset -r cpuset.mem_exclusive=0 /{name}'.format(name=name))
404 raise RuntimeError('Failed to apply cgroup settings.')
407 class LXC(ContainerEngine):
408 """LXC implementation."""
411 """Initialize LXC object."""
412 super(LXC, self).__init__()
414 def acquire(self, force=True):
415 """Acquire a privileged system object where configuration is stored.
417 :param force: If a container exists, destroy it and create a new
420 :raises RuntimeError: If creating the container or writing the container
423 if self.is_container_present():
429 image = self.container.image if self.container.image else\
430 "-d ubuntu -r xenial -a amd64"
432 cmd = 'lxc-create -t download --name {c.name} -- {image} '\
433 '--no-validate'.format(c=self.container, image=image)
435 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
437 raise RuntimeError('Failed to create container.')
439 self._configure_cgroup('lxc')
442 """Create/deploy an application inside a container on system.
444 :raises RuntimeError: If creating the container fails.
446 if self.container.mnt:
447 for mount in self.container.mnt:
448 host_dir, guest_dir = mount.split(':')
449 entry = 'lxc.mount.entry = {host_dir} '\
450 '/var/lib/lxc/{c.name}/rootfs{guest_dir} none ' \
451 'bind,create=dir 0 0'.format(c=self.container,
454 ret, _, _ = self.container.ssh.exec_command_sudo(
455 "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'".
456 format(e=entry, c=self.container))
458 raise RuntimeError('Failed to write {c.name} config.'
459 .format(c=self.container))
461 cpuset_cpus = '{0}'.format(
462 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
463 if self.container.cpuset_cpus else ''
465 ret, _, _ = self.container.ssh.exec_command_sudo(
466 'lxc-start --name {c.name} --daemon'.
467 format(c=self.container))
469 raise RuntimeError('Failed to start container {c.name}.'.
470 format(c=self.container))
471 self._lxc_wait('RUNNING')
473 # Workaround for LXC to be able to allocate all cpus including isolated.
474 ret, _, _ = self.container.ssh.exec_command_sudo(
475 'cgset --copy-from / lxc/')
477 raise RuntimeError('Failed to copy cgroup to LXC')
479 ret, _, _ = self.container.ssh.exec_command_sudo(
480 'lxc-cgroup --name {c.name} cpuset.cpus {cpus}'.
481 format(c=self.container, cpus=cpuset_cpus))
483 raise RuntimeError('Failed to set cpuset.cpus to container '
484 '{c.name}.'.format(c=self.container))
486 def execute(self, command):
487 """Start a process inside a running container.
489 Runs the specified command inside the container specified by name. The
490 container has to be running already.
492 :param command: Command to run inside container.
494 :raises RuntimeError: If running the command failed.
496 env = '--keep-env {0}'.format(
497 ' '.join('--set-var %s' % env for env in self.container.env))\
498 if self.container.env else ''
500 cmd = "lxc-attach {env} --name {c.name} -- /bin/sh -c '{command}; "\
501 "exit $?'".format(env=env, c=self.container, command=command)
503 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
505 raise RuntimeError('Failed to run command inside container '
506 '{c.name}.'.format(c=self.container))
511 :raises RuntimeError: If stopping the container failed.
513 cmd = 'lxc-stop --name {c.name}'.format(c=self.container)
515 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
517 raise RuntimeError('Failed to stop container {c.name}.'
518 .format(c=self.container))
519 self._lxc_wait('STOPPED|FROZEN')
522 """Destroy a container.
524 :raises RuntimeError: If destroying container failed.
526 cmd = 'lxc-destroy --force --name {c.name}'.format(c=self.container)
528 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
530 raise RuntimeError('Failed to destroy container {c.name}.'
531 .format(c=self.container))
534 """Query and shows information about a container.
536 :raises RuntimeError: If getting info about a container failed.
538 cmd = 'lxc-info --name {c.name}'.format(c=self.container)
540 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
542 raise RuntimeError('Failed to get info about container {c.name}.'
543 .format(c=self.container))
545 def system_info(self):
546 """Check the current kernel for LXC support.
548 :raises RuntimeError: If checking LXC support failed.
550 cmd = 'lxc-checkconfig'
552 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
554 raise RuntimeError('Failed to check LXC support.')
556 def is_container_running(self):
557 """Check if container is running on node.
559 :returns: True if container is running.
561 :raises RuntimeError: If getting info about a container failed.
563 cmd = 'lxc-info --no-humanize --state --name {c.name}'\
564 .format(c=self.container)
566 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
568 raise RuntimeError('Failed to get info about container {c.name}.'
569 .format(c=self.container))
570 return True if 'RUNNING' in stdout else False
572 def is_container_present(self):
573 """Check if container is existing on node.
575 :returns: True if container is present.
577 :raises RuntimeError: If getting info about a container failed.
579 cmd = 'lxc-info --no-humanize --name {c.name}'.format(c=self.container)
581 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
582 return False if int(ret) else True
584 def _lxc_wait(self, state):
585 """Wait for a specific container state.
587 :param state: Specify the container state(s) to wait for.
589 :raises RuntimeError: If waiting for state of a container failed.
591 cmd = 'lxc-wait --name {c.name} --state "{s}"'\
592 .format(c=self.container, s=state)
594 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
596 raise RuntimeError('Failed to wait for state "{s}" of container '
597 '{c.name}.'.format(s=state, c=self.container))
600 class Docker(ContainerEngine):
601 """Docker implementation."""
604 """Initialize Docker object."""
605 super(Docker, self).__init__()
607 def acquire(self, force=True):
608 """Pull an image or a repository from a registry.
610 :param force: Destroy a container if exists.
612 :raises RuntimeError: If pulling a container failed.
614 if self.is_container_present():
620 cmd = 'docker pull {c.image}'.format(c=self.container)
622 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
624 raise RuntimeError('Failed to create container {c.name}.'
625 .format(c=self.container))
626 self._configure_cgroup('docker')
629 """Create/deploy container.
631 :raises RuntimeError: If creating a container failed.
633 cpuset_cpus = '--cpuset-cpus={0}'.format(
634 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
635 if self.container.cpuset_cpus else ''
637 cpuset_mems = '--cpuset-mems={0}'.format(self.container.cpuset_mems)\
638 if self.container.cpuset_mems is not None else ''
639 # Temporary workaround - disabling due to bug in memif
643 ' '.join('--env %s' % env for env in self.container.env))\
644 if self.container.env else ''
646 command = '{0}'.format(self.container.command)\
647 if self.container.command else ''
649 publish = '{0}'.format(
650 ' '.join('--publish %s' % var for var in self.container.publish))\
651 if self.container.publish else ''
653 volume = '{0}'.format(
654 ' '.join('--volume %s' % mnt for mnt in self.container.mnt))\
655 if self.container.mnt else ''
658 '--privileged --detach --interactive --tty --rm '\
659 '--cgroup-parent docker {cpuset_cpus} {cpuset_mems} {publish} '\
660 '{env} {volume} --name {container.name} {container.image} '\
661 '{command}'.format(cpuset_cpus=cpuset_cpus, cpuset_mems=cpuset_mems,
662 container=self.container, command=command,
663 env=env, publish=publish, volume=volume)
665 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
667 raise RuntimeError('Failed to create container {c.name}'
668 .format(c=self.container))
672 def execute(self, command):
673 """Start a process inside a running container.
675 Runs the specified command inside the container specified by name. The
676 container has to be running already.
678 :param command: Command to run inside container.
680 :raises RuntimeError: If runnig the command in a container failed.
682 cmd = "docker exec --interactive {c.name} /bin/sh -c '{command}; "\
683 "exit $?'".format(c=self.container, command=command)
685 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
687 raise RuntimeError('Failed to execute command in container '
688 '{c.name}.'.format(c=self.container))
691 """Stop running container.
693 :raises RuntimeError: If stopping a container failed.
695 cmd = 'docker stop {c.name}'.format(c=self.container)
697 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
699 raise RuntimeError('Failed to stop container {c.name}.'
700 .format(c=self.container))
703 """Remove a container.
705 :raises RuntimeError: If removing a container failed.
707 cmd = 'docker rm --force {c.name}'.format(c=self.container)
709 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
711 raise RuntimeError('Failed to destroy container {c.name}.'
712 .format(c=self.container))
715 """Return low-level information on Docker objects.
717 :raises RuntimeError: If getting info about a container failed.
719 cmd = 'docker inspect {c.name}'.format(c=self.container)
721 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
723 raise RuntimeError('Failed to get info about container {c.name}.'
724 .format(c=self.container))
726 def system_info(self):
727 """Display the docker system-wide information.
729 :raises RuntimeError: If displaying system information failed.
731 cmd = 'docker system info'
733 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
735 raise RuntimeError('Failed to get system info.')
737 def is_container_present(self):
738 """Check if container is present on node.
740 :returns: True if container is present.
742 :raises RuntimeError: If getting info about a container failed.
744 cmd = 'docker ps --all --quiet --filter name={c.name}'\
745 .format(c=self.container)
747 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
749 raise RuntimeError('Failed to get info about container {c.name}.'
750 .format(c=self.container))
751 return True if stdout else False
753 def is_container_running(self):
754 """Check if container is running on node.
756 :returns: True if container is running.
758 :raises RuntimeError: If getting info about a container failed.
760 cmd = 'docker ps --quiet --filter name={c.name}'\
761 .format(c=self.container)
763 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
765 raise RuntimeError('Failed to get info about container {c.name}.'
766 .format(c=self.container))
767 return True if stdout else False
770 class Container(object):
771 """Container class."""
774 """Initialize Container object."""
777 def __getattr__(self, attr):
778 """Get attribute custom implementation.
780 :param attr: Attribute to get.
782 :returns: Attribute value or None.
786 return self.__dict__[attr]
790 def __setattr__(self, attr, value):
791 """Set attribute custom implementation.
793 :param attr: Attribute to set.
794 :param value: Value to set.
799 # Check if attribute exists
802 # Creating new attribute
804 self.__dict__['ssh'] = SSH()
805 self.__dict__['ssh'].connect(value)
806 self.__dict__[attr] = value
808 # Updating attribute base of type
809 if isinstance(self.__dict__[attr], list):
810 self.__dict__[attr].append(value)
812 self.__dict__[attr] = value