1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 # Bug workaround in pylint for abstract classes.
15 # pylint: disable=W0223
17 """Library to manipulate Containers."""
19 from collections import OrderedDict, Counter
21 from resources.libraries.python.ssh import SSH
22 from resources.libraries.python.constants import Constants
23 from resources.libraries.python.CpuUtils import CpuUtils
24 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
27 __all__ = ["ContainerManager", "ContainerEngine", "LXC", "Docker", "Container"]
29 SUPERVISOR_CONF = '/etc/supervisord.conf'
32 class ContainerManager(object):
33 """Container lifecycle management class."""
35 def __init__(self, engine):
36 """Initialize Container Manager class.
38 :param engine: Container technology used (LXC/Docker/...).
40 :raises NotImplementedError: If container technology is not implemented.
43 self.engine = globals()[engine]()
45 raise NotImplementedError('{e} is not implemented.'
47 self.containers = OrderedDict()
49 def get_container_by_name(self, name):
50 """Get container instance.
52 :param name: Container name.
54 :returns: Container instance.
56 :raises RuntimeError: If failed to get container with name.
59 return self.containers[name]
61 raise RuntimeError('Failed to get container with name: {n}'
64 def construct_container(self, **kwargs):
65 """Construct container object on node with specified parameters.
67 :param kwargs: Key-value pairs used to construct container.
71 self.engine.initialize()
74 setattr(self.engine.container, key, kwargs[key])
76 # Set additional environmental variables
77 setattr(self.engine.container, 'env',
78 'MICROSERVICE_LABEL={n}'.format(n=kwargs['name']))
80 # Set cpuset.cpus cgroup
81 skip_cnt = kwargs['cpu_skip']
82 if not kwargs['cpu_shared']:
83 skip_cnt += kwargs['i'] * kwargs['cpu_count']
84 self.engine.container.cpuset_cpus = \
85 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
86 cpu_node=kwargs['cpuset_mems'],
88 cpu_cnt=kwargs['cpu_count'],
89 smt_used=kwargs['smt_used'])
91 # Store container instance
92 self.containers[kwargs['name']] = self.engine.container
94 def construct_containers(self, **kwargs):
95 """Construct 1..N container(s) on node with specified name.
97 Ordinal number is automatically added to the name of container as
100 :param kwargs: Named parameters.
103 name = kwargs['name']
104 for i in range(kwargs['count']):
105 # Name will contain ordinal suffix
106 kwargs['name'] = ''.join([name, str(i+1)])
108 self.construct_container(i=i, **kwargs)
110 def acquire_all_containers(self):
111 """Acquire all containers."""
112 for container in self.containers:
113 self.engine.container = self.containers[container]
114 self.engine.acquire()
116 def build_all_containers(self):
117 """Build all containers."""
118 for container in self.containers:
119 self.engine.container = self.containers[container]
122 def create_all_containers(self):
123 """Create all containers."""
124 for container in self.containers:
125 self.engine.container = self.containers[container]
128 def execute_on_container(self, name, command):
129 """Execute command on container with name.
131 :param name: Container name.
132 :param command: Command to execute.
136 self.engine.container = self.get_container_by_name(name)
137 self.engine.execute(command)
139 def execute_on_all_containers(self, command):
140 """Execute command on all containers.
142 :param command: Command to execute.
145 for container in self.containers:
146 self.engine.container = self.containers[container]
147 self.engine.execute(command)
149 def install_vpp_in_all_containers(self):
150 """Install VPP into all containers."""
151 for container in self.containers:
152 self.engine.container = self.containers[container]
153 # We need to install supervisor client/server system to control VPP
155 self.engine.install_supervisor()
156 self.engine.install_vpp()
157 self.engine.restart_vpp()
159 def configure_vpp_in_all_containers(self, vat_template_file):
160 """Configure VPP in all containers.
162 :param vat_template_file: Template file name of a VAT script.
163 :type vat_template_file: str
165 # Count number of DUTs based on node's host information
166 dut_cnt = len(Counter([self.containers[container].node['host']
167 for container in self.containers]))
168 container_cnt = len(self.containers)
169 mod = container_cnt/dut_cnt
171 for i, container in enumerate(self.containers):
174 sid1 = i % mod * 2 + 1
175 sid2 = i % mod * 2 + 2
176 self.engine.container = self.containers[container]
177 self.engine.create_vpp_startup_config()
178 self.engine.create_vpp_exec_config(vat_template_file, mid1=mid1,
179 mid2=mid2, sid1=sid1, sid2=sid2,
180 socket1='memif-{c.name}-{sid}'
181 .format(c=self.engine.container,
183 socket2='memif-{c.name}-{sid}'
184 .format(c=self.engine.container,
187 def stop_all_containers(self):
188 """Stop all containers."""
189 for container in self.containers:
190 self.engine.container = self.containers[container]
193 def destroy_all_containers(self):
194 """Destroy all containers."""
195 for container in self.containers:
196 self.engine.container = self.containers[container]
197 self.engine.destroy()
200 class ContainerEngine(object):
201 """Abstract class for container engine."""
204 """Init ContainerEngine object."""
205 self.container = None
207 def initialize(self):
208 """Initialize container object."""
209 self.container = Container()
211 def acquire(self, force):
212 """Acquire/download container.
214 :param force: Destroy a container if exists and create.
217 raise NotImplementedError
220 """Build container (compile)."""
221 raise NotImplementedError
224 """Create/deploy container."""
225 raise NotImplementedError
227 def execute(self, command):
228 """Execute process inside container.
230 :param command: Command to run inside container.
233 raise NotImplementedError
236 """Stop container."""
237 raise NotImplementedError
240 """Destroy/remove container."""
241 raise NotImplementedError
244 """Info about container."""
245 raise NotImplementedError
247 def system_info(self):
249 raise NotImplementedError
251 def install_supervisor(self):
252 """Install supervisord inside a container."""
253 self.execute('sleep 3')
254 self.execute('apt-get update')
255 self.execute('apt-get install -y supervisor')
256 self.execute('echo "{0}" > {1}'
258 '[unix_http_server]\n'
259 'file = /tmp/supervisor.sock\n\n'
260 '[rpcinterface:supervisor]\n'
261 'supervisor.rpcinterface_factory = '
262 'supervisor.rpcinterface:make_main_rpcinterface\n\n'
264 'serverurl = unix:///tmp/supervisor.sock\n\n'
266 'pidfile = /tmp/supervisord.pid\n'
267 'identifier = supervisor\n'
269 'logfile=/tmp/supervisord.log\n'
271 'nodaemon=false\n\n',
273 self.execute('supervisord -c {0}'.format(SUPERVISOR_CONF))
275 def install_vpp(self, install_dkms=False):
276 """Install VPP inside a container.
278 :param install_dkms: If install dkms package. This will impact install
279 time. Dkms is required for installation of vpp-dpdk-dkms. Default is
281 :type install_dkms: bool
283 self.execute('ln -s /dev/null /etc/sysctl.d/80-vpp.conf')
284 self.execute('apt-get update')
286 self.execute('apt-get install -y dkms && '
287 'dpkg -i --force-all {0}/install_dir/*.deb'
288 .format(self.container.guest_dir))
290 self.execute('for i in $(ls -I \"*dkms*\" {0}/install_dir/); '
291 'do dpkg -i --force-all {0}/install_dir/$i; done'
292 .format(self.container.guest_dir))
293 self.execute('apt-get -f install -y')
294 self.execute('apt-get install -y ca-certificates')
295 self.execute('echo "{0}" >> {1}'
298 'command=/usr/bin/vpp -c /etc/vpp/startup.conf\n'
299 'autorestart=false\n'
300 'redirect_stderr=true\n'
303 self.execute('supervisorctl reload')
305 def restart_vpp(self):
306 """Restart VPP service inside a container."""
307 self.execute('supervisorctl restart vpp')
308 self.execute('cat /tmp/supervisord.log')
310 def create_vpp_startup_config(self,
311 config_filename='/etc/vpp/startup.conf'):
312 """Create base startup configuration of VPP on container.
314 :param config_filename: Startup configuration file name.
315 :type config_filename: str
317 cpuset_cpus = self.container.cpuset_cpus
319 # Create config instance
320 vpp_config = VppConfigGenerator()
321 vpp_config.set_node(self.container.node)
322 vpp_config.add_unix_cli_listen()
323 vpp_config.add_unix_nodaemon()
324 vpp_config.add_unix_exec('/tmp/running.exec')
325 # We will pop first core from list to be main core
326 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
327 # if this is not only core in list, the rest will be used as workers.
329 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
330 vpp_config.add_cpu_corelist_workers(corelist_workers)
331 vpp_config.add_plugin('disable', 'dpdk_plugin.so')
333 self.execute('mkdir -p /etc/vpp/')
334 self.execute('echo "{c}" | tee {f}'
335 .format(c=vpp_config.get_config_str(),
338 def create_vpp_exec_config(self, vat_template_file, **kwargs):
339 """Create VPP exec configuration on container.
341 :param vat_template_file: File name of a VAT template script.
342 :param kwargs: Parameters for VAT script.
343 :type vat_template_file: str
346 vat_file_path = '{p}/{f}'.format(p=Constants.RESOURCES_TPL_VAT,
349 with open(vat_file_path, 'r') as template_file:
350 cmd_template = template_file.readlines()
351 for line_tmpl in cmd_template:
352 vat_cmd = line_tmpl.format(**kwargs)
353 self.execute('echo "{c}" >> /tmp/running.exec'
354 .format(c=vat_cmd.replace('\n', '')))
356 def is_container_running(self):
357 """Check if container is running."""
358 raise NotImplementedError
360 def is_container_present(self):
361 """Check if container is present."""
362 raise NotImplementedError
364 def _configure_cgroup(self, name):
365 """Configure the control group associated with a container.
367 By default the cpuset cgroup is using exclusive CPU/MEM. When Docker
368 container is initialized a new cgroup /docker or /lxc is created under
369 cpuset parent tree. This newly created cgroup is inheriting parent
370 setting for cpu/mem exclusive parameter and thus cannot be overriden
371 within /docker or /lxc cgroup. This patch is supposed to set cpu/mem
372 exclusive parameter for both parent and subgroup.
374 :param name: Name of cgroup.
376 :raises RuntimeError: If applying cgroup settings via cgset failed.
378 ret, _, _ = self.container.ssh.exec_command_sudo(
379 'cgset -r cpuset.cpu_exclusive=0 /')
381 raise RuntimeError('Failed to apply cgroup settings.')
383 ret, _, _ = self.container.ssh.exec_command_sudo(
384 'cgset -r cpuset.mem_exclusive=0 /')
386 raise RuntimeError('Failed to apply cgroup settings.')
388 ret, _, _ = self.container.ssh.exec_command_sudo(
389 'cgcreate -g cpuset:/{name}'.format(name=name))
391 raise RuntimeError('Failed to copy cgroup settings from root.')
393 ret, _, _ = self.container.ssh.exec_command_sudo(
394 'cgset -r cpuset.cpu_exclusive=0 /{name}'.format(name=name))
396 raise RuntimeError('Failed to apply cgroup settings.')
398 ret, _, _ = self.container.ssh.exec_command_sudo(
399 'cgset -r cpuset.mem_exclusive=0 /{name}'.format(name=name))
401 raise RuntimeError('Failed to apply cgroup settings.')
404 class LXC(ContainerEngine):
405 """LXC implementation."""
408 """Initialize LXC object."""
409 super(LXC, self).__init__()
411 def acquire(self, force=True):
412 """Acquire a privileged system object where configuration is stored.
414 :param force: If a container exists, destroy it and create a new
417 :raises RuntimeError: If creating the container or writing the container
420 if self.is_container_present():
426 image = self.container.image if self.container.image else\
427 "-d ubuntu -r xenial -a amd64"
429 cmd = 'lxc-create -t download --name {c.name} -- {image} '\
430 '--no-validate'.format(c=self.container, image=image)
432 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
434 raise RuntimeError('Failed to create container.')
436 if self.container.host_dir and self.container.guest_dir:
437 entry = 'lxc.mount.entry = '\
438 '{c.host_dir} /var/lib/lxc/{c.name}/rootfs{c.guest_dir} ' \
439 'none bind,create=dir 0 0'.format(c=self.container)
440 ret, _, _ = self.container.ssh.exec_command_sudo(
441 "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'"
442 .format(e=entry, c=self.container))
444 raise RuntimeError('Failed to write {c.name} config.'
445 .format(c=self.container))
446 self._configure_cgroup('lxc')
449 """Create/deploy an application inside a container on system.
451 :raises RuntimeError: If creating the container fails.
453 cpuset_cpus = '{0}'.format(
454 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
455 if self.container.cpuset_cpus else ''
457 cmd = 'lxc-start --name {c.name} --daemon'.format(c=self.container)
459 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
461 raise RuntimeError('Failed to start container {c.name}.'
462 .format(c=self.container))
463 self._lxc_wait('RUNNING')
465 # Workaround for LXC to be able to allocate all cpus including isolated.
466 cmd = 'cgset --copy-from / lxc/'
467 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
469 raise RuntimeError('Failed to copy cgroup to LXC')
471 cmd = 'lxc-cgroup --name {c.name} cpuset.cpus {cpus}'\
472 .format(c=self.container, cpus=cpuset_cpus)
473 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
475 raise RuntimeError('Failed to set cpuset.cpus to container '
476 '{c.name}.'.format(c=self.container))
478 def execute(self, command):
479 """Start a process inside a running container.
481 Runs the specified command inside the container specified by name. The
482 container has to be running already.
484 :param command: Command to run inside container.
486 :raises RuntimeError: If running the command failed.
488 env = '--keep-env {0}'.format(
489 ' '.join('--set-var %s' % env for env in self.container.env))\
490 if self.container.env else ''
492 cmd = "lxc-attach {env} --name {c.name} -- /bin/sh -c '{command}'"\
493 .format(env=env, c=self.container, command=command)
495 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
497 raise RuntimeError('Failed to run command inside container '
498 '{c.name}.'.format(c=self.container))
503 :raises RuntimeError: If stopping the container failed.
505 cmd = 'lxc-stop --name {c.name}'.format(c=self.container)
507 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
509 raise RuntimeError('Failed to stop container {c.name}.'
510 .format(c=self.container))
511 self._lxc_wait('STOPPED|FROZEN')
514 """Destroy a container.
516 :raises RuntimeError: If destroying container failed.
518 cmd = 'lxc-destroy --force --name {c.name}'.format(c=self.container)
520 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
522 raise RuntimeError('Failed to destroy container {c.name}.'
523 .format(c=self.container))
526 """Query and shows information about a container.
528 :raises RuntimeError: If getting info about a container failed.
530 cmd = 'lxc-info --name {c.name}'.format(c=self.container)
532 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
534 raise RuntimeError('Failed to get info about container {c.name}.'
535 .format(c=self.container))
537 def system_info(self):
538 """Check the current kernel for LXC support.
540 :raises RuntimeError: If checking LXC support failed.
542 cmd = 'lxc-checkconfig'
544 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
546 raise RuntimeError('Failed to check LXC support.')
548 def is_container_running(self):
549 """Check if container is running on node.
551 :returns: True if container is running.
553 :raises RuntimeError: If getting info about a container failed.
555 cmd = 'lxc-info --no-humanize --state --name {c.name}'\
556 .format(c=self.container)
558 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
560 raise RuntimeError('Failed to get info about container {c.name}.'
561 .format(c=self.container))
562 return True if 'RUNNING' in stdout else False
564 def is_container_present(self):
565 """Check if container is existing on node.
567 :returns: True if container is present.
569 :raises RuntimeError: If getting info about a container failed.
571 cmd = 'lxc-info --no-humanize --name {c.name}'.format(c=self.container)
573 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
574 return False if int(ret) else True
576 def _lxc_wait(self, state):
577 """Wait for a specific container state.
579 :param state: Specify the container state(s) to wait for.
581 :raises RuntimeError: If waiting for state of a container failed.
583 cmd = 'lxc-wait --name {c.name} --state "{s}"'\
584 .format(c=self.container, s=state)
586 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
588 raise RuntimeError('Failed to wait for state "{s}" of container '
589 '{c.name}.'.format(s=state, c=self.container))
592 class Docker(ContainerEngine):
593 """Docker implementation."""
596 """Initialize Docker object."""
597 super(Docker, self).__init__()
599 def acquire(self, force=True):
600 """Pull an image or a repository from a registry.
602 :param force: Destroy a container if exists.
604 :raises RuntimeError: If pulling a container failed.
606 if self.is_container_present():
612 cmd = 'docker pull {c.image}'.format(c=self.container)
614 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
616 raise RuntimeError('Failed to create container {c.name}.'
617 .format(c=self.container))
618 self._configure_cgroup('docker')
621 """Create/deploy container.
623 :raises RuntimeError: If creating a container failed.
625 cpuset_cpus = '--cpuset-cpus={0}'.format(
626 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
627 if self.container.cpuset_cpus else ''
629 cpuset_mems = '--cpuset-mems={0}'.format(self.container.cpuset_mems)\
630 if self.container.cpuset_mems is not None else ''
631 # Temporary workaround - disabling due to bug in memif
635 ' '.join('--env %s' % env for env in self.container.env))\
636 if self.container.env else ''
638 command = '{0}'.format(self.container.command)\
639 if self.container.command else ''
641 publish = '{0}'.format(
642 ' '.join('--publish %s' % var for var in self.container.publish))\
643 if self.container.publish else ''
645 volume = '--volume {c.host_dir}:{c.guest_dir}'.format(c=self.container)\
646 if self.container.host_dir and self.container.guest_dir else ''
649 '--privileged --detach --interactive --tty --rm '\
650 '--cgroup-parent docker {cpuset_cpus} {cpuset_mems} {publish} '\
651 '{env} {volume} --name {container.name} {container.image} '\
652 '{command}'.format(cpuset_cpus=cpuset_cpus, cpuset_mems=cpuset_mems,
653 container=self.container, command=command,
654 env=env, publish=publish, volume=volume)
656 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
658 raise RuntimeError('Failed to create container {c.name}'
659 .format(c=self.container))
663 def execute(self, command):
664 """Start a process inside a running container.
666 Runs the specified command inside the container specified by name. The
667 container has to be running already.
669 :param command: Command to run inside container.
671 :raises RuntimeError: If runnig the command in a container failed.
673 cmd = "docker exec --interactive {c.name} /bin/sh -c '{command}'"\
674 .format(c=self.container, command=command)
676 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
678 raise RuntimeError('Failed to execute command in container '
679 '{c.name}.'.format(c=self.container))
682 """Stop running container.
684 :raises RuntimeError: If stopping a container failed.
686 cmd = 'docker stop {c.name}'.format(c=self.container)
688 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
690 raise RuntimeError('Failed to stop container {c.name}.'
691 .format(c=self.container))
694 """Remove a container.
696 :raises RuntimeError: If removing a container failed.
698 cmd = 'docker rm --force {c.name}'.format(c=self.container)
700 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
702 raise RuntimeError('Failed to destroy container {c.name}.'
703 .format(c=self.container))
706 """Return low-level information on Docker objects.
708 :raises RuntimeError: If getting info about a container failed.
710 cmd = 'docker inspect {c.name}'.format(c=self.container)
712 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
714 raise RuntimeError('Failed to get info about container {c.name}.'
715 .format(c=self.container))
717 def system_info(self):
718 """Display the docker system-wide information.
720 :raises RuntimeError: If displaying system information failed.
722 cmd = 'docker system info'
724 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
726 raise RuntimeError('Failed to get system info.')
728 def is_container_present(self):
729 """Check if container is present on node.
731 :returns: True if container is present.
733 :raises RuntimeError: If getting info about a container failed.
735 cmd = 'docker ps --all --quiet --filter name={c.name}'\
736 .format(c=self.container)
738 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
740 raise RuntimeError('Failed to get info about container {c.name}.'
741 .format(c=self.container))
742 return True if stdout else False
744 def is_container_running(self):
745 """Check if container is running on node.
747 :returns: True if container is running.
749 :raises RuntimeError: If getting info about a container failed.
751 cmd = 'docker ps --quiet --filter name={c.name}'\
752 .format(c=self.container)
754 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
756 raise RuntimeError('Failed to get info about container {c.name}.'
757 .format(c=self.container))
758 return True if stdout else False
761 class Container(object):
762 """Container class."""
765 """Initialize Container object."""
768 def __getattr__(self, attr):
769 """Get attribute custom implementation.
771 :param attr: Attribute to get.
773 :returns: Attribute value or None.
777 return self.__dict__[attr]
781 def __setattr__(self, attr, value):
782 """Set attribute custom implementation.
784 :param attr: Attribute to set.
785 :param value: Value to set.
790 # Check if attribute exists
793 # Creating new attribute
795 self.__dict__['ssh'] = SSH()
796 self.__dict__['ssh'].connect(value)
797 self.__dict__[attr] = value
799 # Updating attribute base of type
800 if isinstance(self.__dict__[attr], list):
801 self.__dict__[attr].append(value)
803 self.__dict__[attr] = value