1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 # Bug workaround in pylint for abstract classes.
15 #pylint: disable=W0223
17 """Library to manipulate Containers."""
19 from collections import OrderedDict, Counter
21 from resources.libraries.python.ssh import SSH
22 from resources.libraries.python.constants import Constants
23 from resources.libraries.python.CpuUtils import CpuUtils
24 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
27 __all__ = ["ContainerManager", "ContainerEngine", "LXC", "Docker", "Container"]
29 SUPERVISOR_CONF = '/etc/supervisord.conf'
32 class ContainerManager(object):
33 """Container lifecycle management class."""
35 def __init__(self, engine):
36 """Initialize Container Manager class.
38 :param engine: Container technology used (LXC/Docker/...).
40 :raises NotImplementedError: If container technology is not implemented.
43 self.engine = globals()[engine]()
45 raise NotImplementedError('{e} is not implemented.'
47 self.containers = OrderedDict()
49 def get_container_by_name(self, name):
50 """Get container instance.
52 :param name: Container name.
54 :returns: Container instance.
56 :raises RuntimeError: If failed to get container with name.
59 return self.containers[name]
61 raise RuntimeError('Failed to get container with name: {n}'
64 def construct_container(self, **kwargs):
65 """Construct container object on node with specified parameters.
67 :param kwargs: Key-value pairs used to construct container.
71 self.engine.initialize()
74 setattr(self.engine.container, key, kwargs[key])
76 # Set additional environmental variables
77 setattr(self.engine.container, 'env',
78 'MICROSERVICE_LABEL={n}'.format(n=kwargs['name']))
80 # Set cpuset.cpus cgroup
81 skip_cnt = kwargs['cpu_skip']
82 if not kwargs['cpu_shared']:
83 skip_cnt += kwargs['i'] * kwargs['cpu_count']
84 self.engine.container.cpuset_cpus = \
85 CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
86 cpu_node=kwargs['cpuset_mems'],
88 cpu_cnt=kwargs['cpu_count'],
89 smt_used=kwargs['smt_used'])
91 # Store container instance
92 self.containers[kwargs['name']] = self.engine.container
94 def construct_containers(self, **kwargs):
95 """Construct 1..N container(s) on node with specified name.
96 Ordinal number is automatically added to the name of container as
99 :param kwargs: Name of container.
102 name = kwargs['name']
103 for i in range(kwargs['count']):
104 # Name will contain ordinal suffix
105 kwargs['name'] = ''.join([name, str(i+1)])
107 self.construct_container(i=i, **kwargs)
109 def acquire_all_containers(self):
110 """Acquire all containers."""
111 for container in self.containers:
112 self.engine.container = self.containers[container]
113 self.engine.acquire()
115 def build_all_containers(self):
116 """Build all containers."""
117 for container in self.containers:
118 self.engine.container = self.containers[container]
121 def create_all_containers(self):
122 """Create all containers."""
123 for container in self.containers:
124 self.engine.container = self.containers[container]
127 def execute_on_container(self, name, command):
128 """Execute command on container with name.
130 :param name: Container name.
131 :param command: Command to execute.
135 self.engine.container = self.get_container_by_name(name)
136 self.engine.execute(command)
138 def execute_on_all_containers(self, command):
139 """Execute command on all containers.
141 :param command: Command to execute.
144 for container in self.containers:
145 self.engine.container = self.containers[container]
146 self.engine.execute(command)
148 def install_vpp_in_all_containers(self):
149 """Install VPP into all containers."""
150 for container in self.containers:
151 self.engine.container = self.containers[container]
152 # We need to install supervisor client/server system to control VPP
154 self.engine.install_supervisor()
155 self.engine.install_vpp()
156 self.engine.restart_vpp()
158 def configure_vpp_in_all_containers(self, vat_template_file):
159 """Configure VPP in all containers.
161 :param vat_template_file: Template file name of a VAT script.
162 :type vat_template_file: str
164 # Count number of DUTs based on node's host information
165 dut_cnt = len(Counter([self.containers[container].node['host']
166 for container in self.containers]))
167 container_cnt = len(self.containers)
168 mod = dut_cnt/container_cnt
170 for i, container in enumerate(self.containers):
171 self.engine.container = self.containers[container]
172 self.engine.create_vpp_startup_config()
173 self.engine.create_vpp_exec_config(vat_template_file,
174 memif_id1=i % mod * 2 + 1,
175 memif_id2=i % mod * 2 + 2,
176 socket1='memif-{c.name}-1'
177 .format(c=self.engine.container),
178 socket2='memif-{c.name}-2'
179 .format(c=self.engine.container))
181 def stop_all_containers(self):
182 """Stop all containers."""
183 for container in self.containers:
184 self.engine.container = self.containers[container]
187 def destroy_all_containers(self):
188 """Destroy all containers."""
189 for container in self.containers:
190 self.engine.container = self.containers[container]
191 self.engine.destroy()
194 class ContainerEngine(object):
195 """Abstract class for container engine."""
198 """Init ContainerEngine object."""
199 self.container = None
201 def initialize(self):
202 """Initialize container object."""
203 self.container = Container()
205 def acquire(self, force):
206 """Acquire/download container.
208 :param force: Destroy a container if exists and create.
211 raise NotImplementedError
214 """Build container (compile)."""
215 raise NotImplementedError
218 """Create/deploy container."""
219 raise NotImplementedError
221 def execute(self, command):
222 """Execute process inside container.
224 :param command: Command to run inside container.
227 raise NotImplementedError
230 """Stop container."""
231 raise NotImplementedError
234 """Destroy/remove container."""
235 raise NotImplementedError
238 """Info about container."""
239 raise NotImplementedError
241 def system_info(self):
243 raise NotImplementedError
245 def install_supervisor(self):
246 """Install supervisord inside a container."""
247 self.execute('sleep 3')
248 self.execute('apt-get update')
249 self.execute('apt-get install -y supervisor')
250 self.execute('echo "{0}" > {1}'
252 '[unix_http_server]\n'
253 'file = /tmp/supervisor.sock\n\n'
254 '[rpcinterface:supervisor]\n'
255 'supervisor.rpcinterface_factory = '
256 'supervisor.rpcinterface:make_main_rpcinterface\n\n'
258 'serverurl = unix:///tmp/supervisor.sock\n\n'
260 'pidfile = /tmp/supervisord.pid\n'
261 'identifier = supervisor\n'
263 'logfile=/tmp/supervisord.log\n'
265 'nodaemon=false\n\n',
267 self.execute('supervisord -c {0}'.format(SUPERVISOR_CONF))
269 def install_vpp(self, install_dkms=False):
270 """Install VPP inside a container.
272 :param install_dkms: If install dkms package. This will impact install
273 time. Dkms is required for installation of vpp-dpdk-dkms. Default is
275 :type install_dkms: bool
277 self.execute('ln -s /dev/null /etc/sysctl.d/80-vpp.conf')
278 self.execute('apt-get update')
280 self.execute('apt-get install -y dkms && '
281 'dpkg -i --force-all {0}/install_dir/*.deb'
282 .format(self.container.guest_dir))
284 self.execute('for i in $(ls -I \"*dkms*\" {0}/install_dir/); '
285 'do dpkg -i --force-all {0}/install_dir/$i; done'
286 .format(self.container.guest_dir))
287 self.execute('apt-get -f install -y')
288 self.execute('echo "{0}" >> {1}'
291 'command=/usr/bin/vpp -c /etc/vpp/startup.conf\n'
292 'autorestart=false\n'
293 'redirect_stderr=true\n'
296 self.execute('supervisorctl reload')
298 def restart_vpp(self):
299 """Restart VPP service inside a container."""
300 self.execute('supervisorctl restart vpp')
302 def create_vpp_startup_config(self,
303 config_filename='/etc/vpp/startup.conf'):
304 """Create base startup configuration of VPP on container.
306 :param config_filename: Startup configuration file name.
307 :type config_filename: str
309 cpuset_cpus = self.container.cpuset_cpus
311 # Create config instance
312 vpp_config = VppConfigGenerator()
313 vpp_config.set_node(self.container.node)
314 vpp_config.set_config_filename(config_filename)
315 vpp_config.add_unix_cli_listen()
316 vpp_config.add_unix_nodaemon()
317 vpp_config.add_unix_exec('/tmp/running.exec')
318 # We will pop first core from list to be main core
319 vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
320 # if this is not only core in list, the rest will be used as workers.
322 corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
323 vpp_config.add_cpu_corelist_workers(corelist_workers)
324 vpp_config.add_plugin_disable('dpdk_plugin.so')
326 self.execute('mkdir -p /etc/vpp/')
327 self.execute('echo "{c}" | tee {f}'
328 .format(c=vpp_config.get_config_str(),
329 f=vpp_config.get_config_filename()))
331 def create_vpp_exec_config(self, vat_template_file, **args):
332 """Create VPP exec configuration on container.
334 :param vat_template_file: File name of a VAT template script.
335 :param args: Parameters for VAT script.
336 :type vat_template_file: str
339 vat_file_path = '{p}/{f}'.format(p=Constants.RESOURCES_TPL_VAT,
342 with open(vat_file_path, 'r') as template_file:
343 cmd_template = template_file.readlines()
344 for line_tmpl in cmd_template:
345 vat_cmd = line_tmpl.format(**args)
346 self.execute('echo "{c}" >> /tmp/running.exec'
347 .format(c=vat_cmd.replace('\n', '')))
349 def is_container_running(self):
350 """Check if container is running."""
351 raise NotImplementedError
353 def is_container_present(self):
354 """Check if container is present."""
355 raise NotImplementedError
358 class LXC(ContainerEngine):
359 """LXC implementation."""
362 """Initialize LXC object."""
363 super(LXC, self).__init__()
365 def acquire(self, force=True):
366 """Acquire a privileged system object where configuration is stored and
367 where user information can be stored.
369 :param force: If a container exists, destroy it and create a new
372 :raises RuntimeError: If creating the container or writing the container
375 if self.is_container_present():
381 image = self.container.image if self.container.image else\
382 "-d ubuntu -r xenial -a amd64"
384 cmd = 'lxc-create -t download --name {c.name} -- {image} '\
385 '--no-validate'.format(c=self.container, image=image)
387 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
389 raise RuntimeError('Failed to create container.')
391 if self.container.host_dir and self.container.guest_dir:
392 entry = 'lxc.mount.entry = '\
393 '{c.host_dir} /var/lib/lxc/{c.name}/rootfs{c.guest_dir} ' \
394 'none bind,create=dir 0 0'.format(c=self.container)
395 ret, _, _ = self.container.ssh.exec_command_sudo(
396 "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'"
397 .format(e=entry, c=self.container))
399 raise RuntimeError('Failed to write {c.name} config.'
400 .format(c=self.container))
403 """Create/deploy an application inside a container on system.
405 :raises RuntimeError: If creating the container fails.
407 cpuset_cpus = '{0}'.format(
408 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
409 if self.container.cpuset_cpus else ''
411 cmd = 'lxc-start --name {c.name} --daemon'.format(c=self.container)
413 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
415 raise RuntimeError('Failed to start container {c.name}.'
416 .format(c=self.container))
417 self._lxc_wait('RUNNING')
418 self._lxc_cgroup(state_object='cpuset.cpus',
421 def execute(self, command):
422 """Start a process inside a running container. Runs the specified
423 command inside the container specified by name. The container has to
426 :param command: Command to run inside container.
428 :raises RuntimeError: If running the command failed.
430 env = '--keep-env {0}'.format(
431 ' '.join('--set-var %s' % env for env in self.container.env))\
432 if self.container.env else ''
434 cmd = "lxc-attach {env} --name {c.name} -- /bin/sh -c '{command}'"\
435 .format(env=env, c=self.container, command=command)
437 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
439 raise RuntimeError('Failed to run command inside container '
440 '{c.name}.'.format(c=self.container))
445 :raises RuntimeError: If stopping the container failed.
447 cmd = 'lxc-stop --name {c.name}'.format(c=self.container)
449 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
451 raise RuntimeError('Failed to stop container {c.name}.'
452 .format(c=self.container))
453 self._lxc_wait('STOPPED|FROZEN')
456 """Destroy a container.
458 :raises RuntimeError: If destroying container failed.
460 cmd = 'lxc-destroy --force --name {c.name}'.format(c=self.container)
462 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
464 raise RuntimeError('Failed to destroy container {c.name}.'
465 .format(c=self.container))
468 """Query and shows information about a container.
470 :raises RuntimeError: If getting info about a container failed.
472 cmd = 'lxc-info --name {c.name}'.format(c=self.container)
474 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
476 raise RuntimeError('Failed to get info about container {c.name}.'
477 .format(c=self.container))
479 def system_info(self):
480 """Check the current kernel for LXC support.
482 :raises RuntimeError: If checking LXC support failed.
484 cmd = 'lxc-checkconfig'
486 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
488 raise RuntimeError('Failed to check LXC support.')
490 def is_container_running(self):
491 """Check if container is running on node.
493 :returns: True if container is running.
495 :raises RuntimeError: If getting info about a container failed.
497 cmd = 'lxc-info --no-humanize --state --name {c.name}'\
498 .format(c=self.container)
500 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
502 raise RuntimeError('Failed to get info about container {c.name}.'
503 .format(c=self.container))
504 return True if 'RUNNING' in stdout else False
506 def is_container_present(self):
507 """Check if container is existing on node.
509 :returns: True if container is present.
511 :raises RuntimeError: If getting info about a container failed.
513 cmd = 'lxc-info --no-humanize --name {c.name}'.format(c=self.container)
515 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
516 return False if int(ret) else True
518 def _lxc_wait(self, state):
519 """Wait for a specific container state.
521 :param state: Specify the container state(s) to wait for.
523 :raises RuntimeError: If waiting for state of a container failed.
525 cmd = 'lxc-wait --name {c.name} --state "{s}"'\
526 .format(c=self.container, s=state)
528 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
530 raise RuntimeError('Failed to wait for state "{s}" of container '
531 '{c.name}.'.format(s=state, c=self.container))
533 def _lxc_cgroup(self, state_object, value=''):
534 """Manage the control group associated with a container.
536 :param state_object: Specify the state object name.
537 :param value: Specify the value to assign to the state object. If empty,
538 then action is GET, otherwise is action SET.
539 :type state_object: str
541 :raises RuntimeError: If getting/setting state of a container failed.
543 cmd = 'lxc-cgroup --name {c.name} {s} {v}'\
544 .format(c=self.container, s=state_object, v=value)
546 ret, _, _ = self.container.ssh.exec_command_sudo(
547 'cgset --copy-from / lxc')
549 raise RuntimeError('Failed to copy cgroup settings from root.')
551 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
554 raise RuntimeError('Failed to set {s} of container {c.name}.'
555 .format(s=state_object, c=self.container))
557 raise RuntimeError('Failed to get {s} of container {c.name}.'
558 .format(s=state_object, c=self.container))
561 class Docker(ContainerEngine):
562 """Docker implementation."""
565 """Initialize Docker object."""
566 super(Docker, self).__init__()
568 def acquire(self, force=True):
569 """Pull an image or a repository from a registry.
571 :param force: Destroy a container if exists.
573 :raises RuntimeError: If pulling a container failed.
575 if self.is_container_present():
581 cmd = 'docker pull {c.image}'.format(c=self.container)
583 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
585 raise RuntimeError('Failed to create container {c.name}.'
586 .format(c=self.container))
589 """Create/deploy container.
591 :raises RuntimeError: If creating a container failed.
593 cpuset_cpus = '--cpuset-cpus={0}'.format(
594 ','.join('%s' % cpu for cpu in self.container.cpuset_cpus))\
595 if self.container.cpuset_cpus else ''
597 cpuset_mems = '--cpuset-mems={0}'.format(self.container.cpuset_mems)\
598 if self.container.cpuset_mems is not None else ''
601 ' '.join('--env %s' % env for env in self.container.env))\
602 if self.container.env else ''
604 command = '{0}'.format(self.container.command)\
605 if self.container.command else ''
607 publish = '{0}'.format(
608 ' '.join('--publish %s' % var for var in self.container.publish))\
609 if self.container.publish else ''
611 volume = '--volume {c.host_dir}:{c.guest_dir}'.format(c=self.container)\
612 if self.container.host_dir and self.container.guest_dir else ''
615 '--privileged --detach --interactive --tty --rm '\
616 '--cgroup-parent lxc {cpuset_cpus} {cpuset_mems} {publish} '\
617 '{env} {volume} --name {container.name} {container.image} '\
618 '{command}'.format(cpuset_cpus=cpuset_cpus, cpuset_mems=cpuset_mems,
619 container=self.container, command=command,
620 env=env, publish=publish, volume=volume)
622 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
624 raise RuntimeError('Failed to create container {c.name}'
625 .format(c=self.container))
629 def execute(self, command):
630 """Start a process inside a running container. Runs the specified
631 command inside the container specified by name. The container has to
634 :param command: Command to run inside container.
636 :raises RuntimeError: If runnig the command in a container failed.
638 cmd = "docker exec --interactive {c.name} /bin/sh -c '{command}'"\
639 .format(c=self.container, command=command)
641 ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=180)
643 raise RuntimeError('Failed to execute command in container '
644 '{c.name}.'.format(c=self.container))
647 """Stop running container.
649 :raises RuntimeError: If stopping a container failed.
651 cmd = 'docker stop {c.name}'.format(c=self.container)
653 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
655 raise RuntimeError('Failed to stop container {c.name}.'
656 .format(c=self.container))
659 """Remove a container.
661 :raises RuntimeError: If removing a container failed.
663 cmd = 'docker rm --force {c.name}'.format(c=self.container)
665 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
667 raise RuntimeError('Failed to destroy container {c.name}.'
668 .format(c=self.container))
671 """Return low-level information on Docker objects.
673 :raises RuntimeError: If getting info about a container failed.
675 cmd = 'docker inspect {c.name}'.format(c=self.container)
677 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
679 raise RuntimeError('Failed to get info about container {c.name}.'
680 .format(c=self.container))
682 def system_info(self):
683 """Display the docker system-wide information.
685 :raises RuntimeError: If displaying system information failed.
687 cmd = 'docker system info'
689 ret, _, _ = self.container.ssh.exec_command_sudo(cmd)
691 raise RuntimeError('Failed to get system info.')
693 def is_container_present(self):
694 """Check if container is present on node.
696 :returns: True if container is present.
698 :raises RuntimeError: If getting info about a container failed.
700 cmd = 'docker ps --all --quiet --filter name={c.name}'\
701 .format(c=self.container)
703 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
705 raise RuntimeError('Failed to get info about container {c.name}.'
706 .format(c=self.container))
707 return True if stdout else False
709 def is_container_running(self):
710 """Check if container is running on node.
712 :returns: True if container is running.
714 :raises RuntimeError: If getting info about a container failed.
716 cmd = 'docker ps --quiet --filter name={c.name}'\
717 .format(c=self.container)
719 ret, stdout, _ = self.container.ssh.exec_command_sudo(cmd)
721 raise RuntimeError('Failed to get info about container {c.name}.'
722 .format(c=self.container))
723 return True if stdout else False
726 class Container(object):
727 """Container class."""
730 """Initialize Container object."""
733 def __getattr__(self, attr):
735 return self.__dict__[attr]
739 def __setattr__(self, attr, value):
741 # Check if attribute exists
744 # Creating new attribute
746 self.__dict__['ssh'] = SSH()
747 self.__dict__['ssh'].connect(value)
748 self.__dict__[attr] = value
750 # Updating attribute base of type
751 if isinstance(self.__dict__[attr], list):
752 self.__dict__[attr].append(value)
754 self.__dict__[attr] = value