1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
16 from time import sleep
17 from string import Template
20 # Disable due to pylint bug
21 # pylint: disable=no-name-in-module,import-error
22 from distutils.version import StrictVersion
24 from robot.api import logger
25 from resources.libraries.python.Constants import Constants
26 from resources.libraries.python.DpdkUtil import DpdkUtil
27 from resources.libraries.python.DUTSetup import DUTSetup
28 from resources.libraries.python.OptionString import OptionString
29 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
30 from resources.libraries.python.VPPUtil import VPPUtil
31 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
32 from resources.libraries.python.topology import NodeType, Topology
34 __all__ = ["QemuUtils"]
37 class QemuUtils(object):
40 # Use one instance of class per tests.
41 ROBOT_LIBRARY_SCOPE = 'TEST CASE'
43 def __init__(self, node, qemu_id=1, smp=1, mem=512, vnf=None,
44 img=Constants.QEMU_VM_IMAGE):
45 """Initialize QemuUtil class.
47 :param node: Node to run QEMU on.
48 :param qemu_id: QEMU identifier.
49 :param smp: Number of virtual SMP units (cores).
50 :param mem: Amount of memory.
51 :param vnf: Network function workload.
52 :param img: QEMU disk image or kernel image path.
65 'port': 10021 + qemu_id,
66 'serial': 4555 + qemu_id,
71 if node['port'] != 22:
72 self._vm_info['host_port'] = node['port']
73 self._vm_info['host_username'] = node['username']
74 self._vm_info['host_password'] = node['password']
77 self._opt['qemu_id'] = qemu_id
78 self._opt['mem'] = int(mem)
79 self._opt['smp'] = int(smp)
80 self._opt['img'] = img
81 self._opt['vnf'] = vnf
84 self._temp['pidfile'] = '/var/run/qemu_{id}.pid'.format(id=qemu_id)
85 if '/var/lib/vm/' in img:
86 self._opt['vm_type'] = 'nestedvm'
87 self._temp['qmp'] = '/var/run/qmp_{id}.sock'.format(id=qemu_id)
88 self._temp['qga'] = '/var/run/qga_{id}.sock'.format(id=qemu_id)
89 elif '/opt/boot/vmlinuz' in img:
90 self._opt['vm_type'] = 'kernelvm'
91 self._temp['log'] = '/tmp/serial_{id}.log'.format(id=qemu_id)
92 self._temp['ini'] = '/etc/vm_init_{id}.conf'.format(id=qemu_id)
94 raise RuntimeError('QEMU: Unknown VM image option!')
95 # Computed parameters for QEMU command line.
96 self._params = OptionString(prefix='-')
100 """Set QEMU command line parameters."""
101 self.add_default_params()
102 if self._opt.get('vm_type', '') == 'nestedvm':
103 self.add_nestedvm_params()
104 elif self._opt.get('vm_type', '') == 'kernelvm':
105 self.add_kernelvm_params()
107 raise RuntimeError('QEMU: Unsupported VM type!')
109 def add_default_params(self):
110 """Set default QEMU command line parameters."""
111 self._params.add('daemonize')
112 self._params.add('nodefaults')
113 self._params.add_with_value('name', 'vnf{qemu},debug-threads=on'.format(
114 qemu=self._opt.get('qemu_id')))
115 self._params.add('no-user-config')
116 self._params.add_with_value('monitor', 'none')
117 self._params.add_with_value('display', 'none')
118 self._params.add_with_value('vga', 'none')
119 self._params.add('enable-kvm')
120 self._params.add_with_value('pidfile', self._temp.get('pidfile'))
121 self._params.add_with_value('cpu', 'host')
122 self._params.add_with_value(
123 'machine', 'pc,accel=kvm,usb=off,mem-merge=off')
124 self._params.add_with_value(
125 'smp', '{smp},sockets=1,cores={smp},threads=1'.format(
126 smp=self._opt.get('smp')))
127 self._params.add_with_value(
128 'object', 'memory-backend-file,id=mem,size={mem}M,'
129 'mem-path=/dev/hugepages,share=on'.format(mem=self._opt.get('mem')))
130 self._params.add_with_value(
131 'm', '{mem}M'.format(mem=self._opt.get('mem')))
132 self._params.add_with_value('numa', 'node,memdev=mem')
133 self._params.add_with_value('balloon', 'none')
135 def add_nestedvm_params(self):
136 """Set NestedVM QEMU parameters."""
137 self._params.add_with_value(
138 'net', 'nic,macaddr=52:54:00:00:{qemu:02x}:ff'.format(
139 qemu=self._opt.get('qemu_id')))
140 self._params.add_with_value(
141 'net', 'user,hostfwd=tcp::{info[port]}-:22'.format(
143 # TODO: Remove try except after fully migrated to Bionic or
144 # qemu_set_node is removed.
146 locking = ',file.locking=off'\
147 if self.qemu_version(version='2.10') else ''
148 except AttributeError:
150 self._params.add_with_value(
151 'drive', 'file={img},format=raw,cache=none,if=virtio{locking}'.
152 format(img=self._opt.get('img'), locking=locking))
153 self._params.add_with_value(
154 'qmp', 'unix:{qmp},server,nowait'.format(qmp=self._temp.get('qmp')))
155 self._params.add_with_value(
156 'chardev', 'socket,host=127.0.0.1,port={info[serial]},'
157 'id=gnc0,server,nowait'.format(info=self._vm_info))
158 self._params.add_with_value('device', 'isa-serial,chardev=gnc0')
159 self._params.add_with_value(
160 'chardev', 'socket,path={qga},server,nowait,id=qga0'.format(
161 qga=self._temp.get('qga')))
162 self._params.add_with_value('device', 'isa-serial,chardev=qga0')
164 def add_kernelvm_params(self):
165 """Set KernelVM QEMU parameters."""
166 self._params.add_with_value(
167 'chardev', 'file,id=char0,path={log}'.format(
168 log=self._temp.get('log')))
169 self._params.add_with_value('device', 'isa-serial,chardev=char0')
170 self._params.add_with_value(
171 'fsdev', 'local,id=root9p,path=/,security_model=none')
172 self._params.add_with_value(
173 'device', 'virtio-9p-pci,fsdev=root9p,mount_tag=/dev/root')
174 self._params.add_with_value(
175 'kernel', '$(readlink -m {img}* | tail -1)'.format(
176 img=self._opt.get('img')))
177 self._params.add_with_value(
178 'append', '"ro rootfstype=9p rootflags=trans=virtio console=ttyS0'
179 ' tsc=reliable hugepages=256 init={init}"'.format(
180 init=self._temp.get('ini')))
182 def create_kernelvm_config_vpp(self, **kwargs):
183 """Create QEMU VPP config files.
185 :param kwargs: Key-value pairs to replace content of VPP configuration
189 startup = ('/etc/vpp/vm_startup_{id}.conf'.
190 format(id=self._opt.get('qemu_id')))
191 running = ('/etc/vpp/vm_running_{id}.exec'.
192 format(id=self._opt.get('qemu_id')))
194 self._temp['startup'] = startup
195 self._temp['running'] = running
196 self._opt['vnf_bin'] = ('/usr/bin/vpp -c {startup}'.
197 format(startup=startup))
199 # Create VPP startup configuration.
200 vpp_config = VppConfigGenerator()
201 vpp_config.set_node(self._node)
202 vpp_config.add_unix_nodaemon()
203 vpp_config.add_unix_cli_listen()
204 vpp_config.add_unix_exec(running)
205 vpp_config.add_cpu_main_core('0')
206 vpp_config.add_cpu_corelist_workers('1-{smp}'.
207 format(smp=self._opt.get('smp')-1))
208 vpp_config.add_dpdk_dev('0000:00:06.0', '0000:00:07.0')
209 vpp_config.add_dpdk_dev_default_rxq(kwargs['queues'])
210 vpp_config.add_dpdk_log_level('debug')
211 if not kwargs['jumbo_frames']:
212 vpp_config.add_dpdk_no_multi_seg()
213 vpp_config.add_dpdk_no_tx_checksum_offload()
214 vpp_config.add_plugin('disable', 'default')
215 vpp_config.add_plugin('enable', 'dpdk_plugin.so')
216 vpp_config.write_config(startup)
218 # Create VPP running configuration.
219 template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,
220 tpl=self._opt.get('vnf'))
221 exec_cmd_no_error(self._node, 'rm -f {running}'.format(running=running),
224 with open(template, 'r') as src_file:
225 src = Template(src_file.read())
227 self._node, "echo '{out}' | sudo tee {running}".format(
228 out=src.safe_substitute(**kwargs), running=running))
230 def create_kernelvm_config_testpmd_io(self, **kwargs):
231 """Create QEMU testpmd-io command line.
233 :param kwargs: Key-value pairs to construct command line parameters.
236 testpmd_path = ('{path}/{arch}-native-linuxapp-gcc/app'.
237 format(path=Constants.QEMU_VM_DPDK,
238 arch=Topology.get_node_arch(self._node)))
239 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
240 eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
244 pmd_rxq=kwargs['queues'],
245 pmd_txq=kwargs['queues'],
246 pmd_tx_offloads=False,
247 pmd_disable_hw_vlan=False,
248 pmd_max_pkt_len=9200 if kwargs['jumbo_frames'] else None,
249 pmd_nb_cores=str(self._opt.get('smp') - 1))
251 self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
252 format(testpmd_path=testpmd_path,
253 testpmd_cmd=testpmd_cmd))
255 def create_kernelvm_config_testpmd_mac(self, **kwargs):
256 """Create QEMU testpmd-mac command line.
258 :param kwargs: Key-value pairs to construct command line parameters.
261 testpmd_path = ('{path}/{arch}-native-linuxapp-gcc/app'.
262 format(path=Constants.QEMU_VM_DPDK,
263 arch=Topology.get_node_arch(self._node)))
264 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
265 eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
270 pmd_eth_peer_0='0,{mac}'.format(mac=kwargs['vif1_mac']),
271 pmd_eth_peer_1='1,{mac}'.format(mac=kwargs['vif2_mac']),
272 pmd_rxq=kwargs['queues'],
273 pmd_txq=kwargs['queues'],
274 pmd_tx_offloads=False,
275 pmd_disable_hw_vlan=False,
276 pmd_max_pkt_len=9200 if kwargs['jumbo_frames'] else None,
277 pmd_nb_cores=str(self._opt.get('smp') - 1))
279 self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
280 format(testpmd_path=testpmd_path,
281 testpmd_cmd=testpmd_cmd))
283 def create_kernelvm_init(self, **kwargs):
284 """Create QEMU init script.
286 :param kwargs: Key-value pairs to replace content of init startup file.
289 template = '{res}/init.sh'.format(res=Constants.RESOURCES_TPL_VM)
290 init = self._temp.get('ini')
292 self._node, 'rm -f {init}'.format(init=init), sudo=True)
294 with open(template, 'r') as src_file:
295 src = Template(src_file.read())
297 self._node, "echo '{out}' | sudo tee {init}".format(
298 out=src.safe_substitute(**kwargs), init=init))
300 self._node, "chmod +x {init}".format(init=init), sudo=True)
302 def configure_kernelvm_vnf(self, **kwargs):
303 """Create KernelVM VNF configurations.
305 :param kwargs: Key-value pairs for templating configs.
308 if 'vpp' in self._opt.get('vnf'):
309 self.create_kernelvm_config_vpp(**kwargs)
310 elif 'testpmd_io' in self._opt.get('vnf'):
311 self.create_kernelvm_config_testpmd_io(**kwargs)
312 elif 'testpmd_mac' in self._opt.get('vnf'):
313 self.create_kernelvm_config_testpmd_mac(**kwargs)
315 raise RuntimeError('QEMU: Unsupported VNF!')
316 self.create_kernelvm_init(vnf_bin=self._opt['vnf_bin'])
318 def get_qemu_pids(self):
319 """Get QEMU CPU pids.
321 :returns: List of QEMU CPU pids.
324 command = ("grep -rwl 'CPU' /proc/$(sudo cat {pidfile})/task/*/comm ".
325 format(pidfile=self._temp.get('pidfile')))
326 command += (r"| xargs dirname | sed -e 's/\/.*\///g' | uniq")
328 stdout, _ = exec_cmd_no_error(self._node, command)
329 return stdout.splitlines()
331 def qemu_set_affinity(self, *host_cpus):
332 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
333 of CPU cores. Function tries to execute 3 times to avoid race condition
334 in getting thread PIDs.
336 :param host_cpus: List of CPU cores.
337 :type host_cpus: list
341 qemu_cpus = self.get_qemu_pids()
343 if len(qemu_cpus) != len(host_cpus):
346 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
347 command = ('taskset -pc {host_cpu} {thread}'.
348 format(host_cpu=host_cpu, thread=qemu_cpu))
349 message = ('QEMU: Set affinity failed on {host}!'.
350 format(host=self._node['host']))
351 exec_cmd_no_error(self._node, command, sudo=True,
354 except (RuntimeError, ValueError):
359 raise RuntimeError('Failed to set Qemu threads affinity!')
361 def qemu_set_scheduler_policy(self):
362 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
365 :raises RuntimeError: Set scheduler policy failed.
368 qemu_cpus = self.get_qemu_pids()
370 for qemu_cpu in qemu_cpus:
371 command = ('chrt -r -p 1 {thread}'.
372 format(thread=qemu_cpu))
373 message = ('QEMU: Set SCHED_RR failed on {host}'.
374 format(host=self._node['host']))
375 exec_cmd_no_error(self._node, command, sudo=True,
377 except (RuntimeError, ValueError):
381 def qemu_add_vhost_user_if(self, socket, server=True, jumbo_frames=False,
382 queue_size=None, queues=1):
383 """Add Vhost-user interface.
385 :param socket: Path of the unix socket.
386 :param server: If True the socket shall be a listening socket.
387 :param jumbo_frames: Set True if jumbo frames are used in the test.
388 :param queue_size: Vring queue size.
389 :param queues: Number of queues.
392 :type jumbo_frames: bool
393 :type queue_size: int
397 self._params.add_with_value(
398 'chardev', 'socket,id=char{vhost},path={socket}{server}'.format(
399 vhost=self._vhost_id, socket=socket,
400 server=',server' if server is True else ''))
401 self._params.add_with_value(
402 'netdev', 'vhost-user,id=vhost{vhost},chardev=char{vhost},'
403 'queues={queues}'.format(vhost=self._vhost_id, queues=queues))
404 mac = ('52:54:00:00:{qemu:02x}:{vhost:02x}'.
405 format(qemu=self._opt.get('qemu_id'), vhost=self._vhost_id))
406 queue_size = ('rx_queue_size={queue_size},tx_queue_size={queue_size}'.
407 format(queue_size=queue_size)) if queue_size else ''
408 mbuf = 'on,host_mtu=9200'
409 self._params.add_with_value(
410 'device', 'virtio-net-pci,netdev=vhost{vhost},mac={mac},bus=pci.0,'
411 'addr={addr}.0,mq=on,vectors={vectors},csum=off,gso=off,'
412 'guest_tso4=off,guest_tso6=off,guest_ecn=off,mrg_rxbuf={mbuf},'
413 '{queue_size}'.format(
414 addr=self._vhost_id+5, vhost=self._vhost_id, mac=mac,
415 mbuf=mbuf if jumbo_frames else 'off', queue_size=queue_size,
416 vectors=(2 * queues + 2)))
418 # Add interface MAC and socket to the node dict.
419 if_data = {'mac_address': mac, 'socket': socket}
420 if_name = 'vhost{vhost}'.format(vhost=self._vhost_id)
421 self._vm_info['interfaces'][if_name] = if_data
422 # Add socket to temporary file list.
423 self._temp[if_name] = socket
425 def _qemu_qmp_exec(self, cmd):
426 """Execute QMP command.
428 QMP is JSON based protocol which allows to control QEMU instance.
430 :param cmd: QMP command to execute.
432 :returns: Command output in python representation of JSON format. The
433 { "return": {} } response is QMP's success response. An error
434 response will contain the "error" keyword instead of "return".
436 # To enter command mode, the qmp_capabilities command must be issued.
437 command = ('echo "{{ \\"execute\\": \\"qmp_capabilities\\" }}'
438 '{{ \\"execute\\": \\"{cmd}\\" }}" | '
439 'sudo -S socat - UNIX-CONNECT:{qmp}'.
440 format(cmd=cmd, qmp=self._temp.get('qmp')))
441 message = ('QMP execute "{cmd}" failed on {host}'.
442 format(cmd=cmd, host=self._node['host']))
443 stdout, _ = exec_cmd_no_error(
444 self._node, command, sudo=False, message=message)
446 # Skip capabilities negotiation messages.
447 out_list = stdout.splitlines()
448 if len(out_list) < 3:
450 'Invalid QMP output on {host}'.format(host=self._node['host']))
451 return json.loads(out_list[2])
453 def _qemu_qga_flush(self):
454 """Flush the QGA parser state."""
455 command = ('(printf "\xFF"; sleep 1) | '
456 'sudo -S socat - UNIX-CONNECT:{qga}'.
457 format(qga=self._temp.get('qga')))
458 message = ('QGA flush failed on {host}'.format(host=self._node['host']))
459 stdout, _ = exec_cmd_no_error(
460 self._node, command, sudo=False, message=message)
462 return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
464 def _qemu_qga_exec(self, cmd):
465 """Execute QGA command.
467 QGA provide access to a system-level agent via standard QMP commands.
469 :param cmd: QGA command to execute.
472 command = ('(echo "{{ \\"execute\\": \\"{cmd}\\" }}"; sleep 1) | '
473 'sudo -S socat - UNIX-CONNECT:{qga}'.
474 format(cmd=cmd, qga=self._temp.get('qga')))
475 message = ('QGA execute "{cmd}" failed on {host}'.
476 format(cmd=cmd, host=self._node['host']))
477 stdout, _ = exec_cmd_no_error(
478 self._node, command, sudo=False, message=message)
480 return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
482 def _wait_until_vm_boot(self):
483 """Wait until QEMU with NestedVM is booted."""
484 if self._opt.get('vm_type') == 'nestedvm':
485 self._wait_until_nestedvm_boot()
486 self._update_vm_interfaces()
487 elif self._opt.get('vm_type') == 'kernelvm':
488 self._wait_until_kernelvm_boot()
490 raise RuntimeError('QEMU: Unsupported VM type!')
492 def _wait_until_nestedvm_boot(self, retries=12):
493 """Wait until QEMU with NestedVM is booted.
495 First try to flush qga until there is output.
496 Then ping QEMU guest agent each 5s until VM booted or timeout.
498 :param retries: Number of retries with 5s between trials.
501 for _ in range(retries):
504 out = self._qemu_qga_flush()
506 logger.trace('QGA qga flush unexpected output {out}'.
508 # Empty output - VM not booted yet
514 raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
515 format(host=self._node['host']))
516 for _ in range(retries):
519 out = self._qemu_qga_exec('guest-ping')
521 logger.trace('QGA guest-ping unexpected output {out}'.
523 # Empty output - VM not booted yet.
526 # Non-error return - VM booted.
527 elif out.get('return') is not None:
529 # Skip error and wait.
530 elif out.get('error') is not None:
533 # If there is an unexpected output from QGA guest-info, try
534 # again until timeout.
535 logger.trace('QGA guest-ping unexpected output {out}'.
538 raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
539 format(host=self._node['host']))
541 def _wait_until_kernelvm_boot(self, retries=60):
542 """Wait until QEMU KernelVM is booted.
544 :param retries: Number of retries.
547 vpp_ver = VPPUtil.vpp_show_version(self._node)
549 for _ in range(retries):
550 command = ('tail -1 {log}'.format(log=self._temp.get('log')))
553 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
557 if vpp_ver in stdout or 'Press enter to exit' in stdout:
559 if 'reboot: Power down' in stdout:
560 raise RuntimeError('QEMU: NF failed to run on {host}!'.
561 format(host=self._node['host']))
563 raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
564 format(host=self._node['host']))
566 def _update_vm_interfaces(self):
567 """Update interface names in VM node dict."""
568 # Send guest-network-get-interfaces command via QGA, output example:
569 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
570 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
571 out = self._qemu_qga_exec('guest-network-get-interfaces')
572 interfaces = out.get('return')
575 raise RuntimeError('Get VM interface list failed on {host}'.
576 format(host=self._node['host']))
577 # Create MAC-name dict.
578 for interface in interfaces:
579 if 'hardware-address' not in interface:
581 mac_name[interface['hardware-address']] = interface['name']
582 # Match interface by MAC and save interface name.
583 for interface in self._vm_info['interfaces'].values():
584 mac = interface.get('mac_address')
585 if_name = mac_name.get(mac)
588 'Interface name for MAC {mac} not found'.format(mac=mac))
590 interface['name'] = if_name
592 def qemu_start(self):
593 """Start QEMU and wait until VM boot.
595 :returns: VM node info.
598 cmd_opts = OptionString()
599 cmd_opts.add('{bin_path}/qemu-system-{arch}'.format(
600 bin_path=Constants.QEMU_BIN_PATH,
601 arch=Topology.get_node_arch(self._node)))
602 cmd_opts.extend(self._params)
603 message = ('QEMU: Start failed on {host}!'.
604 format(host=self._node['host']))
606 DUTSetup.check_huge_page(
607 self._node, '/dev/hugepages', self._opt.get('mem'))
610 self._node, cmd_opts, timeout=300, sudo=True, message=message)
611 self._wait_until_vm_boot()
618 """Kill qemu process."""
619 exec_cmd(self._node, 'chmod +r {pidfile}'.
620 format(pidfile=self._temp.get('pidfile')), sudo=True)
621 exec_cmd(self._node, 'kill -SIGKILL $(cat {pidfile})'.
622 format(pidfile=self._temp.get('pidfile')), sudo=True)
624 for value in self._temp.values():
625 exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
626 exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
628 def qemu_kill_all(self):
629 """Kill all qemu processes on DUT node if specified."""
630 exec_cmd(self._node, 'pkill -SIGKILL qemu', sudo=True)
632 for value in self._temp.values():
633 exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
634 exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
636 def qemu_version(self, version=None):
637 """Return Qemu version or compare if version is higher than parameter.
639 :param version: Version to compare.
641 :returns: Qemu version or Boolean if version is higher than parameter.
644 command = ('{bin_path}/qemu-system-{arch} --version'.format(
645 bin_path=Constants.QEMU_BIN_PATH,
646 arch=Topology.get_node_arch(self._node)))
648 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
649 ver = match(r'QEMU emulator version ([\d.]*)', stdout).group(1)
650 return StrictVersion(ver) > StrictVersion(version) \