X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=extras%2Fvpp_config%2Fvpplib%2FVPPUtil.py;h=38c71081ac04cd8b7765242ea287d28e574048b3;hb=b2f09146c957f87edd431e75a24f8a456d539a02;hp=350b7759a0389729c5dedee041bf54fc028a96e7;hpb=68b0ee3a38e3a86f0389d8cc695915df190c3dfb;p=vpp.git diff --git a/extras/vpp_config/vpplib/VPPUtil.py b/extras/vpp_config/vpplib/VPPUtil.py index 350b7759a03..38c71081ac0 100644 --- a/extras/vpp_config/vpplib/VPPUtil.py +++ b/extras/vpp_config/vpplib/VPPUtil.py @@ -11,16 +11,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + """VPP util library""" import logging import re import subprocess -import platform +import requests from collections import Counter -# VPP_VERSION = '1707' -VPP_VERSION = '1710' +import distro + +ubuntu_pkgs = {'release': ['vpp', 'vpp-plugin-core', 'vpp-plugin-dpdk', 'vpp-api-python', 'python3-vpp-api', + 'vpp-dbg', 'vpp-dev'], + 'master': ['vpp', 'vpp-plugin-core', 'vpp-plugin-dpdk', 'vpp-api-python', 'python3-vpp-api', + 'vpp-dbg', 'vpp-dev']} + +centos_pkgs = {'release': ['vpp', 'vpp-selinux-policy', 'vpp-plugins', 'vpp-api-lua', + 'vpp-api-python', 'vpp-debuginfo', 'vpp-devel', 'libvpp0'], + 'master': ['vpp', 'vpp-selinux-policy', 'vpp-plugins', 'vpp-api-lua', + 'vpp-api-python', 'vpp-debuginfo', 'vpp-devel', 'libvpp0']} class VPPUtil(object): @@ -47,13 +58,19 @@ class VPPUtil(object): stderr=subprocess.PIPE) with prc.stdout: - for line in iter(prc.stdout.readline, b''): + lines = prc.stdout.readlines() + for line in lines: + if type(line) != str: + line = line.decode() logging.info(" {}".format(line.strip('\n'))) out += line with prc.stderr: - for line in iter(prc.stderr.readline, b''): - logging.warn(" {}".format(line.strip('\n'))) + lines = prc.stderr.readlines() + for line in lines: + if type(line) != str: + line = line.decode() + logging.warning(" {}".format(line.strip('\n'))) err += line ret = prc.wait() @@ -79,48 +96,15 @@ class VPPUtil(object): if ret != 0: logging.debug(stderr) - def _install_vpp_pkg_ubuntu(self, node, pkg): - """ - Install the VPP packages - - :param node: Node dictionary - :param pkg: The vpp packages - :type node: dict - :type pkg: string - """ - - cmd = 'apt-get -y install {}'.format(pkg) - (ret, stdout, stderr) = self.exec_command(cmd) - if ret != 0: - raise RuntimeError('{} failed on node {} {} {}'.format( - cmd, node['host'], stdout, stderr)) - - def _install_vpp_pkg_centos(self, node, pkg): - """ - Install the VPP packages - - :param node: Node dictionary - :param pkg: The vpp packages - :type node: dict - :type pkg: string - """ - - cmd = 'yum -y install {}'.format(pkg) - (ret, stdout, stderr) = self.exec_command(cmd) - if ret != 0: - raise RuntimeError('{} failed on node {} {} {}'.format( - cmd, node['host'], stdout, stderr)) - - def _install_vpp_ubuntu(self, node, fdio_release=VPP_VERSION, - ubuntu_version='xenial'): + def _install_vpp_ubuntu(self, node, branch, ubuntu_version='xenial'): """ Install the VPP packages :param node: Node dictionary with cpuinfo. - :param fdio_release: VPP release number + :param branch: VPP branch :param ubuntu_version: Ubuntu Version :type node: dict - :type fdio_release: string + :type branch: string :type ubuntu_version: string """ @@ -130,20 +114,18 @@ class VPPUtil(object): # Backup the sources list self._autoconfig_backup_file(sfile) - # Remove the current file - cmd = 'rm {}'.format(sfile) - (ret, stdout, stderr) = self.exec_command(cmd) - if ret != 0: - logging.debug('{} failed on node {} {}'.format( - cmd, - node['host'], - stderr)) + reps = 'deb [trusted=yes] https://packagecloud.io/fdio/' + reps += '{}/ubuntu {} main\n'.format(branch, ubuntu_version) + + with open(sfile, 'w') as sfd: + sfd.write(reps) + sfd.close() - reps = 'deb [trusted=yes] https://nexus.fd.io/content/' - reps += 'repositories/fd.io.stable.{}.ubuntu.{}.main/ ./\n' \ - .format(fdio_release, ubuntu_version) + # Add the key - cmd = 'echo "{0}" | sudo tee {1}'.format(reps, sfile) + key = requests.get( + 'https://packagecloud.io/fdio/{}/gpgkey'.format(branch)) + cmd = 'echo "{}" | apt-key add -'.format(key.content.decode(key.encoding)) (ret, stdout, stderr) = self.exec_command(cmd) if ret != 0: raise RuntimeError('{} failed on node {} {}'.format( @@ -160,30 +142,44 @@ class VPPUtil(object): node['host'], stderr)) - self._install_vpp_pkg_ubuntu(node, 'vpp-lib') - self._install_vpp_pkg_ubuntu(node, 'vpp') - self._install_vpp_pkg_ubuntu(node, 'vpp-plugins') - self._install_vpp_pkg_ubuntu(node, 'vpp-dpdk-dkms') - self._install_vpp_pkg_ubuntu(node, 'vpp-dpdk-dev') - self._install_vpp_pkg_ubuntu(node, 'vpp-api-python') - self._install_vpp_pkg_ubuntu(node, 'vpp-api-java') - self._install_vpp_pkg_ubuntu(node, 'vpp-api-lua') - self._install_vpp_pkg_ubuntu(node, 'vpp-dev') - self._install_vpp_pkg_ubuntu(node, 'vpp-dbg') + # Get the package list + pkgstr = '' + for ps in ubuntu_pkgs[branch]: + pkgstr += ps + ' ' - def _install_vpp_centos(self, node, fdio_release=VPP_VERSION, - centos_version='centos7'): + cmd = 'apt-get -y install {}'.format(pkgstr) + (ret, stdout, stderr) = self.exec_command(cmd) + if ret != 0: + raise RuntimeError('{} failed on node {} {} {}'.format( + cmd, node['host'], stdout, stderr)) + + def _install_vpp_centos(self, node, branch): """ Install the VPP packages :param node: Node dictionary with cpuinfo. - :param fdio_release: VPP release number - :param centos_version: Ubuntu Version + :param branch: The branch name release or master :type node: dict - :type fdio_release: string - :type centos_version: string + :type branch: string """ + # Be sure the correct system packages are installed + cmd = 'yum -y update' + (ret, stdout, stderr) = self.exec_command(cmd) + if ret != 0: + logging.debug('{} failed on node {} {}'.format( + cmd, + node['host'], + stderr)) + + cmd = 'yum -y install pygpgme yum-utils' + (ret, stdout, stderr) = self.exec_command(cmd) + if ret != 0: + logging.debug('{} failed on node {} {}'.format( + cmd, + node['host'], + stderr)) + # Modify the sources list sfile = '/etc/yum.repos.d/fdio-release.repo' @@ -199,135 +195,129 @@ class VPPUtil(object): node['host'], stderr)) - reps = '[fdio-stable-{}]\n'.format(fdio_release) - reps += 'name=fd.io stable/{} branch latest merge\n'.format(fdio_release) - reps += 'baseurl=https://nexus.fd.io/content/repositories/fd.io.stable.{}.{}/\n'.\ - format(fdio_release, centos_version) - reps += 'enabled=1\n' - reps += 'gpgcheck=0' + # Get the file contents + + reps = '\n'.join([ + '[fdio_{}]'.format(branch), + 'name=fdio_{}'.format(branch), + 'baseurl=https://packagecloud.io/fdio/{}/el/7/$basearch'.format( + branch), + 'repo_gpgcheck=1', + 'gpgcheck=0', + 'enabled=1', + 'gpgkey=https://packagecloud.io/fdio/{}/gpgkey'.format(branch), + 'sslverify=1', + 'sslcacert=/etc/pki/tls/certs/ca-bundle.crt', + 'metadata_expire=300\n', + '[fdio_{}-source]'.format(branch), + 'name=fdio_release-{}'.format(branch), + 'baseurl=https://packagecloud.io/fdio/{}/el/7/SRPMS'.format( + branch), + 'repo_gpgcheck=1', + 'gpgcheck=0', + 'enabled=1', + 'gpgkey=https://packagecloud.io/fdio/{}/gpgkey'.format(branch), + 'sslverify =1', + 'sslcacert=/etc/pki/tls/certs/ca-bundle.crt', + 'metadata_expire=300\n' + ]) + with open(sfile, 'w') as sfd: + sfd.write(reps) + sfd.close() + + # Update the fdio repo + cmd = 'yum clean all' + (ret, stdout, stderr) = self.exec_command(cmd) + if ret != 0: + logging.debug('{} failed on node {} {}'.format( + cmd, + node['host'], + stderr)) - cmd = 'echo "{0}" | sudo tee {1}'.format(reps, sfile) + cmd = "yum -q makecache -y --disablerepo='*' " \ + "--enablerepo='fdio_{}'".format(branch) (ret, stdout, stderr) = self.exec_command(cmd) if ret != 0: - raise RuntimeError('{} failed on node {} {}'.format( + logging.debug('{} failed on node {} {}'.format( cmd, node['host'], stderr)) - # Install the packages - - self._install_vpp_pkg_centos(node, 'vpp-lib') - self._install_vpp_pkg_centos(node, 'vpp') - self._install_vpp_pkg_centos(node, 'vpp-plugins') - # jadfix Check with Ole - # self._install_vpp_pkg_centos(node, 'vpp-dpdk-devel') - self._install_vpp_pkg_centos(node, 'vpp-api-python') - self._install_vpp_pkg_centos(node, 'vpp-api-java') - self._install_vpp_pkg_centos(node, 'vpp-api-lua') - self._install_vpp_pkg_centos(node, 'vpp-devel') + # Get the package list + pkgstr = '' + for ps in centos_pkgs[branch]: + pkgstr += ps + ' ' - def install_vpp(self, node): + cmd = 'yum -y install {}'.format(pkgstr) + (ret, stdout, stderr) = self.exec_command(cmd) + if ret != 0: + raise RuntimeError('{} failed on node {} {} {}'.format( + cmd, node['host'], stdout, stderr)) + + def install_vpp(self, node, branch): """ Install the VPP packages :param node: Node dictionary with cpuinfo. + :param branch: The branch name :type node: dict + :type branch: string + """ distro = self.get_linux_distro() + logging.info(" {}".format(distro[0])) if distro[0] == 'Ubuntu': - self._install_vpp_ubuntu(node) + logging.info("Install Ubuntu") + self._install_vpp_ubuntu(node, branch, ubuntu_version=distro[2]) elif distro[0] == 'CentOS Linux': logging.info("Install CentOS") - self._install_vpp_centos(node) + self._install_vpp_centos(node, branch) else: - return + logging.info("Install CentOS (default)") + self._install_vpp_centos(node, branch) + return - def _uninstall_vpp_pkg_ubuntu(self, node, pkg): + def _uninstall_vpp_ubuntu(self, node): """ Uninstall the VPP packages - :param node: Node dictionary - :param pkg: The vpp packages + :param node: Node dictionary with cpuinfo. :type node: dict - :type pkg: string """ - cmd = 'dpkg --purge {}'.format(pkg) - (ret, stdout, stderr) = self.exec_command(cmd) - if ret != 0: - raise RuntimeError('{} failed on node {} {} {}'.format( - cmd, node['host'], stdout, stderr)) - def _uninstall_vpp_pkg_centos(self, node, pkg): - """ - Uninstall the VPP packages + # get the package list + pkgstr = '' + pkgs = self.get_installed_vpp_pkgs() + for pkg in pkgs: + pkgname = pkg['name'] + pkgstr += pkgname + ' ' - :param node: Node dictionary - :param pkg: The vpp packages - :type node: dict - :type pkg: string - """ - cmd = 'yum -y remove {}'.format(pkg) + cmd = 'dpkg --purge {}'.format(pkgstr) (ret, stdout, stderr) = self.exec_command(cmd) if ret != 0: raise RuntimeError('{} failed on node {} {} {}'.format( cmd, node['host'], stdout, stderr)) - def _uninstall_vpp_ubuntu(self, node): - """ - Uninstall the VPP packages - - :param node: Node dictionary with cpuinfo. - :type node: dict - """ - pkgs = self.get_installed_vpp_pkgs() - - if len(pkgs) > 0: - if 'version' in pkgs[0]: - logging.info("Uninstall Ubuntu Packages") - self._uninstall_vpp_pkg_ubuntu(node, 'vpp-api-python') - self._uninstall_vpp_pkg_ubuntu(node, 'vpp-api-java') - self._uninstall_vpp_pkg_ubuntu(node, 'vpp-api-lua') - self._uninstall_vpp_pkg_ubuntu(node, 'vpp-plugins') - self._uninstall_vpp_pkg_ubuntu(node, 'vpp-dpdk-dev') - self._uninstall_vpp_pkg_ubuntu(node, 'vpp-dpdk-dkms') - self._uninstall_vpp_pkg_ubuntu(node, 'vpp-dev') - self._uninstall_vpp_pkg_ubuntu(node, 'vpp-dbg') - self._uninstall_vpp_pkg_ubuntu(node, 'vpp') - self._uninstall_vpp_pkg_ubuntu(node, 'vpp-lib') - else: - logging.info("Uninstall locally installed Ubuntu Packages") - for pkg in pkgs: - self._uninstall_vpp_pkg_ubuntu(node, pkg['name']) - else: - logging.error("There are no Ubuntu packages installed") - def _uninstall_vpp_centos(self, node): """ Uninstall the VPP packages :param node: Node dictionary with cpuinfo. :type node: dict - """ + """ + pkgstr = '' pkgs = self.get_installed_vpp_pkgs() + for pkg in pkgs: + pkgname = pkg['name'] + pkgstr += pkgname + ' ' - if len(pkgs) > 0: - if 'version' in pkgs[0]: - logging.info("Uninstall CentOS Packages") - self._uninstall_vpp_pkg_centos(node, 'vpp-api-python') - self._uninstall_vpp_pkg_centos(node, 'vpp-api-java') - self._uninstall_vpp_pkg_centos(node, 'vpp-api-lua') - self._uninstall_vpp_pkg_centos(node, 'vpp-plugins') - self._uninstall_vpp_pkg_centos(node, 'vpp-dpdk-devel') - self._uninstall_vpp_pkg_centos(node, 'vpp-devel') - self._uninstall_vpp_pkg_centos(node, 'vpp') - self._uninstall_vpp_pkg_centos(node, 'vpp-lib') - else: - logging.info("Uninstall locally installed CentOS Packages") - for pkg in pkgs: - self._uninstall_vpp_pkg_centos(node, pkg['name']) - else: - logging.error("There are no CentOS packages installed") + logging.info("Uninstalling {}".format(pkgstr)) + cmd = 'yum -y remove {}'.format(pkgstr) + (ret, stdout, stderr) = self.exec_command(cmd) + if ret != 0: + raise RuntimeError('{} failed on node {} {} {}'.format( + cmd, node['host'], stdout, stderr)) def uninstall_vpp(self, node): """ @@ -336,13 +326,19 @@ class VPPUtil(object): :param node: Node dictionary with cpuinfo. :type node: dict """ + + # First stop VPP + self.stop(node) distro = self.get_linux_distro() if distro[0] == 'Ubuntu': + logging.info("Uninstall Ubuntu") self._uninstall_vpp_ubuntu(node) elif distro[0] == 'CentOS Linux': logging.info("Uninstall CentOS") self._uninstall_vpp_centos(node) else: + logging.info("Uninstall CentOS (Default)") + self._uninstall_vpp_centos(node) return def show_vpp_settings(self, *additional_cmds): @@ -372,6 +368,64 @@ class VPPUtil(object): for _, value in def_setting_tb_displayed.items(): self.exec_command('vppctl sh {}'.format(value)) + @staticmethod + def get_vms(node): + """ + Get a list of VMs that are connected to VPP interfaces + + :param node: VPP node. + :type node: dict + :returns: Dictionary containing a list of VMs and the interfaces + that are connected to VPP + :rtype: dictionary + """ + + vmdict = {} + + print("Need to implement get vms") + + return vmdict + + @staticmethod + def get_int_ip(node): + """ + Get the VPP interfaces and IP addresses + + :param node: VPP node. + :type node: dict + :returns: Dictionary containing VPP interfaces and IP addresses + :rtype: dictionary + """ + interfaces = {} + cmd = 'vppctl show int addr' + (ret, stdout, stderr) = VPPUtil.exec_command(cmd) + if ret != 0: + return interfaces + + lines = stdout.split('\n') + if len(lines[0]) != 0: + if lines[0].split(' ')[0] == 'FileNotFoundError': + return interfaces + + name = '' + for line in lines: + if len(line) == 0: + continue + + # If the first character is not whitespace + # create a new interface + if len(re.findall(r'\s', line[0])) == 0: + spl = line.split() + name = spl[0] + if name == 'local0': + continue + interfaces[name] = {} + interfaces[name]['state'] = spl[1].lstrip('(').rstrip('):\r') + else: + interfaces[name]['address'] = line.lstrip(' ').rstrip('\r') + + return interfaces + @staticmethod def get_hardware(node): """ @@ -380,7 +434,7 @@ class VPPUtil(object): :param node: VPP node. :type node: dict - :returns: Dictionary containing improtant VPP information + :returns: Dictionary containing VPP hardware information :rtype: dictionary """ @@ -391,17 +445,17 @@ class VPPUtil(object): return interfaces lines = stdout.split('\n') - if len(lines[0]) is not 0: + if len(lines[0]) != 0: if lines[0].split(' ')[0] == 'FileNotFoundError': return interfaces for line in lines: - if len(line) is 0: + if len(line) == 0: continue # If the first character is not whitespace # create a new interface - if len(re.findall(r'\s', line[0])) is 0: + if len(re.findall(r'\s', line[0])) == 0: spl = line.split() name = spl[0] interfaces[name] = {} @@ -421,19 +475,26 @@ class VPPUtil(object): interfaces[name]['carrier'] = spl[1] # Socket - rfall = re.findall(r'cpu socket', line) + spl = '' + rfall = re.findall(r'numa \d+', line) if rfall: - spl = line.split('cpu socket ') - interfaces[name]['cpu socket'] = spl[1] + spl = rfall[0].split() + interfaces[name]['numa'] = rfall[0].split()[1] # Queues and Descriptors - rfall = re.findall(r'rx queues', line) + rfall = re.findall(r'rx\: queues \d+', line) if rfall: - spl = line.split(',') - interfaces[name]['rx queues'] = spl[0].lstrip(' ').split(' ')[2] - interfaces[name]['rx descs'] = spl[1].split(' ')[3] - interfaces[name]['tx queues'] = spl[2].split(' ')[3] - interfaces[name]['tx descs'] = spl[3].split(' ')[3] + interfaces[name]['rx queues'] = rfall[0].split()[2] + rdesc = re.findall(r'desc \d+', line) + if rdesc: + interfaces[name]['rx descs'] = rdesc[0].split()[1] + + rfall = re.findall(r'tx\: queues \d+', line) + if rfall: + interfaces[name]['tx queues'] = rfall[0].split()[2] + rdesc = re.findall(r'desc \d+', line) + if rdesc: + interfaces[name]['tx descs'] = rdesc[0].split()[1] return interfaces @@ -507,6 +568,7 @@ class VPPUtil(object): elif distro[0] == 'CentOS Linux': pkgs = self._get_installed_vpp_pkgs_centos() else: + pkgs = self._get_installed_vpp_pkgs_centos() return [] return pkgs @@ -515,8 +577,10 @@ class VPPUtil(object): def get_interfaces_numa_node(node, *iface_keys): """Get numa node on which are located most of the interfaces. - Return numa node with highest count of interfaces provided as arguments. - Return 0 if the interface does not have numa_node information available. + Return numa node with highest count of interfaces provided as + arguments. + Return 0 if the interface does not have numa_node information + available. If all interfaces have unknown location (-1), then return 0. If most of interfaces have unknown location (-1), but there are some interfaces with known location, then return the second most @@ -543,6 +607,23 @@ class VPPUtil(object): return 0 + @staticmethod + def restart(node): + """ + + Starts vpp for a given node + + :param node: VPP node. + :type node: dict + """ + + cmd = 'service vpp restart' + (ret, stdout, stderr) = VPPUtil.exec_command(cmd) + if ret != 0: + raise RuntimeError('{} failed on node {} {} {}'. + format(cmd, node['host'], + stdout, stderr)) + @staticmethod def start(node): """ @@ -573,10 +654,11 @@ class VPPUtil(object): cmd = 'service vpp stop' (ret, stdout, stderr) = VPPUtil.exec_command(cmd) if ret != 0: - raise RuntimeError('{} failed on node {} {} {}'. - format(cmd, node['host'], - stdout, stderr)) + logging.debug('{} failed on node {} {} {}'. + format(cmd, node['host'], + stdout, stderr)) + # noinspection RegExpRedundantEscape @staticmethod def status(node): """ @@ -624,13 +706,14 @@ class VPPUtil(object): :rtype: list """ - distro = platform.linux_distribution() - if distro[0] == 'Ubuntu' or \ - distro[0] == 'CentOS Linux' or \ - distro[:26] == 'Linux Distribution Red Hat': - return distro + dist = distro.linux_distribution() + if dist[0] == 'Ubuntu' or \ + dist[0] == 'CentOS Linux' or \ + dist[:7] == 'Red Hat': + return dist else: - raise RuntimeError('Linux Distribution {} is not supported'.format(distro[0])) + raise RuntimeError( + 'Linux Distribution {} is not supported'.format(dist[0])) @staticmethod def version(): @@ -649,14 +732,62 @@ class VPPUtil(object): return version lines = stdout.split('\n') - if len(lines[0]) is not 0: + if len(lines[0]) != 0: if lines[0].split(' ')[0] == 'FileNotFoundError': return version for line in lines: - if len(line) is 0: + if len(line) == 0: continue dct = line.split(':') version[dct[0]] = dct[1].lstrip(' ') return version + + @staticmethod + def show_bridge(node): + """ + Shows the current bridge configuration + + :param node: VPP node. + :type node: dict + :returns: A list of interfaces + """ + + ifaces = [] + cmd = 'vppctl show bridge' + (ret, stdout, stderr) = VPPUtil.exec_command(cmd) + if ret != 0: + raise RuntimeError('{} failed on node {} {} {}'. + format(cmd, node['host'], + stdout, stderr)) + lines = stdout.split('\r\n') + bridges = [] + for line in lines: + if line == 'no bridge-domains in use': + print(line) + return ifaces + if len(line) == 0: + continue + + lspl = line.lstrip(' ').split() + if lspl[0] != 'BD-ID': + bridges.append(lspl[0]) + + for bridge in bridges: + cmd = 'vppctl show bridge {} detail'.format(bridge) + (ret, stdout, stderr) = VPPUtil.exec_command(cmd) + if ret != 0: + raise RuntimeError('{} failed on node {} {} {}'. + format(cmd, node['host'], + stdout, stderr)) + + lines = stdout.split('\r\n') + for line in lines: + iface = re.findall(r'[a-zA-z]+\d+/\d+/\d+', line) + if len(iface): + ifcidx = {'name': iface[0], 'index': line.split()[1]} + ifaces.append(ifcidx) + + print(stdout) + return ifaces