resources/tools/virl/bin/start-testcase

   1 #!/usr/bin/python
   2
   3 # Copyright (c) 2016 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 """This script is handling starting of VIRL simulations."""
  17
  18 __author__ = 'ckoester@cisco.com'
  19
  20 import argparse
  21 import netifaces
  22 import os
  23 import paramiko
  24 import random
  25 import re
  26 import shutil
  27 import sys
  28 import tempfile
  29 import time
  30
  31 import requests
  32
  33 IPS_PER_SIMULATION = 5
  34
  35
  36 def indent(lines, amount, fillchar=' '):
  37     """Indent the string by amount of fill chars.
  38
  39     :param lines: String to indent.
  40     :param amount: Number of fill chars.
  41     :param fillchar: Filling character.
  42     :type lines: str
  43     :type amount: int
  44     :type fillchar: str
  45     :returns: Indented string.
  46     :rtype: str
  47     """
  48     padding = amount * fillchar
  49     return padding + ('\n'+padding).join(lines.split('\n'))
  50
  51
  52 def print_to_stderr(msg, end='\n'):
  53     """Writes any text to stderr.
  54
  55     :param msg: Message to print.
  56     :param end: By default print new line at the end.
  57     :type msg: str
  58     :type end: str
  59     """
  60     try:
  61         sys.stderr.write(str(msg) + end)
  62     except ValueError:
  63         pass
  64
  65
  66 def get_assigned_interfaces(args, network="flat"):
  67     """Retrieve assigned interfaces in openstack network.
  68
  69     :param args: Command line params.
  70     :param network: Openstack network.
  71     :type args: ArgumentParser
  72     :type network: str
  73     :returns: Assigned interfaces.
  74     :rtype: list
  75     :raises RuntimeError: If response is not 200.
  76     """
  77     req = requests.get('http://{}/openstack/rest/ports/{}'
  78                        .format(args.virl_ip, network),
  79                        auth=(args.username, args.password))
  80     if req.status_code == 200:
  81         return req.json()
  82     else:
  83         raise RuntimeError("ERROR: Retrieving ports in use - "
  84                            "Status other than 200 HTTP OK:\n{}"
  85                            .format(req.content))
  86
  87
  88 def get_assigned_interfaces_count(args, network="flat"):
  89     """Count assigned interfaces in openstack network.
  90
  91     :param args: Command line params.
  92     :param network: Openstack network.
  93     :type args: ArgumentParser
  94     :type network: str
  95     :returns: Assigned interfaces count.
  96     :rtype: int
  97     """
  98     return len(get_assigned_interfaces(args, network=network))
  99
 100
 101 def check_ip_addresses(args):
 102     """Check IP address availability.
 103
 104     :param args: Command line params.
 105     :type args: ArgumentParser
 106     :raises RuntimeError: If not enough free addresses available.
 107     """
 108     for i in range(args.wait_count):
 109         if (args.quota -
 110                 get_assigned_interfaces_count(args) >= IPS_PER_SIMULATION):
 111             break
 112         if args.verbosity >= 2:
 113             print_to_stderr("DEBUG: - Attempt {} out of {}, waiting for free "
 114                             "IP addresses".format(i, args.wait_count))
 115         # Wait random amount of time within range 1-3 minutes
 116         time.sleep(random.randint(60,180))
 117     else:
 118         raise RuntimeError("ERROR: Not enough IP addresses to run simulation")
 119
 120
 121 def check_virl_resources(args):
 122     """Check virl resources availability.
 123
 124     :param args: Command line params.
 125     :type args: ArgumentParser
 126     """
 127     check_ip_addresses(args)
 128
 129 #
 130 # FIXME: Right now, this is really coded like a shell script, as one big
 131 # function executed in sequence. This should be broken down into multiple
 132 # functions.
 133 #
 134
 135
 136 def main():
 137     """ Main function."""
 138     #
 139     # Get our default interface IP address. This will become the default
 140     # value for the "NFS Server IP" option.
 141     #
 142     gws = netifaces.gateways()
 143     addrs = netifaces.ifaddresses(gws['default'][netifaces.AF_INET][1])
 144     default_addr = addrs[netifaces.AF_INET][0]['addr']
 145
 146     #
 147     # Verify CLI parameters and try to download our VPP image into a temporary
 148     # file first
 149     #
 150     parser = argparse.ArgumentParser()
 151     parser.add_argument("topology", help="the base topology to be started")
 152     parser.add_argument("packages", help="Path to the VPP .deb(s) or .rpm(s) " +
 153                         "that is/are to be installed", nargs='+')
 154     parser.add_argument("-c", "--copy", help="Copy the VPP packages, " +
 155                         "leaving the originals in place. Default is to " +
 156                         "move them.", action='store_true')
 157     parser.add_argument("-k", "--keep", help="Keep (do not delete) the " +
 158                         "simulation in case of error", action='store_true')
 159     parser.add_argument("-v", "--verbosity", action="count", default=0)
 160     parser.add_argument("-nip", "--nfs-server-ip", help="NFS server (our) IP " +
 161                         "default is derived from routing table: " +
 162                         "{}".format(default_addr), default=default_addr)
 163     parser.add_argument("-ns", "--nfs-scratch-directory",
 164                         help="Server location for NFS scratch directory",
 165                         default="/nfs/scratch")
 166     parser.add_argument("-nc", "--nfs-common-directory",
 167                         help="Server location for NFS common (read-only) " +
 168                         "directory", default="/nfs/common")
 169     parser.add_argument("-wc", "--wait-count",
 170                         help="number of intervals to wait for simulation to " +
 171                         "be ready", type=int, default=48)
 172     parser.add_argument("-wt", "--wait-time",
 173                         help="length of a single interval to wait for " +
 174                         "simulation to be ready", type=int, default=5)
 175     parser.add_argument("-vip", "--virl-ip",
 176                         help="VIRL IP and Port (e.g. 127.0.0.1:19399)",
 177                         default="127.0.0.1:19399")
 178     parser.add_argument("-u", "--username", help="VIRL username",
 179                         default="tb4-virl")
 180     parser.add_argument("-au", "--admin-username", help="VIRL admin username",
 181                         default="uwmadmin")
 182     parser.add_argument("-p", "--password", help="VIRL password",
 183                         default="Cisco1234")
 184     parser.add_argument("-su", "--ssh-user", help="SSH username",
 185                         default="cisco")
 186     parser.add_argument("-e", "--expiry", help="Simulation expiry",
 187                         default="120")
 188     parser.add_argument("-spr", "--ssh-privkey", help="SSH private keyfile",
 189                         default="/home/jenkins-in/.ssh/id_rsa_virl")
 190     parser.add_argument("-spu", "--ssh-pubkey", help="SSH public keyfile",
 191                         default="/home/jenkins-in/.ssh/id_rsa_virl.pub")
 192     parser.add_argument("-r", "--release", help="VM disk image/release " +
 193                         "(ex. \"csit-ubuntu-16.04.1_2016-12-19_1.6\")",
 194                         default="csit-ubuntu-16.04.1_2016-12-19_1.6")
 195     parser.add_argument("--topology-directory", help="Topology directory",
 196                         default="/home/jenkins-in/testcase-infra/topologies")
 197     parser.add_argument("-q", "--quota",
 198                         help="VIRL quota for max number of allowed IPs",
 199                         type=int, default=74)
 200     parser.add_argument("-si", "--skip-install", help="Skip VPP installation",
 201                         action='store_true')
 202
 203     args = parser.parse_args()
 204
 205     #
 206     # Check if topology and template exist
 207     #
 208     if args.verbosity >= 2:
 209         print_to_stderr("DEBUG: Running with topology {}"
 210                         .format(args.topology))
 211
 212     topology_virl_filename = os.path.join(args.topology_directory,
 213                                           args.topology + ".virl")
 214     topology_yaml_filename = os.path.join(args.topology_directory,
 215                                           args.topology + ".yaml")
 216
 217     if not os.path.isfile(topology_virl_filename):
 218         print_to_stderr("ERROR: Topology VIRL file {} does not exist"
 219                         .format(topology_virl_filename))
 220         sys.exit(1)
 221     if not os.path.isfile(topology_yaml_filename):
 222         print_to_stderr("ERROR: Topology YAML file {} does not exist"
 223                         .format(topology_yaml_filename))
 224         sys.exit(1)
 225
 226     #
 227     # Check if VPP package exists
 228     #
 229     if not args.skip_install:
 230         for package in args.packages:
 231             if args.verbosity >= 2:
 232                 print_to_stderr("DEBUG: Checking if file {} exists"
 233                                 .format(package))
 234             if not os.path.isfile(package):
 235                 print_to_stderr("ERROR: Required package {} does not exist"
 236                                 .format(package))
 237                 sys.exit(1)
 238
 239     #
 240     # Start VIRL topology
 241     #
 242     if args.verbosity >= 1:
 243         print_to_stderr("DEBUG: Starting VIRL topology")
 244     temp_handle, temp_topology = tempfile.mkstemp()
 245     with open(args.ssh_pubkey, 'r') as pubkey_file:
 246         pub_key = pubkey_file.read().replace('\n', '')
 247     with open(temp_topology, 'w') as new_file, \
 248         open(topology_virl_filename, 'r') as old_file:
 249         for line in old_file:
 250             line = line.replace("  - VIRL-USER-SSH-PUBLIC-KEY", "  - "+pub_key)
 251             line = line.replace("$$NFS_SERVER_SCRATCH$$",
 252                                 args.nfs_server_ip+":"+args.nfs_scratch_directory)
 253             line = line.replace("$$NFS_SERVER_COMMON$$",
 254                                 args.nfs_server_ip+":"+args.nfs_common_directory)
 255             line = line.replace("$$VM_IMAGE$$", "server-"+args.release)
 256             new_file.write(line)
 257     os.close(temp_handle)
 258
 259     try:
 260         data = open(temp_topology, 'rb')
 261         check_virl_resources(args)
 262         req = requests.post('http://' + args.virl_ip + '/simengine/rest/launch',
 263                             auth=(args.username, args.password),
 264                             data=data)
 265         if args.verbosity >= 2:
 266             print_to_stderr("DEBUG: - Request URL {}"
 267                             .format(req.url))
 268             print_to_stderr("{}"
 269                             .format(req.text))
 270             print_to_stderr("DEBUG: - Response Code {}"
 271                             .format(req.status_code))
 272         new_file.close()
 273         if req.status_code != 200:
 274             raise RuntimeError("ERROR: Launching VIRL simulation - "
 275                                "Status other than 200 HTTP OK:\n{}"
 276                                .format(req.content))
 277     except (requests.exceptions.RequestException,
 278             RuntimeError) as ex_error:
 279         print_to_stderr(ex_error)
 280         os.remove(temp_topology)
 281         sys.exit(1)
 282
 283     # If we got here, we had a good response. The response content is the
 284     # session ID.
 285     session_id = req.content
 286     if args.verbosity >= 1:
 287         print_to_stderr("DEBUG: VIRL simulation session-id: {}"
 288                         .format(session_id))
 289
 290     # Set session expiry to autokill sessions if not done from jenkins
 291     if not args.keep:
 292         if args.verbosity >= 1:
 293             print_to_stderr("DEBUG: Setting expire for session-id: {}"
 294                             .format(session_id))
 295         try:
 296             req = requests.put('http://' + args.virl_ip +
 297                                '/simengine/rest/admin-update/' + session_id +
 298                                '/expiry',
 299                                auth=(args.admin_username, args.password),
 300                                params={'user': args.username,
 301                                        'expires': args.expiry})
 302             if args.verbosity >= 2:
 303                 print_to_stderr("DEBUG: - Request URL {}"
 304                                 .format(req.url))
 305                 print_to_stderr("{}"
 306                                 .format(req.text))
 307                 print_to_stderr("DEBUG: - Response Code {}"
 308                                 .format(req.status_code))
 309             if req.status_code != 200:
 310                 raise RuntimeError("ERROR: Setting expiry to simulation - "
 311                                    "Status other than 200 HTTP OK:\n{}"
 312                                    .format(req.content))
 313         except (requests.exceptions.RequestException,
 314                 RuntimeError) as ex_error:
 315             print_to_stderr(ex_error)
 316             req = requests.get('http://' + args.virl_ip +
 317                                '/simengine/rest/stop/' + session_id,
 318                                auth=(args.username, args.password))
 319             os.remove(temp_topology)
 320             print "{}".format(session_id)
 321             sys.exit(1)
 322
 323     #
 324     # Create simulation scratch directory. Move topology file into that
 325     # directory. Copy or move debian packages into that directory.
 326     #
 327     scratch_directory = os.path.join(args.nfs_scratch_directory, session_id)
 328     os.mkdir(scratch_directory)
 329     shutil.move(temp_topology, os.path.join(scratch_directory,
 330                                             "virl_topology.virl"))
 331     os.mkdir(os.path.join(scratch_directory, "vpp"))
 332     for package in args.packages:
 333         if args.copy:
 334             shutil.copy(package, os.path.join(scratch_directory, "vpp",
 335                                               os.path.basename(package)))
 336         else:
 337             shutil.move(package, os.path.join(scratch_directory, "vpp",
 338                                               os.path.basename(package)))
 339
 340     #
 341     # Wait for simulation to become active
 342     #
 343     if args.verbosity >= 1:
 344         print_to_stderr("DEBUG: Waiting for simulation to become active")
 345
 346     sim_is_started = False
 347     nodelist = []
 348
 349     count = args.wait_count
 350     while (count > 0) and not sim_is_started:
 351         time.sleep(args.wait_time)
 352         count -= 1
 353
 354         req = requests.get('http://' + args.virl_ip + '/simengine/rest/nodes/' +
 355                            session_id, auth=(args.username, args.password))
 356         data = req.json()
 357
 358         active = 0
 359         total = 0
 360
 361         # Flush the node list every time, keep the last one
 362         nodelist = []
 363
 364         # Hosts are the keys of the inner dictionary
 365         for key in data[session_id].keys():
 366             if data[session_id][key]['management-proxy'] == "self":
 367                 continue
 368             nodelist.append(key)
 369             total += 1
 370             if data[session_id][key]['state'] == "ACTIVE":
 371                 active += 1
 372         if args.verbosity >= 2:
 373             print_to_stderr("DEBUG: - Attempt {} out of {}, total {} hosts, "
 374                             "{} active".format(args.wait_count-count,
 375                                                args.wait_count, total, active))
 376         if active == total:
 377             sim_is_started = True
 378
 379     if not sim_is_started:
 380         print_to_stderr("ERROR: Simulation nodes never changed to ACTIVE state")
 381         print_to_stderr("Last VIRL response:")
 382         print_to_stderr(data)
 383         if not args.keep:
 384             req = requests.get('http://' + args.virl_ip +
 385                                '/simengine/rest/stop/' + session_id,
 386                                auth=(args.username, args.password))
 387             try:
 388                 shutil.rmtree(scratch_directory)
 389             except shutil.Error:
 390                 print_to_stderr("ERROR: Removing scratch directory")
 391             print "{}".format(session_id)
 392         sys.exit(1)
 393
 394     if args.verbosity >= 2:
 395         print_to_stderr("DEBUG: Nodes: {}"
 396                         .format(", ".join(nodelist)))
 397
 398     #
 399     # Fetch simulation's IPs and create files
 400     # (ansible hosts file, topology YAML file)
 401     #
 402     try:
 403         req = requests.get('http://' + args.virl_ip +
 404                            '/simengine/rest/interfaces/' + session_id,
 405                            auth=(args.username, args.password),
 406                            params={'fetch-state': '1'})
 407         if args.verbosity >= 2:
 408             print_to_stderr("DEBUG: - Request URL {}"
 409                             .format(req.url))
 410             print_to_stderr("DEBUG: - Request Text")
 411             print_to_stderr("{}".format(req.text))
 412             print_to_stderr("DEBUG: - Response Code {}"
 413                             .format(req.status_code))
 414         if req.status_code != 200:
 415             raise RuntimeError("ERROR:Fetching IP's of simulation - "
 416                                "Status other than 200 HTTP OK:\n{}"
 417                                .format(req.content))
 418     except (requests.exceptions.RequestException,
 419             RuntimeError) as ex_error:
 420         print_to_stderr(ex_error)
 421         if not args.keep:
 422             req = requests.get('http://' + args.virl_ip +
 423                                '/simengine/rest/stop/' + session_id,
 424                                auth=(args.username, args.password))
 425             try:
 426                 shutil.rmtree(scratch_directory)
 427             except shutil.Error:
 428                 print_to_stderr("ERROR: Removing scratch directory")
 429             print "{}".format(session_id)
 430         sys.exit(1)
 431     data = req.json()
 432
 433     # Populate node addresses
 434     nodeaddrs = {}
 435     topology = {}
 436     for key in nodelist:
 437         nodetype = re.split('[0-9]', key)[0]
 438         if not nodetype in nodeaddrs:
 439             nodeaddrs[nodetype] = {}
 440         nodeaddrs[nodetype][key] = re.split('\\/', \
 441             data[session_id][key]['management']['ip-address'])[0]
 442         if args.verbosity >= 2:
 443             print_to_stderr("DEBUG: Node {} is of type {} and has mgmt IP {}"
 444                             .format(key, nodetype, nodeaddrs[nodetype][key]))
 445
 446         topology[key] = {}
 447         for key2 in data[session_id][key]:
 448             topology[key]["nic-"+key2] = data[session_id][key][key2]
 449             if 'ip-address' in topology[key]["nic-"+key2]:
 450                 if topology[key]["nic-"+key2]['ip-address'] is not None:
 451                     topology[key]["nic-"+key2]['ip-addr'] = re.split('\\/', \
 452                         topology[key]["nic-"+key2]['ip-address'])[0]
 453
 454     # Write ansible file
 455     ansiblehosts = open(os.path.join(scratch_directory, 'ansible-hosts'), 'w')
 456     for key1 in nodeaddrs:
 457         ansiblehosts.write("[{}]\n".format(key1))
 458         for key2 in nodeaddrs[key1]:
 459             ansiblehosts.write("{} hostname={}\n".format(nodeaddrs[key1][key2],
 460                                                          key2))
 461     ansiblehosts.close()
 462
 463     # Process topology YAML template
 464     with open(args.ssh_privkey, 'r') as privkey_file:
 465         priv_key = indent(privkey_file.read(), 6)
 466
 467     with open(os.path.join(scratch_directory, "topology.yaml"), 'w') as \
 468         new_file, open(topology_yaml_filename, 'r') as old_file:
 469         for line in old_file:
 470             new_file.write(line.format(priv_key=priv_key, topology=topology))
 471
 472     #
 473     # Wait for hosts to become reachable over SSH
 474     #
 475     if args.verbosity >= 1:
 476         print_to_stderr("DEBUG: Waiting for hosts to become reachable over SSH")
 477
 478     missing = -1
 479     count = args.wait_count
 480     while (count > 0) and missing != 0:
 481         time.sleep(args.wait_time)
 482         count -= 1
 483
 484         missing = 0
 485         for key in nodelist:
 486             if not os.path.exists(os.path.join(scratch_directory, key)):
 487                 missing += 1
 488         if args.verbosity >= 2:
 489             print_to_stderr("DEBUG: Attempt {} out of {}, waiting for {} hosts"
 490                             .format(args.wait_count-count, args.wait_count,
 491                                     missing))
 492
 493     if missing != 0:
 494         print_to_stderr("ERROR: Simulation started OK but {} hosts never "
 495                         "mounted their NFS directory".format(missing))
 496         if not args.keep:
 497             req = requests.get('http://' + args.virl_ip +
 498                                '/simengine/rest/stop/' + session_id,
 499                                auth=(args.username, args.password))
 500             try:
 501                 shutil.rmtree(scratch_directory)
 502             except shutil.Error:
 503                 print_to_stderr("ERROR: Removing scratch directory")
 504             print "{}".format(session_id)
 505         sys.exit(1)
 506
 507     #
 508     # Upgrade VPP
 509     #
 510     if not args.skip_install:
 511         if args.verbosity >= 1:
 512             print_to_stderr("DEBUG: Uprading VPP")
 513
 514         for key1 in nodeaddrs:
 515             if not key1 == 'tg':
 516                 for key2 in nodeaddrs[key1]:
 517                     ipaddr = nodeaddrs[key1][key2]
 518                     if args.verbosity >= 2:
 519                         print_to_stderr("DEBUG: Upgrading VPP on node {}"
 520                                         .format(ipaddr))
 521                     paramiko.util.log_to_file(os.path.join(scratch_directory,
 522                                                            "ssh.log"))
 523                     client = paramiko.SSHClient()
 524                     client.load_system_host_keys()
 525                     client.load_host_keys("/dev/null")
 526                     client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 527                     client.connect(ipaddr, username=args.ssh_user,
 528                                    key_filename=args.ssh_privkey)
 529                     if 'centos' in args.topology:
 530                         if args.verbosity >= 1:
 531                             print_to_stderr("DEBUG: Installing RPM packages")
 532                         vpp_install_command = 'sudo rpm -ivh /scratch/vpp/*.rpm'
 533                     elif 'trusty' in args.topology or 'xenial' in args.topology:
 534                         if args.verbosity >= 1:
 535                             print_to_stderr("DEBUG: Installing DEB packages")
 536                         vpp_install_command = 'sudo dpkg -i --force-all ' \
 537                                               '/scratch/vpp/*.deb'
 538                     else:
 539                         print_to_stderr("ERROR: Unsupported OS requested: {}"
 540                                          .format(args.topology))
 541                         vpp_install_command = ''
 542                     _, stdout, stderr = \
 543                         client.exec_command(vpp_install_command)
 544                     c_stdout = stdout.read()
 545                     c_stderr = stderr.read()
 546                     if args.verbosity >= 2:
 547                         print_to_stderr("DEBUG: Command output was:")
 548                         print_to_stderr(c_stdout)
 549                         print_to_stderr("DEBUG: Command stderr was:")
 550                         print_to_stderr(c_stderr)
 551
 552     #
 553     # Write a file with timestamp to scratch directory. We can use this to track
 554     # how long a simulation has been running.
 555     #
 556     with open(os.path.join(scratch_directory, 'start_time'), 'a') as \
 557         timestampfile:
 558         timestampfile.write('{}\n'.format(int(time.time())))
 559
 560     #
 561     # Declare victory
 562     #
 563     if args.verbosity >= 1:
 564         print_to_stderr("SESSION ID: {}".format(session_id))
 565
 566     print "{}".format(session_id)
 567
 568 if __name__ == "__main__":
 569     sys.exit(main())