resources/tools/virl/bin/start-testcase

   1 #!/usr/bin/python
   2
   3 # Copyright (c) 2016 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 """This script is handling starting of VIRL simulations."""
  17
  18 __author__ = 'ckoester@cisco.com'
  19
  20 import sys
  21 import re
  22 import os
  23 import argparse
  24 import tempfile
  25 import shutil
  26 import time
  27 import paramiko
  28 import netifaces
  29
  30 import requests
  31
  32 def indent(lines, amount, fillchar=' '):
  33     """Indent the string by amount of fill chars.
  34
  35     :param lines: String to indent.
  36     :param amount: Number of fill chars.
  37     :param fillchar: Filling character.
  38     :type lines: str
  39     :type amount: int
  40     :type fillchar: str
  41     :returns: Indented string.
  42     :rtype: str
  43     """
  44     padding = amount * fillchar
  45     return padding + ('\n'+padding).join(lines.split('\n'))
  46
  47 def print_to_stderr(msg, end='\n'):
  48     """Writes any text to stderr.
  49
  50     :param msg: Message to print.
  51     :param end: By default print new line at the end.
  52     :type msg: str
  53     :type end: str
  54     """
  55     try:
  56         sys.stderr.write(str(msg) + end)
  57     except ValueError:
  58         pass
  59
  60 #
  61 # FIXME: Right now, this is really coded like a shell script, as one big
  62 # function executed in sequence. This should be broken down into multiple
  63 # functions.
  64 #
  65 def main():
  66     """ Main function."""
  67     #
  68     # Get our default interface IP address. This will become the default
  69     # value for the "NFS Server IP" option.
  70     #
  71     gws = netifaces.gateways()
  72     addrs = netifaces.ifaddresses(gws['default'][netifaces.AF_INET][1])
  73     default_addr = addrs[netifaces.AF_INET][0]['addr']
  74
  75     #
  76     # Verify CLI parameters and try to download our VPP image into a temporary
  77     # file first
  78     #
  79     parser = argparse.ArgumentParser()
  80     parser.add_argument("topology", help="the base topology to be started")
  81     parser.add_argument("packages", help="Path to the VPP .deb(s) or .rpm(s) " +
  82                         "that is/are to be installed", nargs='+')
  83     parser.add_argument("-c", "--copy", help="Copy the VPP packages, " +
  84                         "leaving the originals in place. Default is to " +
  85                         "move them.", action='store_true')
  86     parser.add_argument("-k", "--keep", help="Keep (do not delete) the " +
  87                         "simulation in case of error", action='store_true')
  88     parser.add_argument("-v", "--verbosity", action="count", default=0)
  89     parser.add_argument("-nip", "--nfs-server-ip", help="NFS server (our) IP " +
  90                         "default is derived from routing table: " +
  91                         "{}".format(default_addr), default=default_addr)
  92     parser.add_argument("-ns", "--nfs-scratch-directory",
  93                         help="Server location for NFS scratch directory",
  94                         default="/nfs/scratch")
  95     parser.add_argument("-nc", "--nfs-common-directory",
  96                         help="Server location for NFS common (read-only) " +
  97                         "directory", default="/nfs/common")
  98     parser.add_argument("-wc", "--wait-count",
  99                         help="number of intervals to wait for simulation to " +
 100                         "be ready", type=int, default=48)
 101     parser.add_argument("-wt", "--wait-time",
 102                         help="length of a single interval to wait for " +
 103                         "simulation to be ready", type=int, default=5)
 104     parser.add_argument("-vip", "--virl-ip",
 105                         help="VIRL IP and Port (e.g. 127.0.0.1:19399)",
 106                         default="127.0.0.1:19399")
 107     parser.add_argument("-u", "--username", help="VIRL username",
 108                         default="tb4-virl")
 109     parser.add_argument("-au", "--admin-username", help="VIRL admin username",
 110                         default="uwmadmin")
 111     parser.add_argument("-p", "--password", help="VIRL password",
 112                         default="Cisco1234")
 113     parser.add_argument("-su", "--ssh-user", help="SSH username",
 114                         default="cisco")
 115     parser.add_argument("-e", "--expiry", help="Simulation expiry",
 116                         default="120")
 117     parser.add_argument("-spr", "--ssh-privkey", help="SSH private keyfile",
 118                         default="/home/jenkins-in/.ssh/id_rsa_virl")
 119     parser.add_argument("-spu", "--ssh-pubkey", help="SSH public keyfile",
 120                         default="/home/jenkins-in/.ssh/id_rsa_virl.pub")
 121     parser.add_argument("-r", "--release", help="VM disk image/release " +
 122                         "(ex. \"csit-ubuntu-16.04.1_2016-12-19_1.6\")",
 123                         default="csit-ubuntu-16.04.1_2016-12-19_1.6")
 124     parser.add_argument("--topology-directory", help="Topology directory",
 125                         default="/home/jenkins-in/testcase-infra/topologies")
 126
 127     args = parser.parse_args()
 128
 129     #
 130     # Check if topology and template exist
 131     #
 132     if args.verbosity >= 2:
 133         print_to_stderr("DEBUG: Running with topology {}"
 134                         .format(args.topology))
 135
 136     topology_virl_filename = os.path.join(args.topology_directory,
 137                                           args.topology + ".virl")
 138     topology_yaml_filename = os.path.join(args.topology_directory,
 139                                           args.topology + ".yaml")
 140
 141     if not os.path.isfile(topology_virl_filename):
 142         print_to_stderr("ERROR: Topology VIRL file {} does not exist"
 143                         .format(topology_virl_filename))
 144         sys.exit(1)
 145     if not os.path.isfile(topology_yaml_filename):
 146         print_to_stderr("ERROR: Topology YAML file {} does not exist"
 147                         .format(topology_yaml_filename))
 148         sys.exit(1)
 149
 150     #
 151     # Check if VPP package exists
 152     #
 153     for package in args.packages:
 154         if args.verbosity >= 2:
 155             print_to_stderr("DEBUG: Checking if file {} exists"
 156                             .format(package))
 157         if not os.path.isfile(package):
 158             print_to_stderr("ERROR: Debian package {} does not exist"
 159                             .format(package))
 160             sys.exit(1)
 161
 162     #
 163     # Start VIRL topology
 164     #
 165     if args.verbosity >= 1:
 166         print_to_stderr("DEBUG: Starting VIRL topology")
 167     temp_handle, temp_topology = tempfile.mkstemp()
 168     with open(args.ssh_pubkey, 'r') as pubkey_file:
 169         pub_key = pubkey_file.read().replace('\n', '')
 170     with open(temp_topology, 'w') as new_file, \
 171         open(topology_virl_filename, 'r') as old_file:
 172         for line in old_file:
 173             line = line.replace("  - VIRL-USER-SSH-PUBLIC-KEY", "  - "+pub_key)
 174             line = line.replace("$$NFS_SERVER_SCRATCH$$",
 175                                 args.nfs_server_ip+":"+args.nfs_scratch_directory)
 176             line = line.replace("$$NFS_SERVER_COMMON$$",
 177                                 args.nfs_server_ip+":"+args.nfs_common_directory)
 178             line = line.replace("$$VM_IMAGE$$", "server-"+args.release)
 179             new_file.write(line)
 180     os.close(temp_handle)
 181
 182     try:
 183         data = open(temp_topology, 'rb')
 184         req = requests.post('http://' + args.virl_ip + '/simengine/rest/launch',
 185                             auth=(args.username, args.password),
 186                             data=data)
 187         if args.verbosity >= 2:
 188             print_to_stderr("DEBUG: - Request URL {}"
 189                             .format(req.url))
 190             print_to_stderr("{}"
 191                             .format(req.text))
 192             print_to_stderr("DEBUG: - Response Code {}"
 193                             .format(req.status_code))
 194         new_file.close()
 195         if req.status_code != 200:
 196             raise RuntimeError("ERROR: Launching VIRL simulation - "
 197                                "Status other than 200 HTTP OK:\n{}"
 198                                .format(req.content))
 199     except (requests.exceptions.RequestException,
 200             RuntimeError) as ex_error:
 201         print_to_stderr(ex_error)
 202         os.remove(temp_topology)
 203         sys.exit(1)
 204
 205     # If we got here, we had a good response. The response content is the
 206     # session ID.
 207     session_id = req.content
 208     if args.verbosity >= 1:
 209         print_to_stderr("DEBUG: VIRL simulation session-id: {}"
 210                         .format(session_id))
 211
 212     # Set session expiry to autokill sessions if not done from jenkins
 213     if not args.keep:
 214         if args.verbosity >= 1:
 215             print_to_stderr("DEBUG: Setting expire for session-id: {}"
 216                             .format(session_id))
 217         try:
 218             req = requests.put('http://' + args.virl_ip +
 219                                '/simengine/rest/admin-update/' + session_id +
 220                                '/expiry',
 221                                auth=(args.admin_username, args.password),
 222                                params={'user': args.username,
 223                                        'expires': args.expiry})
 224             if args.verbosity >= 2:
 225                 print_to_stderr("DEBUG: - Request URL {}"
 226                                 .format(req.url))
 227                 print_to_stderr("{}"
 228                                 .format(req.text))
 229                 print_to_stderr("DEBUG: - Response Code {}"
 230                                 .format(req.status_code))
 231             if req.status_code != 200:
 232                 raise RuntimeError("ERROR: Setting expiry to simulation - "
 233                                    "Status other than 200 HTTP OK:\n{}"
 234                                    .format(req.content))
 235         except (requests.exceptions.RequestException,
 236                 RuntimeError) as ex_error:
 237             print_to_stderr(ex_error)
 238             req = requests.get('http://' + args.virl_ip +
 239                                '/simengine/rest/stop/' + session_id,
 240                                auth=(args.username, args.password))
 241             os.remove(temp_topology)
 242             print "{}".format(session_id)
 243             sys.exit(1)
 244
 245     #
 246     # Create simulation scratch directory. Move topology file into that
 247     # directory. Copy or move debian packages into that directory.
 248     #
 249     scratch_directory = os.path.join(args.nfs_scratch_directory, session_id)
 250     os.mkdir(scratch_directory)
 251     shutil.move(temp_topology, os.path.join(scratch_directory,
 252                                             "virl_topology.virl"))
 253     os.mkdir(os.path.join(scratch_directory, "vpp"))
 254     for package in args.packages:
 255         if args.copy:
 256             shutil.copy(package, os.path.join(scratch_directory, "vpp",
 257                                               os.path.basename(package)))
 258         else:
 259             shutil.move(package, os.path.join(scratch_directory, "vpp",
 260                                               os.path.basename(package)))
 261
 262     #
 263     # Wait for simulation to become active
 264     #
 265     if args.verbosity >= 1:
 266         print_to_stderr("DEBUG: Waiting for simulation to become active")
 267
 268     sim_is_started = False
 269     nodelist = []
 270
 271     count = args.wait_count
 272     while (count > 0) and not sim_is_started:
 273         time.sleep(args.wait_time)
 274         count -= 1
 275
 276         req = requests.get('http://' + args.virl_ip + '/simengine/rest/nodes/' +
 277                            session_id, auth=(args.username, args.password))
 278         data = req.json()
 279
 280         active = 0
 281         total = 0
 282
 283         # Flush the node list every time, keep the last one
 284         nodelist = []
 285
 286         # Hosts are the keys of the inner dictionary
 287         for key in data[session_id].keys():
 288             if data[session_id][key]['management-proxy'] == "self":
 289                 continue
 290             nodelist.append(key)
 291             total += 1
 292             if data[session_id][key]['state'] == "ACTIVE":
 293                 active += 1
 294         if args.verbosity >= 2:
 295             print_to_stderr("DEBUG: - Attempt {} out of {}, total {} hosts, "
 296                             "{} active".format(args.wait_count-count,
 297                                                args.wait_count, total, active))
 298         if active == total:
 299             sim_is_started = True
 300
 301     if not sim_is_started:
 302         print_to_stderr("ERROR: Simulation nodes never changed to ACTIVE state")
 303         print_to_stderr("Last VIRL response:")
 304         print_to_stderr(data)
 305         if not args.keep:
 306             req = requests.get('http://' + args.virl_ip +
 307                                '/simengine/rest/stop/' + session_id,
 308                                auth=(args.username, args.password))
 309             try:
 310                 shutil.rmtree(scratch_directory)
 311             except shutil.Error:
 312                 print_to_stderr("ERROR: Removing scratch directory")
 313         print "{}".format(session_id)
 314         sys.exit(1)
 315
 316     if args.verbosity >= 2:
 317         print_to_stderr("DEBUG: Nodes: {}"
 318                         .format(", ".join(nodelist)))
 319
 320     #
 321     # Fetch simulation's IPs and create files
 322     # (ansible hosts file, topology YAML file)
 323     #
 324     try:
 325         req = requests.get('http://' + args.virl_ip +
 326                            '/simengine/rest/interfaces/' + session_id,
 327                            auth=(args.username, args.password),
 328                            params={'fetch-state': '1'})
 329         if args.verbosity >= 2:
 330             print_to_stderr("DEBUG: - Request URL {}"
 331                             .format(req.url))
 332             print_to_stderr("DEBUG: - Request Text")
 333             print_to_stderr("{}".format(req.text))
 334             print_to_stderr("DEBUG: - Response Code {}"
 335                             .format(req.status_code))
 336         if req.status_code != 200:
 337             raise RuntimeError("ERROR:Fetching IP's of simulation - "
 338                                "Status other than 200 HTTP OK:\n{}"
 339                                .format(req.content))
 340     except (requests.exceptions.RequestException,
 341             RuntimeError) as ex_error:
 342         print_to_stderr(ex_error)
 343         if not args.keep:
 344             req = requests.get('http://' + args.virl_ip +
 345                                '/simengine/rest/stop/' + session_id,
 346                                auth=(args.username, args.password))
 347             try:
 348                 shutil.rmtree(scratch_directory)
 349             except shutil.Error:
 350                 print_to_stderr("ERROR: Removing scratch directory")
 351         print "{}".format(session_id)
 352         sys.exit(1)
 353     data = req.json()
 354
 355     # Populate node addresses
 356     nodeaddrs = {}
 357     topology = {}
 358     for key in nodelist:
 359         nodetype = re.split('[0-9]', key)[0]
 360         if not nodetype in nodeaddrs:
 361             nodeaddrs[nodetype] = {}
 362         nodeaddrs[nodetype][key] = re.split('\\/', \
 363             data[session_id][key]['management']['ip-address'])[0]
 364         if args.verbosity >= 2:
 365             print_to_stderr("DEBUG: Node {} is of type {} and has mgmt IP {}"
 366                             .format(key, nodetype, nodeaddrs[nodetype][key]))
 367
 368         topology[key] = {}
 369         for key2 in data[session_id][key]:
 370             topology[key]["nic-"+key2] = data[session_id][key][key2]
 371             if 'ip-address' in topology[key]["nic-"+key2]:
 372                 if topology[key]["nic-"+key2]['ip-address'] is not None:
 373                     topology[key]["nic-"+key2]['ip-addr'] = re.split('\\/', \
 374                         topology[key]["nic-"+key2]['ip-address'])[0]
 375
 376     # Write ansible file
 377     ansiblehosts = open(os.path.join(scratch_directory, 'ansible-hosts'), 'w')
 378     for key1 in nodeaddrs:
 379         ansiblehosts.write("[{}]\n".format(key1))
 380         for key2 in nodeaddrs[key1]:
 381             ansiblehosts.write("{} hostname={}\n".format(nodeaddrs[key1][key2],
 382                                                          key2))
 383     ansiblehosts.close()
 384
 385     # Process topology YAML template
 386     with open(args.ssh_privkey, 'r') as privkey_file:
 387         priv_key = indent(privkey_file.read(), 6)
 388
 389     with open(os.path.join(scratch_directory, "topology.yaml"), 'w') as \
 390         new_file, open(topology_yaml_filename, 'r') as old_file:
 391         for line in old_file:
 392             new_file.write(line.format(priv_key=priv_key, topology=topology))
 393
 394     #
 395     # Wait for hosts to become reachable over SSH
 396     #
 397     if args.verbosity >= 1:
 398         print_to_stderr("DEBUG: Waiting for hosts to become reachable over SSH")
 399
 400     missing = -1
 401     count = args.wait_count
 402     while (count > 0) and missing != 0:
 403         time.sleep(args.wait_time)
 404         count -= 1
 405
 406         missing = 0
 407         for key in nodelist:
 408             if not os.path.exists(os.path.join(scratch_directory, key)):
 409                 missing += 1
 410         if args.verbosity >= 2:
 411             print_to_stderr("DEBUG: Attempt {} out of {}, waiting for {} hosts"
 412                             .format(args.wait_count-count, args.wait_count,
 413                                     missing))
 414
 415     if missing != 0:
 416         print_to_stderr("ERROR: Simulation started OK but {} hosts never "
 417                         "mounted their NFS directory".format(missing))
 418         if not args.keep:
 419             req = requests.get('http://' + args.virl_ip +
 420                                '/simengine/rest/stop/' + session_id,
 421                                auth=(args.username, args.password))
 422             try:
 423                 shutil.rmtree(scratch_directory)
 424             except shutil.Error:
 425                 print_to_stderr("ERROR: Removing scratch directory")
 426         print "{}".format(session_id)
 427         sys.exit(1)
 428
 429     #
 430     # Upgrade VPP
 431     #
 432     if args.verbosity >= 1:
 433         print_to_stderr("DEBUG: Uprading VPP")
 434
 435     for key1 in nodeaddrs:
 436         if not key1 == 'tg':
 437             for key2 in nodeaddrs[key1]:
 438                 ipaddr = nodeaddrs[key1][key2]
 439                 if args.verbosity >= 2:
 440                     print_to_stderr("DEBUG: Upgrading VPP on node {}"
 441                                     .format(ipaddr))
 442                 paramiko.util.log_to_file(os.path.join(scratch_directory,
 443                                                        "ssh.log"))
 444                 client = paramiko.SSHClient()
 445                 client.load_system_host_keys()
 446                 client.load_host_keys("/dev/null")
 447                 client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 448                 client.connect(ipaddr, username=args.ssh_user,
 449                                key_filename=args.ssh_privkey)
 450                 if 'centos' in args.topology:
 451                     if args.verbosity >= 1:
 452                         print_to_stderr("DEBUG: Installing RPM packages")
 453                     vpp_install_command = 'sudo rpm -ivh /scratch/vpp/*.rpm'
 454                 elif 'trusty' in args.topology or 'xenial' in args.topology:
 455                     if args.verbosity >= 1:
 456                         print_to_stderr("DEBUG: Installing DEB packages")
 457                     vpp_install_command = 'sudo dpkg -i --force-all ' \
 458                                           '/scratch/vpp/*.deb'
 459                 else:
 460                     print_to_stderr("ERROR: Unsupported OS requested: {}"
 461                                     .format(args.topology))
 462                     vpp_install_command = ''
 463                 _, stdout, stderr = \
 464                     client.exec_command(vpp_install_command)
 465                 c_stdout = stdout.read()
 466                 c_stderr = stderr.read()
 467                 if args.verbosity >= 2:
 468                     print_to_stderr("DEBUG: Command output was:")
 469                     print_to_stderr(c_stdout)
 470                     print_to_stderr("DEBUG: Command stderr was:")
 471                     print_to_stderr(c_stderr)
 472
 473     #
 474     # Write a file with timestamp to scratch directory. We can use this to track
 475     # how long a simulation has been running.
 476     #
 477     with open(os.path.join(scratch_directory, 'start_time'), 'a') as \
 478         timestampfile:
 479         timestampfile.write('{}\n'.format(int(time.time())))
 480
 481     #
 482     # Declare victory
 483     #
 484     if args.verbosity >= 1:
 485         print_to_stderr("SESSION ID: {}".format(session_id))
 486
 487     print "{}".format(session_id)
 488
 489 if __name__ == "__main__":
 490     sys.exit(main())