resources/tools/virl/bin/start-testcase-DMM

   1 #!/usr/bin/python
   2
   3 # Copyright (c) 2018 Huawei Technologies Co.,Ltd.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 """This script is handling starting of VIRL simulations."""
  17
  18 import argparse
  19 import netifaces
  20 import os
  21 import paramiko
  22 import random
  23 import re
  24 import shutil
  25 import sys
  26 import tempfile
  27 import time
  28
  29 import requests
  30
  31 IPS_PER_SIMULATION = 5
  32
  33 def indent(lines, amount, fillchar=' '):
  34     """Indent the string by amount of fill chars.
  35
  36     :param lines: String to indent.
  37     :param amount: Number of fill chars.
  38     :param fillchar: Filling character.
  39     :type lines: str
  40     :type amount: int
  41     :type fillchar: str
  42     :returns: Indented string.
  43     :rtype: str
  44     """
  45     padding = amount * fillchar
  46     return padding + ('\n'+padding).join(lines.split('\n'))
  47
  48 def print_to_stderr(msg, end='\n'):
  49     """Writes any text to stderr.
  50
  51     :param msg: Message to print.
  52     :param end: By default print new line at the end.
  53     :type msg: str
  54     :type end: str
  55     """
  56     try:
  57         sys.stderr.write(str(msg) + end)
  58     except ValueError:
  59         pass
  60
  61 def get_assigned_interfaces(args, network="flat"):
  62     """Retrieve assigned interfaces in openstack network.
  63
  64     :param args: Command line params.
  65     :param network: Openstack network.
  66     :type args: ArgumentParser
  67     :type network: str
  68     :returns: Assigned interfaces.
  69     :rtype: list
  70     :raises RuntimeError: If response is not 200.
  71     """
  72     req = requests.get('http://{}/openstack/rest/ports/{}'
  73                        .format(args.virl_ip, network),
  74                        auth=(args.username, args.password))
  75     if req.status_code == 200:
  76         return req.json()
  77     else:
  78         raise RuntimeError("ERROR: Retrieving ports in use - "
  79                            "Status other than 200 HTTP OK:\n{}"
  80                            .format(req.content))
  81
  82 def get_assigned_interfaces_count(args, network="flat"):
  83     """Count assigned interfaces in openstack network.
  84
  85     :param args: Command line params.
  86     :param network: Openstack network.
  87     :type args: ArgumentParser
  88     :type network: str
  89     :returns: Assigned interfaces count.
  90     :rtype: int
  91     """
  92     return len(get_assigned_interfaces(args, network=network))
  93
  94 def check_ip_addresses(args):
  95     """Check IP address availability.
  96
  97     :param args: Command line params.
  98     :type args: ArgumentParser
  99     :raises RuntimeError: If not enough free addresses available.
 100     """
 101     for i in range(args.wait_count):
 102         if args.quota - \
 103             get_assigned_interfaces_count(args) >= IPS_PER_SIMULATION:
 104             break
 105         if args.verbosity >= 2:
 106             print_to_stderr("DEBUG: - Attempt {} out of {}, waiting for free "
 107                             "IP addresses".format(i, args.wait_count))
 108         # Wait random amount of time within range 1-3 minutes
 109         time.sleep(random.randint(60, 180))
 110     else:
 111         raise RuntimeError("ERROR: Not enough IP addresses to run simulation")
 112
 113 def check_virl_resources(args):
 114     """Check virl resources availability.
 115
 116     :param args: Command line params.
 117     :type args: ArgumentParser
 118     """
 119     check_ip_addresses(args)
 120
 121 #
 122 # FIXME: Right now, this is really coded like a shell script, as one big
 123 # function executed in sequence. This should be broken down into multiple
 124 # functions.
 125 #
 126 def main():
 127     """ Main function."""
 128     #
 129     # Get our default interface IP address. This will become the default
 130     # value for the "NFS Server IP" option.
 131     #
 132     gws = netifaces.gateways()
 133     addrs = netifaces.ifaddresses(gws['default'][netifaces.AF_INET][1])
 134     default_addr = addrs[netifaces.AF_INET][0]['addr']
 135
 136     #
 137     # Verify CLI parameters and try to download our VPP image into a temporary
 138     # file first
 139     #
 140     parser = argparse.ArgumentParser()
 141     parser.add_argument("topology", help="the base topology to be started")
 142     parser.add_argument("packages", help="Path to the dmm package file that " +
 143                         "is/are to be installed", nargs='+')
 144     parser.add_argument("-c", "--copy", help="Copy the dmm packages, " +
 145                         "leaving the originals in place. Default is to " +
 146                         "move them.", action='store_true')
 147     parser.add_argument("-k", "--keep", help="Keep (do not delete) the " +
 148                         "simulation in case of error", action='store_true')
 149     parser.add_argument("-v", "--verbosity", action="count", default=0)
 150     parser.add_argument("-nip", "--nfs-server-ip", help="NFS server (our) IP " +
 151                         "default is derived from routing table: " +
 152                         "{}".format(default_addr), default=default_addr)
 153     parser.add_argument("-ns", "--nfs-scratch-directory",
 154                         help="Server location for NFS scratch diretory",
 155                         default="/nfs/scratch")
 156     parser.add_argument("-nc", "--nfs-common-directory",
 157                         help="Server location for NFS common (read-only) " +
 158                         "directory", default="/nfs/common")
 159     parser.add_argument("-wc", "--wait-count",
 160                         help="number of intervals to wait for simulation to " +
 161                         "be ready", type=int, default=24)
 162     parser.add_argument("-wt", "--wait-time",
 163                         help="length of a single interval to wait for " +
 164                         "simulation to be ready", type=int, default=5)
 165     parser.add_argument("-vip", "--virl-ip",
 166                         help="VIRL IP and Port (e.g. 127.0.0.1:19399)",
 167                         default="127.0.0.1:19399")
 168     parser.add_argument("-u", "--username", help="VIRL username",
 169                         default="tb4-virl")
 170     parser.add_argument("-au", "--admin-username", help="VIRL admin username",
 171                         default="uwmadmin")
 172     parser.add_argument("-p", "--password", help="VIRL password",
 173                         default="Cisco1234")
 174     parser.add_argument("-su", "--ssh-user", help="SSH username",
 175                         default="cisco")
 176     parser.add_argument("-e", "--expiry", help="Simulation expiry",
 177                         default="120")
 178     parser.add_argument("-spr", "--ssh-privkey", help="SSH private keyfile",
 179                         default="/home/jenkins-in/.ssh/id_rsa_virl")
 180     parser.add_argument("-spu", "--ssh-pubkey", help="SSH public keyfile",
 181                         default="/home/jenkins-in/.ssh/id_rsa_virl.pub")
 182     parser.add_argument("-r", "--release", help="VM disk image/release " +
 183                         "(ex. \"csit-ubuntu-16.04.1_2016-12-19_1.6\")",
 184                         default="csit-ubuntu-16.04.1_2016-12-19_1.6")
 185     parser.add_argument("--topology-directory", help="Topology directory",
 186                         default="/home/jenkins-in/testcase-infra/topologies")
 187     parser.add_argument("-q", "--quota",
 188                         help="VIRL quota for max number of allowed IPs",
 189                         type=int, default=74)
 190
 191     args = parser.parse_args()
 192
 193     #
 194     # Check if topology and template exist
 195     #
 196     if args.verbosity >= 2:
 197         print_to_stderr("DEBUG: Running with topology {}"
 198                         .format(args.topology))
 199
 200     topology_virl_filename = os.path.join(args.topology_directory,
 201                                           args.topology + ".virl")
 202     topology_yaml_filename = os.path.join(args.topology_directory,
 203                                           args.topology + ".yaml")
 204
 205     if not os.path.isfile(topology_virl_filename):
 206         print_to_stderr("ERROR: Topology VIRL file {} does not exist"
 207                         .format(topology_virl_filename))
 208         sys.exit(1)
 209     if not os.path.isfile(topology_yaml_filename):
 210         print_to_stderr("ERROR: Topology YAML file {} does not exist"
 211                         .format(topology_yaml_filename))
 212         sys.exit(1)
 213
 214     #
 215     # Check if DMM package exists
 216     #
 217     for package in args.packages:
 218         if args.verbosity >= 2:
 219             print_to_stderr("DEBUG: Checking if file {} exists"
 220                             .format(package))
 221         if not os.path.isfile(package):
 222             print_to_stderr("ERROR: DMM package {} does not exist."
 223                             .format(package))
 224             sys.exit(1)
 225
 226     #
 227     # Start VIRL topology
 228     #
 229     if args.verbosity >= 1:
 230         print_to_stderr("DEBUG: Starting VIRL topology")
 231     temp_handle, temp_topology = tempfile.mkstemp()
 232     with open(args.ssh_pubkey, 'r') as pubkey_file:
 233         pub_key = pubkey_file.read().replace('\n', '')
 234     with open(temp_topology, 'w') as new_file, \
 235         open(topology_virl_filename, 'r') as old_file:
 236         for line in old_file:
 237             line = line.replace("  - VIRL-USER-SSH-PUBLIC-KEY", "  - "+pub_key)
 238             line = line.replace("$$NFS_SERVER_SCRATCH$$", \
 239                 args.nfs_server_ip+":"+args.nfs_scratch_directory)
 240             line = line.replace("$$NFS_SERVER_COMMON$$", \
 241                 args.nfs_server_ip+":"+args.nfs_common_directory)
 242             line = line.replace("$$VM_IMAGE$$", "server-"+args.release)
 243             new_file.write(line)
 244     os.close(temp_handle)
 245
 246     try:
 247         data = open(temp_topology, 'rb')
 248         check_virl_resources(args)
 249         req = requests.post('http://' + args.virl_ip + '/simengine/rest/launch',
 250                             auth=(args.username, args.password),
 251                             data=data)
 252         if args.verbosity >= 2:
 253             print_to_stderr("DEBUG: - Request URL {}"
 254                             .format(req.url))
 255             print_to_stderr("{}"
 256                             .format(req.text))
 257             print_to_stderr("DEBUG: - Response Code {}"
 258                             .format(req.status_code))
 259         new_file.close()
 260         if req.status_code != 200:
 261             raise RuntimeError("ERROR: Launching VIRL simulation - "
 262                                "Status other than 200 HTTP OK:\n{}"
 263                                .format(req.content))
 264     except (requests.exceptions.RequestException,
 265             RuntimeError) as ex_error:
 266         print_to_stderr(ex_error)
 267         os.remove(temp_topology)
 268         sys.exit(1)
 269
 270     # If we got here, we had a good response. The response content is the
 271     # session ID.
 272     session_id = req.content
 273     if args.verbosity >= 1:
 274         print_to_stderr("DEBUG: VIRL simulation session-id: {}"
 275                         .format(session_id))
 276
 277     # Set session expiry to autokill sessions if not done from jenkins
 278     if not args.keep:
 279         if args.verbosity >= 1:
 280             print_to_stderr("DEBUG: Setting expire for session-id: {}"
 281                             .format(session_id))
 282         try:
 283             req = requests.put('http://' + args.virl_ip +
 284                                '/simengine/rest/admin-update/' + session_id +
 285                                '/expiry',
 286                                auth=(args.admin_username, args.password),
 287                                params={'user': args.username,
 288                                        'expires': args.expiry})
 289             if args.verbosity >= 2:
 290                 print_to_stderr("DEBUG: - Request URL {}"
 291                                 .format(req.url))
 292                 print_to_stderr("{}"
 293                                 .format(req.text))
 294                 print_to_stderr("DEBUG: - Response Code {}"
 295                                 .format(req.status_code))
 296             if req.status_code != 200:
 297                 raise RuntimeError("ERROR: Setting expiry to simulation - "
 298                                    "Status other than 200 HTTP OK:\n{}"
 299                                    .format(req.content))
 300         except (requests.exceptions.RequestException,
 301                 RuntimeError) as ex_error:
 302             print_to_stderr(ex_error)
 303             req = requests.get('http://' + args.virl_ip +
 304                                '/simengine/rest/stop/' + session_id,
 305                                auth=(args.username, args.password))
 306             os.remove(temp_topology)
 307             print "{}".format(session_id)
 308             sys.exit(1)
 309
 310     #
 311     # Create simulation scratch directory. Move topology file into that
 312     # directory. Copy or move DMM packages into that directory.
 313     #
 314     scratch_directory = os.path.join(args.nfs_scratch_directory, session_id)
 315     os.mkdir(scratch_directory)
 316     shutil.move(temp_topology, os.path.join(scratch_directory,
 317                                             "virl_topology.virl"))
 318     os.mkdir(os.path.join(scratch_directory, "dmmtest"))
 319     for package in args.packages:
 320         if args.copy:
 321             shutil.copy(package, os.path.join(scratch_directory, "dmmtest",
 322                                               os.path.basename(package)))
 323         else:
 324             shutil.move(package, os.path.join(scratch_directory, "dmmtest",
 325                                               os.path.basename(package)))
 326
 327     #
 328     # Wait for simulation to become active
 329     #
 330     if args.verbosity >= 1:
 331         print_to_stderr("DEBUG: Waiting for simulation to become active")
 332
 333     sim_is_started = False
 334     nodelist = []
 335
 336     count = args.wait_count
 337     while (count > 0) and not sim_is_started:
 338         time.sleep(args.wait_time)
 339         count -= 1
 340
 341         req = requests.get('http://' + args.virl_ip + '/simengine/rest/nodes/' +
 342                            session_id, auth=(args.username, args.password))
 343         data = req.json()
 344
 345         active = 0
 346         total = 0
 347
 348         # Flush the node list every time, keep the last one
 349         nodelist = []
 350
 351         # Hosts are the keys of the inner dictionary
 352         for key in data[session_id].keys():
 353             if data[session_id][key]['management-proxy'] == "self":
 354                 continue
 355             nodelist.append(key)
 356             total += 1
 357             if data[session_id][key]['state'] == "ACTIVE":
 358                 active += 1
 359         if args.verbosity >= 2:
 360             print_to_stderr("DEBUG: - Attempt {} out of {}, total {} hosts, "
 361                             "{} active".format(args.wait_count-count,
 362                                                args.wait_count, total, active))
 363         if active == total:
 364             sim_is_started = True
 365
 366     if not sim_is_started:
 367         print_to_stderr("ERROR: Simulation nodes never changed to ACTIVE state")
 368         print_to_stderr("Last VIRL response:")
 369         print_to_stderr(data)
 370         if not args.keep:
 371             req = requests.get('http://' + args.virl_ip +
 372                                '/simengine/rest/stop/' + session_id,
 373                                auth=(args.username, args.password))
 374             try:
 375                 shutil.rmtree(scratch_directory)
 376             except:
 377                 print_to_stderr("ERROR: Removing scratch directory")
 378             print "{}".format(session_id)
 379         sys.exit(1)
 380
 381     if args.verbosity >= 2:
 382         print_to_stderr("DEBUG: Nodes: {}"
 383                         .format(", ".join(nodelist)))
 384
 385     #
 386     # Fetch simulation's IPs and create files
 387     # (ansible hosts file, topology YAML file)
 388     #
 389     try:
 390         req = requests.get('http://' + args.virl_ip +
 391                            '/simengine/rest/interfaces/' + session_id,
 392                            auth=(args.username, args.password),
 393                            params={'fetch-state': '1'})
 394         if args.verbosity >= 2:
 395             print_to_stderr("DEBUG: - Request URL {}"
 396                             .format(req.url))
 397             print_to_stderr("DEBUG: - Request Text")
 398             print_to_stderr("{}".format(req.text))
 399             print_to_stderr("DEBUG: - Response Code {}"
 400                             .format(req.status_code))
 401         if req.status_code != 200:
 402             raise RuntimeError("ERROR:Fetching IP's of simulation - "
 403                                "Status other than 200 HTTP OK:\n{}"
 404                                .format(req.content))
 405     except (requests.exceptions.RequestException,
 406             RuntimeError) as ex_error:
 407         print_to_stderr(ex_error)
 408         if not args.keep:
 409             req = requests.get('http://' + args.virl_ip +
 410                                '/simengine/rest/stop/' + session_id,
 411                                auth=(args.username, args.password))
 412             try:
 413                 shutil.rmtree(scratch_directory)
 414             except:
 415                 print_to_stderr("ERROR: Removing scratch directory")
 416             print "{}".format(session_id)
 417         sys.exit(1)
 418     data = req.json()
 419
 420     # Populate node addresses
 421     nodeaddrs = {}
 422     topology = {}
 423     for key in nodelist:
 424         nodetype = re.split('[0-9]', key)[0]
 425         if not nodetype in nodeaddrs:
 426             nodeaddrs[nodetype] = {}
 427         nodeaddrs[nodetype][key] = re.split('\\/', \
 428             data[session_id][key]['management']['ip-address'])[0]
 429         if args.verbosity >= 2:
 430             print_to_stderr("DEBUG: Node {} is of type {} and has mgmt IP {}"
 431                             .format(key, nodetype, nodeaddrs[nodetype][key]))
 432
 433         topology[key] = {}
 434         for key2 in data[session_id][key]:
 435             topology[key]["nic-"+key2] = data[session_id][key][key2]
 436             if 'ip-address' in topology[key]["nic-"+key2]:
 437                 if topology[key]["nic-"+key2]['ip-address'] is not None:
 438                     topology[key]["nic-"+key2]['ip-addr'] = re.split('\\/', \
 439                         topology[key]["nic-"+key2]['ip-address'])[0]
 440
 441     # Write ansible file
 442     ansiblehosts = open(os.path.join(scratch_directory, 'ansible-hosts'), 'w')
 443     for key1 in nodeaddrs:
 444         ansiblehosts.write("[{}]\n".format(key1))
 445         for key2 in nodeaddrs[key1]:
 446             ansiblehosts.write("{} hostname={}\n".format(nodeaddrs[key1][key2],
 447                                                          key2))
 448     ansiblehosts.close()
 449
 450     # Process topology YAML template
 451     with open(args.ssh_privkey, 'r') as privkey_file:
 452         priv_key = indent(privkey_file.read(), 6)
 453
 454     with open(os.path.join(scratch_directory, "topology.yaml"), 'w') as \
 455         new_file, open(topology_yaml_filename, 'r') as old_file:
 456         for line in old_file:
 457             new_file.write(line.format(priv_key=priv_key, topology=topology))
 458
 459     #
 460     # Wait for hosts to become reachable over SSH
 461     #
 462     if args.verbosity >= 1:
 463         print_to_stderr("DEBUG: Waiting for hosts to become reachable over SSH")
 464
 465     missing = -1
 466     count = args.wait_count
 467     while (count > 0) and missing != 0:
 468         time.sleep(args.wait_time)
 469         count -= 1
 470
 471         missing = 0
 472         for key in nodelist:
 473             if not os.path.exists(os.path.join(scratch_directory, key)):
 474                 missing += 1
 475         if args.verbosity >= 2:
 476             print_to_stderr("DEBUG: Attempt {} out of {}, waiting for {} hosts"
 477                             .format(args.wait_count-count, args.wait_count,
 478                                     missing))
 479
 480     if missing != 0:
 481         print_to_stderr("ERROR: Simulation started OK but {} hosts never "
 482                         "mounted their NFS directory".format(missing))
 483         if not args.keep:
 484             req = requests.get('http://' + args.virl_ip +
 485                                '/simengine/rest/stop/' + session_id,
 486                                auth=(args.username, args.password))
 487             try:
 488                 shutil.rmtree(scratch_directory)
 489             except:
 490                 print_to_stderr("ERROR: Removing scratch directory")
 491             print "{}".format(session_id)
 492         sys.exit(1)
 493
 494     #
 495     # just decompress the DMM tar packages
 496     #
 497     if args.verbosity >= 1:
 498         print_to_stderr("DEBUG: Uprading DMM")
 499
 500     for key1 in nodeaddrs:
 501         if not key1 == 'tg':
 502             for key2 in nodeaddrs[key1]:
 503                 ipaddr = nodeaddrs[key1][key2]
 504                 if args.verbosity >= 2:
 505                     print_to_stderr("DEBUG: Upgrading DMM on node {}"
 506                                     .format(ipaddr))
 507                 paramiko.util.log_to_file(os.path.join(scratch_directory,
 508                                                        "ssh.log"))
 509                 client = paramiko.SSHClient()
 510                 client.load_system_host_keys()
 511                 client.load_host_keys("/dev/null")
 512                 client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 513                 client.connect(ipaddr, username=args.ssh_user,
 514                                key_filename=args.ssh_privkey)
 515                 _, stdout, stderr = \
 516                     client.exec_command('cd /scratch/dmmtest/ && sudo tar zxf dmm_depends.tar.gz')
 517                 c_stdout = stdout.read()
 518                 c_stderr = stderr.read()
 519                 if args.verbosity >= 2:
 520                     print_to_stderr("DEBUG: Command output was:")
 521                     print_to_stderr(c_stdout)
 522                     print_to_stderr("DEBUG: Command stderr was:")
 523                     print_to_stderr(c_stderr)
 524
 525     #
 526     # Write a file with timestamp to scratch directory. We can use this to track
 527     # how long a simulation has been running.
 528     #
 529     with open(os.path.join(scratch_directory, 'start_time'), 'a') as \
 530         timestampfile:
 531         timestampfile.write('{}\n'.format(int(time.time())))
 532
 533     #
 534     # Declare victory
 535     #
 536     if args.verbosity >= 1:
 537         print_to_stderr("SESSION ID: {}".format(session_id))
 538
 539     print "{}".format(session_id)
 540
 541 if __name__ == "__main__":
 542     sys.exit(main())