3 # Copyright (c) 2016 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 """This script is handling starting of VIRL simulations."""
18 __author__ = 'ckoester@cisco.com'
33 IPS_PER_SIMULATION = 5
36 def indent(lines, amount, fillchar=' '):
37 """Indent the string by amount of fill chars.
39 :param lines: String to indent.
40 :param amount: Number of fill chars.
41 :param fillchar: Filling character.
45 :returns: Indented string.
48 padding = amount * fillchar
49 return padding + ('\n'+padding).join(lines.split('\n'))
52 def print_to_stderr(msg, end='\n'):
53 """Writes any text to stderr.
55 :param msg: Message to print.
56 :param end: By default print new line at the end.
61 sys.stderr.write(str(msg) + end)
66 def get_assigned_interfaces(args, network="flat"):
67 """Retrieve assigned interfaces in openstack network.
69 :param args: Command line params.
70 :param network: Openstack network.
71 :type args: ArgumentParser
73 :returns: Assigned interfaces.
75 :raises RuntimeError: If response is not 200.
77 req = requests.get('http://{}/openstack/rest/ports/{}'
78 .format(args.virl_ip, network),
79 auth=(args.username, args.password))
80 if req.status_code == 200:
83 raise RuntimeError("ERROR: Retrieving ports in use - "
84 "Status other than 200 HTTP OK:\n{}"
88 def get_assigned_interfaces_count(args, network="flat"):
89 """Count assigned interfaces in openstack network.
91 :param args: Command line params.
92 :param network: Openstack network.
93 :type args: ArgumentParser
95 :returns: Assigned interfaces count.
98 return len(get_assigned_interfaces(args, network=network))
101 def check_ip_addresses(args):
102 """Check IP address availability.
104 :param args: Command line params.
105 :type args: ArgumentParser
106 :raises RuntimeError: If not enough free addresses available.
108 for i in range(args.wait_count):
110 get_assigned_interfaces_count(args) >= IPS_PER_SIMULATION):
112 if args.verbosity >= 2:
113 print_to_stderr("DEBUG: - Attempt {} out of {}, waiting for free "
114 "IP addresses".format(i, args.wait_count))
115 # Wait random amount of time within range 1-3 minutes
116 time.sleep(random.randint(60,180))
118 raise RuntimeError("ERROR: Not enough IP addresses to run simulation")
121 def check_virl_resources(args):
122 """Check virl resources availability.
124 :param args: Command line params.
125 :type args: ArgumentParser
127 check_ip_addresses(args)
130 # FIXME: Right now, this is really coded like a shell script, as one big
131 # function executed in sequence. This should be broken down into multiple
137 """ Main function."""
139 # Get our default interface IP address. This will become the default
140 # value for the "NFS Server IP" option.
142 gws = netifaces.gateways()
143 addrs = netifaces.ifaddresses(gws['default'][netifaces.AF_INET][1])
144 default_addr = addrs[netifaces.AF_INET][0]['addr']
147 # Verify CLI parameters and try to download our VPP image into a temporary
150 parser = argparse.ArgumentParser()
151 parser.add_argument("topology", help="the base topology to be started")
152 parser.add_argument("packages", help="Path to the VPP .deb(s) or .rpm(s) " +
153 "that is/are to be installed", nargs='+')
154 parser.add_argument("-c", "--copy", help="Copy the VPP packages, " +
155 "leaving the originals in place. Default is to " +
156 "move them.", action='store_true')
157 parser.add_argument("-k", "--keep", help="Keep (do not delete) the " +
158 "simulation in case of error", action='store_true')
159 parser.add_argument("-v", "--verbosity", action="count", default=0)
160 parser.add_argument("-nip", "--nfs-server-ip", help="NFS server (our) IP " +
161 "default is derived from routing table: " +
162 "{}".format(default_addr), default=default_addr)
163 parser.add_argument("-ns", "--nfs-scratch-directory",
164 help="Server location for NFS scratch directory",
165 default="/nfs/scratch")
166 parser.add_argument("-nc", "--nfs-common-directory",
167 help="Server location for NFS common (read-only) " +
168 "directory", default="/nfs/common")
169 parser.add_argument("-wc", "--wait-count",
170 help="number of intervals to wait for simulation to " +
171 "be ready", type=int, default=48)
172 parser.add_argument("-wt", "--wait-time",
173 help="length of a single interval to wait for " +
174 "simulation to be ready", type=int, default=5)
175 parser.add_argument("-vip", "--virl-ip",
176 help="VIRL IP and Port (e.g. 127.0.0.1:19399)",
177 default="127.0.0.1:19399")
178 parser.add_argument("-u", "--username", help="VIRL username",
180 parser.add_argument("-au", "--admin-username", help="VIRL admin username",
182 parser.add_argument("-p", "--password", help="VIRL password",
184 parser.add_argument("-su", "--ssh-user", help="SSH username",
186 parser.add_argument("-e", "--expiry", help="Simulation expiry",
188 parser.add_argument("-spr", "--ssh-privkey", help="SSH private keyfile",
189 default="/home/jenkins-in/.ssh/id_rsa_virl")
190 parser.add_argument("-spu", "--ssh-pubkey", help="SSH public keyfile",
191 default="/home/jenkins-in/.ssh/id_rsa_virl.pub")
192 parser.add_argument("-r", "--release", help="VM disk image/release " +
193 "(ex. \"csit-ubuntu-16.04.1_2016-12-19_1.6\")",
194 default="csit-ubuntu-16.04.1_2016-12-19_1.6")
195 parser.add_argument("--topology-directory", help="Topology directory",
196 default="/home/jenkins-in/testcase-infra/topologies")
197 parser.add_argument("-q", "--quota",
198 help="VIRL quota for max number of allowed IPs",
199 type=int, default=74)
200 parser.add_argument("-si", "--skip-install", help="Skip VPP installation",
203 args = parser.parse_args()
206 # Check if topology and template exist
208 if args.verbosity >= 2:
209 print_to_stderr("DEBUG: Running with topology {}"
210 .format(args.topology))
212 topology_virl_filename = os.path.join(args.topology_directory,
213 args.topology + ".virl")
214 topology_yaml_filename = os.path.join(args.topology_directory,
215 args.topology + ".yaml")
217 if not os.path.isfile(topology_virl_filename):
218 print_to_stderr("ERROR: Topology VIRL file {} does not exist"
219 .format(topology_virl_filename))
221 if not os.path.isfile(topology_yaml_filename):
222 print_to_stderr("ERROR: Topology YAML file {} does not exist"
223 .format(topology_yaml_filename))
227 # Check if VPP package exists
229 if not args.skip_install:
230 for package in args.packages:
231 if args.verbosity >= 2:
232 print_to_stderr("DEBUG: Checking if file {} exists"
234 if not os.path.isfile(package):
235 print_to_stderr("ERROR: Required package {} does not exist"
240 # Start VIRL topology
242 if args.verbosity >= 1:
243 print_to_stderr("DEBUG: Starting VIRL topology")
244 temp_handle, temp_topology = tempfile.mkstemp()
245 with open(args.ssh_pubkey, 'r') as pubkey_file:
246 pub_key = pubkey_file.read().replace('\n', '')
247 with open(temp_topology, 'w') as new_file, \
248 open(topology_virl_filename, 'r') as old_file:
249 for line in old_file:
250 line = line.replace(" - VIRL-USER-SSH-PUBLIC-KEY", " - "+pub_key)
251 line = line.replace("$$NFS_SERVER_SCRATCH$$",
252 args.nfs_server_ip+":"+args.nfs_scratch_directory)
253 line = line.replace("$$NFS_SERVER_COMMON$$",
254 args.nfs_server_ip+":"+args.nfs_common_directory)
255 line = line.replace("$$VM_IMAGE$$", "server-"+args.release)
257 os.close(temp_handle)
260 data = open(temp_topology, 'rb')
261 check_virl_resources(args)
262 req = requests.post('http://' + args.virl_ip + '/simengine/rest/launch',
263 auth=(args.username, args.password),
265 if args.verbosity >= 2:
266 print_to_stderr("DEBUG: - Request URL {}"
270 print_to_stderr("DEBUG: - Response Code {}"
271 .format(req.status_code))
273 if req.status_code != 200:
274 raise RuntimeError("ERROR: Launching VIRL simulation - "
275 "Status other than 200 HTTP OK:\n{}"
276 .format(req.content))
277 except (requests.exceptions.RequestException,
278 RuntimeError) as ex_error:
279 print_to_stderr(ex_error)
280 os.remove(temp_topology)
283 # If we got here, we had a good response. The response content is the
285 session_id = req.content
286 if args.verbosity >= 1:
287 print_to_stderr("DEBUG: VIRL simulation session-id: {}"
290 # Set session expiry to autokill sessions if not done from jenkins
292 if args.verbosity >= 1:
293 print_to_stderr("DEBUG: Setting expire for session-id: {}"
296 req = requests.put('http://' + args.virl_ip +
297 '/simengine/rest/admin-update/' + session_id +
299 auth=(args.admin_username, args.password),
300 params={'user': args.username,
301 'expires': args.expiry})
302 if args.verbosity >= 2:
303 print_to_stderr("DEBUG: - Request URL {}"
307 print_to_stderr("DEBUG: - Response Code {}"
308 .format(req.status_code))
309 if req.status_code != 200:
310 raise RuntimeError("ERROR: Setting expiry to simulation - "
311 "Status other than 200 HTTP OK:\n{}"
312 .format(req.content))
313 except (requests.exceptions.RequestException,
314 RuntimeError) as ex_error:
315 print_to_stderr(ex_error)
316 req = requests.get('http://' + args.virl_ip +
317 '/simengine/rest/stop/' + session_id,
318 auth=(args.username, args.password))
319 os.remove(temp_topology)
320 print "{}".format(session_id)
324 # Create simulation scratch directory. Move topology file into that
325 # directory. Copy or move debian packages into that directory.
327 scratch_directory = os.path.join(args.nfs_scratch_directory, session_id)
328 os.mkdir(scratch_directory)
329 shutil.move(temp_topology, os.path.join(scratch_directory,
330 "virl_topology.virl"))
331 os.mkdir(os.path.join(scratch_directory, "vpp"))
332 for package in args.packages:
334 shutil.copy(package, os.path.join(scratch_directory, "vpp",
335 os.path.basename(package)))
337 shutil.move(package, os.path.join(scratch_directory, "vpp",
338 os.path.basename(package)))
341 # Wait for simulation to become active
343 if args.verbosity >= 1:
344 print_to_stderr("DEBUG: Waiting for simulation to become active")
346 sim_is_started = False
349 count = args.wait_count
350 while (count > 0) and not sim_is_started:
351 time.sleep(args.wait_time)
354 req = requests.get('http://' + args.virl_ip + '/simengine/rest/nodes/' +
355 session_id, auth=(args.username, args.password))
361 # Flush the node list every time, keep the last one
364 # Hosts are the keys of the inner dictionary
365 for key in data[session_id].keys():
366 if data[session_id][key]['management-proxy'] == "self":
370 if data[session_id][key]['state'] == "ACTIVE":
372 if args.verbosity >= 2:
373 print_to_stderr("DEBUG: - Attempt {} out of {}, total {} hosts, "
374 "{} active".format(args.wait_count-count,
375 args.wait_count, total, active))
377 sim_is_started = True
379 if not sim_is_started:
380 print_to_stderr("ERROR: Simulation nodes never changed to ACTIVE state")
381 print_to_stderr("Last VIRL response:")
382 print_to_stderr(data)
384 req = requests.get('http://' + args.virl_ip +
385 '/simengine/rest/stop/' + session_id,
386 auth=(args.username, args.password))
388 shutil.rmtree(scratch_directory)
390 print_to_stderr("ERROR: Removing scratch directory")
391 print "{}".format(session_id)
394 if args.verbosity >= 2:
395 print_to_stderr("DEBUG: Nodes: {}"
396 .format(", ".join(nodelist)))
399 # Fetch simulation's IPs and create files
400 # (ansible hosts file, topology YAML file)
403 req = requests.get('http://' + args.virl_ip +
404 '/simengine/rest/interfaces/' + session_id,
405 auth=(args.username, args.password),
406 params={'fetch-state': '1'})
407 if args.verbosity >= 2:
408 print_to_stderr("DEBUG: - Request URL {}"
410 print_to_stderr("DEBUG: - Request Text")
411 print_to_stderr("{}".format(req.text))
412 print_to_stderr("DEBUG: - Response Code {}"
413 .format(req.status_code))
414 if req.status_code != 200:
415 raise RuntimeError("ERROR:Fetching IP's of simulation - "
416 "Status other than 200 HTTP OK:\n{}"
417 .format(req.content))
418 except (requests.exceptions.RequestException,
419 RuntimeError) as ex_error:
420 print_to_stderr(ex_error)
422 req = requests.get('http://' + args.virl_ip +
423 '/simengine/rest/stop/' + session_id,
424 auth=(args.username, args.password))
426 shutil.rmtree(scratch_directory)
428 print_to_stderr("ERROR: Removing scratch directory")
429 print "{}".format(session_id)
433 # Populate node addresses
437 nodetype = re.split('[0-9]', key)[0]
438 if not nodetype in nodeaddrs:
439 nodeaddrs[nodetype] = {}
440 nodeaddrs[nodetype][key] = re.split('\\/', \
441 data[session_id][key]['management']['ip-address'])[0]
442 if args.verbosity >= 2:
443 print_to_stderr("DEBUG: Node {} is of type {} and has mgmt IP {}"
444 .format(key, nodetype, nodeaddrs[nodetype][key]))
447 for key2 in data[session_id][key]:
448 topology[key]["nic-"+key2] = data[session_id][key][key2]
449 if 'ip-address' in topology[key]["nic-"+key2]:
450 if topology[key]["nic-"+key2]['ip-address'] is not None:
451 topology[key]["nic-"+key2]['ip-addr'] = re.split('\\/', \
452 topology[key]["nic-"+key2]['ip-address'])[0]
455 ansiblehosts = open(os.path.join(scratch_directory, 'ansible-hosts'), 'w')
456 for key1 in nodeaddrs:
457 ansiblehosts.write("[{}]\n".format(key1))
458 for key2 in nodeaddrs[key1]:
459 ansiblehosts.write("{} hostname={}\n".format(nodeaddrs[key1][key2],
463 # Process topology YAML template
464 with open(args.ssh_privkey, 'r') as privkey_file:
465 priv_key = indent(privkey_file.read(), 6)
467 with open(os.path.join(scratch_directory, "topology.yaml"), 'w') as \
468 new_file, open(topology_yaml_filename, 'r') as old_file:
469 for line in old_file:
470 new_file.write(line.format(priv_key=priv_key, topology=topology))
473 # Wait for hosts to become reachable over SSH
475 if args.verbosity >= 1:
476 print_to_stderr("DEBUG: Waiting for hosts to become reachable over SSH")
479 count = args.wait_count
480 while (count > 0) and missing != 0:
481 time.sleep(args.wait_time)
486 if not os.path.exists(os.path.join(scratch_directory, key)):
488 if args.verbosity >= 2:
489 print_to_stderr("DEBUG: Attempt {} out of {}, waiting for {} hosts"
490 .format(args.wait_count-count, args.wait_count,
494 print_to_stderr("ERROR: Simulation started OK but {} hosts never "
495 "mounted their NFS directory".format(missing))
497 req = requests.get('http://' + args.virl_ip +
498 '/simengine/rest/stop/' + session_id,
499 auth=(args.username, args.password))
501 shutil.rmtree(scratch_directory)
503 print_to_stderr("ERROR: Removing scratch directory")
504 print "{}".format(session_id)
510 if not args.skip_install:
511 if args.verbosity >= 1:
512 print_to_stderr("DEBUG: Uprading VPP")
514 for key1 in nodeaddrs:
516 for key2 in nodeaddrs[key1]:
517 ipaddr = nodeaddrs[key1][key2]
518 if args.verbosity >= 2:
519 print_to_stderr("DEBUG: Upgrading VPP on node {}"
521 paramiko.util.log_to_file(os.path.join(scratch_directory,
523 client = paramiko.SSHClient()
524 client.load_system_host_keys()
525 client.load_host_keys("/dev/null")
526 client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
527 client.connect(ipaddr, username=args.ssh_user,
528 key_filename=args.ssh_privkey)
529 if 'centos' in args.topology:
530 if args.verbosity >= 1:
531 print_to_stderr("DEBUG: Installing RPM packages")
532 vpp_install_command = 'sudo rpm -ivh /scratch/vpp/*.rpm'
533 elif 'trusty' in args.topology or 'xenial' in args.topology:
534 if args.verbosity >= 1:
535 print_to_stderr("DEBUG: Installing DEB packages")
536 vpp_install_command = 'sudo dpkg -i --force-all ' \
539 print_to_stderr("ERROR: Unsupported OS requested: {}"
540 .format(args.topology))
541 vpp_install_command = ''
542 _, stdout, stderr = \
543 client.exec_command(vpp_install_command)
544 c_stdout = stdout.read()
545 c_stderr = stderr.read()
546 if args.verbosity >= 2:
547 print_to_stderr("DEBUG: Command output was:")
548 print_to_stderr(c_stdout)
549 print_to_stderr("DEBUG: Command stderr was:")
550 print_to_stderr(c_stderr)
553 # Write a file with timestamp to scratch directory. We can use this to track
554 # how long a simulation has been running.
556 with open(os.path.join(scratch_directory, 'start_time'), 'a') as \
558 timestampfile.write('{}\n'.format(int(time.time())))
563 if args.verbosity >= 1:
564 print_to_stderr("SESSION ID: {}".format(session_id))
566 print "{}".format(session_id)
568 if __name__ == "__main__":