3 # Copyright (c) 2016 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 """This script is handling starting of VIRL simulations."""
18 __author__ = 'ckoester@cisco.com'
33 IPS_PER_SIMULATION = 5
35 def indent(lines, amount, fillchar=' '):
36 """Indent the string by amount of fill chars.
38 :param lines: String to indent.
39 :param amount: Number of fill chars.
40 :param fillchar: Filling character.
44 :returns: Indented string.
47 padding = amount * fillchar
48 return padding + ('\n'+padding).join(lines.split('\n'))
50 def print_to_stderr(msg, end='\n'):
51 """Writes any text to stderr.
53 :param msg: Message to print.
54 :param end: By default print new line at the end.
59 sys.stderr.write(str(msg) + end)
63 def get_assigned_interfaces(args, network="flat"):
64 """Retrieve assigned interfaces in openstack network.
66 :param args: Command line params.
67 :param network: Openstack network.
68 :type args: ArgumentParser
70 :returns: Assigned interfaces.
72 :raises RuntimeError: If response is not 200.
74 req = requests.get('http://{}/openstack/rest/ports/{}'
75 .format(args.virl_ip, network),
76 auth=(args.username, args.password))
77 if req.status_code == 200:
80 raise RuntimeError("ERROR: Retrieving ports in use - "
81 "Status other than 200 HTTP OK:\n{}"
84 def get_assigned_interfaces_count(args, network="flat"):
85 """Count assigned interfaces in openstack network.
87 :param args: Command line params.
88 :param network: Openstack network.
89 :type args: ArgumentParser
91 :returns: Assigned interfaces count.
94 return len(get_assigned_interfaces(args, network=network))
96 def check_ip_addresses(args):
97 """Check IP address availability.
99 :param args: Command line params.
100 :type args: ArgumentParser
101 :raises RuntimeError: If not enough free addresses available.
103 for i in range(args.wait_count):
105 get_assigned_interfaces_count(args) >= IPS_PER_SIMULATION):
107 if args.verbosity >= 2:
108 print_to_stderr("DEBUG: - Attempt {} out of {}, waiting for free "
109 "IP addresses".format(i, args.wait_count))
110 # Wait random amount of time within range 1-3 minutes
111 time.sleep(random.randint(60,180))
113 raise RuntimeError("ERROR: Not enough IP addresses to run simulation")
115 def check_virl_resources(args):
116 """Check virl resources availability.
118 :param args: Command line params.
119 :type args: ArgumentParser
121 check_ip_addresses(args)
124 # FIXME: Right now, this is really coded like a shell script, as one big
125 # function executed in sequence. This should be broken down into multiple
129 """ Main function."""
131 # Get our default interface IP address. This will become the default
132 # value for the "NFS Server IP" option.
134 gws = netifaces.gateways()
135 addrs = netifaces.ifaddresses(gws['default'][netifaces.AF_INET][1])
136 default_addr = addrs[netifaces.AF_INET][0]['addr']
139 # Verify CLI parameters and try to download our VPP image into a temporary
142 parser = argparse.ArgumentParser()
143 parser.add_argument("topology", help="the base topology to be started")
144 parser.add_argument("packages", help="Path to the VPP .deb(s) or .rpm(s) " +
145 "that is/are to be installed", nargs='+')
146 parser.add_argument("-c", "--copy", help="Copy the VPP packages, " +
147 "leaving the originals in place. Default is to " +
148 "move them.", action='store_true')
149 parser.add_argument("-k", "--keep", help="Keep (do not delete) the " +
150 "simulation in case of error", action='store_true')
151 parser.add_argument("-v", "--verbosity", action="count", default=0)
152 parser.add_argument("-nip", "--nfs-server-ip", help="NFS server (our) IP " +
153 "default is derived from routing table: " +
154 "{}".format(default_addr), default=default_addr)
155 parser.add_argument("-ns", "--nfs-scratch-directory",
156 help="Server location for NFS scratch directory",
157 default="/nfs/scratch")
158 parser.add_argument("-nc", "--nfs-common-directory",
159 help="Server location for NFS common (read-only) " +
160 "directory", default="/nfs/common")
161 parser.add_argument("-wc", "--wait-count",
162 help="number of intervals to wait for simulation to " +
163 "be ready", type=int, default=48)
164 parser.add_argument("-wt", "--wait-time",
165 help="length of a single interval to wait for " +
166 "simulation to be ready", type=int, default=5)
167 parser.add_argument("-vip", "--virl-ip",
168 help="VIRL IP and Port (e.g. 127.0.0.1:19399)",
169 default="127.0.0.1:19399")
170 parser.add_argument("-u", "--username", help="VIRL username",
172 parser.add_argument("-au", "--admin-username", help="VIRL admin username",
174 parser.add_argument("-p", "--password", help="VIRL password",
176 parser.add_argument("-su", "--ssh-user", help="SSH username",
178 parser.add_argument("-e", "--expiry", help="Simulation expiry",
180 parser.add_argument("-spr", "--ssh-privkey", help="SSH private keyfile",
181 default="/home/jenkins-in/.ssh/id_rsa_virl")
182 parser.add_argument("-spu", "--ssh-pubkey", help="SSH public keyfile",
183 default="/home/jenkins-in/.ssh/id_rsa_virl.pub")
184 parser.add_argument("-r", "--release", help="VM disk image/release " +
185 "(ex. \"csit-ubuntu-16.04.1_2016-12-19_1.6\")",
186 default="csit-ubuntu-16.04.1_2016-12-19_1.6")
187 parser.add_argument("--topology-directory", help="Topology directory",
188 default="/home/jenkins-in/testcase-infra/topologies")
189 parser.add_argument("-q", "--quota",
190 help="VIRL quota for max number of allowed IPs",
191 type=int, default=74)
193 args = parser.parse_args()
196 # Check if topology and template exist
198 if args.verbosity >= 2:
199 print_to_stderr("DEBUG: Running with topology {}"
200 .format(args.topology))
202 topology_virl_filename = os.path.join(args.topology_directory,
203 args.topology + ".virl")
204 topology_yaml_filename = os.path.join(args.topology_directory,
205 args.topology + ".yaml")
207 if not os.path.isfile(topology_virl_filename):
208 print_to_stderr("ERROR: Topology VIRL file {} does not exist"
209 .format(topology_virl_filename))
211 if not os.path.isfile(topology_yaml_filename):
212 print_to_stderr("ERROR: Topology YAML file {} does not exist"
213 .format(topology_yaml_filename))
217 # Check if VPP package exists
219 for package in args.packages:
220 if args.verbosity >= 2:
221 print_to_stderr("DEBUG: Checking if file {} exists"
223 if not os.path.isfile(package):
224 print_to_stderr("ERROR: Debian package {} does not exist"
229 # Start VIRL topology
231 if args.verbosity >= 1:
232 print_to_stderr("DEBUG: Starting VIRL topology")
233 temp_handle, temp_topology = tempfile.mkstemp()
234 with open(args.ssh_pubkey, 'r') as pubkey_file:
235 pub_key = pubkey_file.read().replace('\n', '')
236 with open(temp_topology, 'w') as new_file, \
237 open(topology_virl_filename, 'r') as old_file:
238 for line in old_file:
239 line = line.replace(" - VIRL-USER-SSH-PUBLIC-KEY", " - "+pub_key)
240 line = line.replace("$$NFS_SERVER_SCRATCH$$",
241 args.nfs_server_ip+":"+args.nfs_scratch_directory)
242 line = line.replace("$$NFS_SERVER_COMMON$$",
243 args.nfs_server_ip+":"+args.nfs_common_directory)
244 line = line.replace("$$VM_IMAGE$$", "server-"+args.release)
246 os.close(temp_handle)
249 data = open(temp_topology, 'rb')
250 check_virl_resources(args)
251 req = requests.post('http://' + args.virl_ip + '/simengine/rest/launch',
252 auth=(args.username, args.password),
254 if args.verbosity >= 2:
255 print_to_stderr("DEBUG: - Request URL {}"
259 print_to_stderr("DEBUG: - Response Code {}"
260 .format(req.status_code))
262 if req.status_code != 200:
263 raise RuntimeError("ERROR: Launching VIRL simulation - "
264 "Status other than 200 HTTP OK:\n{}"
265 .format(req.content))
266 except (requests.exceptions.RequestException,
267 RuntimeError) as ex_error:
268 print_to_stderr(ex_error)
269 os.remove(temp_topology)
272 # If we got here, we had a good response. The response content is the
274 session_id = req.content
275 if args.verbosity >= 1:
276 print_to_stderr("DEBUG: VIRL simulation session-id: {}"
279 # Set session expiry to autokill sessions if not done from jenkins
281 if args.verbosity >= 1:
282 print_to_stderr("DEBUG: Setting expire for session-id: {}"
285 req = requests.put('http://' + args.virl_ip +
286 '/simengine/rest/admin-update/' + session_id +
288 auth=(args.admin_username, args.password),
289 params={'user': args.username,
290 'expires': args.expiry})
291 if args.verbosity >= 2:
292 print_to_stderr("DEBUG: - Request URL {}"
296 print_to_stderr("DEBUG: - Response Code {}"
297 .format(req.status_code))
298 if req.status_code != 200:
299 raise RuntimeError("ERROR: Setting expiry to simulation - "
300 "Status other than 200 HTTP OK:\n{}"
301 .format(req.content))
302 except (requests.exceptions.RequestException,
303 RuntimeError) as ex_error:
304 print_to_stderr(ex_error)
305 req = requests.get('http://' + args.virl_ip +
306 '/simengine/rest/stop/' + session_id,
307 auth=(args.username, args.password))
308 os.remove(temp_topology)
309 print "{}".format(session_id)
313 # Create simulation scratch directory. Move topology file into that
314 # directory. Copy or move debian packages into that directory.
316 scratch_directory = os.path.join(args.nfs_scratch_directory, session_id)
317 os.mkdir(scratch_directory)
318 shutil.move(temp_topology, os.path.join(scratch_directory,
319 "virl_topology.virl"))
320 os.mkdir(os.path.join(scratch_directory, "vpp"))
321 for package in args.packages:
323 shutil.copy(package, os.path.join(scratch_directory, "vpp",
324 os.path.basename(package)))
326 shutil.move(package, os.path.join(scratch_directory, "vpp",
327 os.path.basename(package)))
330 # Wait for simulation to become active
332 if args.verbosity >= 1:
333 print_to_stderr("DEBUG: Waiting for simulation to become active")
335 sim_is_started = False
338 count = args.wait_count
339 while (count > 0) and not sim_is_started:
340 time.sleep(args.wait_time)
343 req = requests.get('http://' + args.virl_ip + '/simengine/rest/nodes/' +
344 session_id, auth=(args.username, args.password))
350 # Flush the node list every time, keep the last one
353 # Hosts are the keys of the inner dictionary
354 for key in data[session_id].keys():
355 if data[session_id][key]['management-proxy'] == "self":
359 if data[session_id][key]['state'] == "ACTIVE":
361 if args.verbosity >= 2:
362 print_to_stderr("DEBUG: - Attempt {} out of {}, total {} hosts, "
363 "{} active".format(args.wait_count-count,
364 args.wait_count, total, active))
366 sim_is_started = True
368 if not sim_is_started:
369 print_to_stderr("ERROR: Simulation nodes never changed to ACTIVE state")
370 print_to_stderr("Last VIRL response:")
371 print_to_stderr(data)
373 req = requests.get('http://' + args.virl_ip +
374 '/simengine/rest/stop/' + session_id,
375 auth=(args.username, args.password))
377 shutil.rmtree(scratch_directory)
379 print_to_stderr("ERROR: Removing scratch directory")
380 print "{}".format(session_id)
383 if args.verbosity >= 2:
384 print_to_stderr("DEBUG: Nodes: {}"
385 .format(", ".join(nodelist)))
388 # Fetch simulation's IPs and create files
389 # (ansible hosts file, topology YAML file)
392 req = requests.get('http://' + args.virl_ip +
393 '/simengine/rest/interfaces/' + session_id,
394 auth=(args.username, args.password),
395 params={'fetch-state': '1'})
396 if args.verbosity >= 2:
397 print_to_stderr("DEBUG: - Request URL {}"
399 print_to_stderr("DEBUG: - Request Text")
400 print_to_stderr("{}".format(req.text))
401 print_to_stderr("DEBUG: - Response Code {}"
402 .format(req.status_code))
403 if req.status_code != 200:
404 raise RuntimeError("ERROR:Fetching IP's of simulation - "
405 "Status other than 200 HTTP OK:\n{}"
406 .format(req.content))
407 except (requests.exceptions.RequestException,
408 RuntimeError) as ex_error:
409 print_to_stderr(ex_error)
411 req = requests.get('http://' + args.virl_ip +
412 '/simengine/rest/stop/' + session_id,
413 auth=(args.username, args.password))
415 shutil.rmtree(scratch_directory)
417 print_to_stderr("ERROR: Removing scratch directory")
418 print "{}".format(session_id)
422 # Populate node addresses
426 nodetype = re.split('[0-9]', key)[0]
427 if not nodetype in nodeaddrs:
428 nodeaddrs[nodetype] = {}
429 nodeaddrs[nodetype][key] = re.split('\\/', \
430 data[session_id][key]['management']['ip-address'])[0]
431 if args.verbosity >= 2:
432 print_to_stderr("DEBUG: Node {} is of type {} and has mgmt IP {}"
433 .format(key, nodetype, nodeaddrs[nodetype][key]))
436 for key2 in data[session_id][key]:
437 topology[key]["nic-"+key2] = data[session_id][key][key2]
438 if 'ip-address' in topology[key]["nic-"+key2]:
439 if topology[key]["nic-"+key2]['ip-address'] is not None:
440 topology[key]["nic-"+key2]['ip-addr'] = re.split('\\/', \
441 topology[key]["nic-"+key2]['ip-address'])[0]
444 ansiblehosts = open(os.path.join(scratch_directory, 'ansible-hosts'), 'w')
445 for key1 in nodeaddrs:
446 ansiblehosts.write("[{}]\n".format(key1))
447 for key2 in nodeaddrs[key1]:
448 ansiblehosts.write("{} hostname={}\n".format(nodeaddrs[key1][key2],
452 # Process topology YAML template
453 with open(args.ssh_privkey, 'r') as privkey_file:
454 priv_key = indent(privkey_file.read(), 6)
456 with open(os.path.join(scratch_directory, "topology.yaml"), 'w') as \
457 new_file, open(topology_yaml_filename, 'r') as old_file:
458 for line in old_file:
459 new_file.write(line.format(priv_key=priv_key, topology=topology))
462 # Wait for hosts to become reachable over SSH
464 if args.verbosity >= 1:
465 print_to_stderr("DEBUG: Waiting for hosts to become reachable over SSH")
468 count = args.wait_count
469 while (count > 0) and missing != 0:
470 time.sleep(args.wait_time)
475 if not os.path.exists(os.path.join(scratch_directory, key)):
477 if args.verbosity >= 2:
478 print_to_stderr("DEBUG: Attempt {} out of {}, waiting for {} hosts"
479 .format(args.wait_count-count, args.wait_count,
483 print_to_stderr("ERROR: Simulation started OK but {} hosts never "
484 "mounted their NFS directory".format(missing))
486 req = requests.get('http://' + args.virl_ip +
487 '/simengine/rest/stop/' + session_id,
488 auth=(args.username, args.password))
490 shutil.rmtree(scratch_directory)
492 print_to_stderr("ERROR: Removing scratch directory")
493 print "{}".format(session_id)
499 if args.verbosity >= 1:
500 print_to_stderr("DEBUG: Uprading VPP")
502 for key1 in nodeaddrs:
504 for key2 in nodeaddrs[key1]:
505 ipaddr = nodeaddrs[key1][key2]
506 if args.verbosity >= 2:
507 print_to_stderr("DEBUG: Upgrading VPP on node {}"
509 paramiko.util.log_to_file(os.path.join(scratch_directory,
511 client = paramiko.SSHClient()
512 client.load_system_host_keys()
513 client.load_host_keys("/dev/null")
514 client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
515 client.connect(ipaddr, username=args.ssh_user,
516 key_filename=args.ssh_privkey)
517 if 'centos' in args.topology:
518 if args.verbosity >= 1:
519 print_to_stderr("DEBUG: Installing RPM packages")
520 vpp_install_command = 'sudo rpm -ivh /scratch/vpp/*.rpm'
521 elif 'trusty' in args.topology or 'xenial' in args.topology:
522 if args.verbosity >= 1:
523 print_to_stderr("DEBUG: Installing DEB packages")
524 vpp_install_command = 'sudo dpkg -i --force-all ' \
527 print_to_stderr("ERROR: Unsupported OS requested: {}"
528 .format(args.topology))
529 vpp_install_command = ''
530 _, stdout, stderr = \
531 client.exec_command(vpp_install_command)
532 c_stdout = stdout.read()
533 c_stderr = stderr.read()
534 if args.verbosity >= 2:
535 print_to_stderr("DEBUG: Command output was:")
536 print_to_stderr(c_stdout)
537 print_to_stderr("DEBUG: Command stderr was:")
538 print_to_stderr(c_stderr)
541 # Write a file with timestamp to scratch directory. We can use this to track
542 # how long a simulation has been running.
544 with open(os.path.join(scratch_directory, 'start_time'), 'a') as \
546 timestampfile.write('{}\n'.format(int(time.time())))
551 if args.verbosity >= 1:
552 print_to_stderr("SESSION ID: {}".format(session_id))
554 print "{}".format(session_id)
556 if __name__ == "__main__":