- for node in nodes:
- if node['type'] == NodeType.DUT:
- HoneycombSetup.print_ports(node)
- try:
- status_code, _ = HTTPRequest.get(node, path,
- enable_logging=False)
- except HTTPRequestError:
- ssh = SSH()
- ssh.connect(node)
- ssh.exec_command("tail -n 100 /var/log/syslog")
- raise
- if status_code == HTTPCodes.OK:
- logger.info("Honeycomb on node {0} is up and running".
- format(node['host']))
- elif status_code in expected_status_codes:
- if status_code == HTTPCodes.UNAUTHORIZED:
- logger.info('Unauthorized. If this triggers keyword '
- 'timeout, verify Honeycomb username and '
- 'password.')
- raise HoneycombError('Honeycomb on node {0} running but '
- 'not yet ready.'.format(node['host']),
- enable_logging=False)
- else:
- raise HoneycombError('Unexpected return code: {0}.'.
- format(status_code))
-
- status_code, _ = HcUtil.get_honeycomb_data(
- node, "config_vpp_interfaces")
- if status_code != HTTPCodes.OK:
- raise HoneycombError('Honeycomb on node {0} running but '
- 'not yet ready.'.format(node['host']),
- enable_logging=False)
- return True
+ ssh = SSH()
+ ssh.connect(node)
+ ret_code, pid, _ = ssh.exec_command("pgrep honeycomb")
+ if ret_code != 0:
+ raise HoneycombError("No process named 'honeycomb' found.")
+
+ pid = int(pid)
+ count = 0
+ start = time()
+ while time() - start < timeout and count < retries:
+ count += 1
+ ret_code, _, _ = ssh.exec_command(
+ " | ".join([
+ "sudo tail -n 1000 /var/log/syslog",
+ "grep {pid}".format(pid=pid),
+ "grep 'Honeycomb started successfully!'"])
+ )
+ if ret_code != 0:
+ logger.debug(
+ "Attempt #{count} failed on log check.".format(
+ count=count))
+ sleep(interval)
+ continue
+ status_code_version, _ = HcUtil.get_honeycomb_data(
+ node, "oper_vpp_version")
+ status_code_if_cfg, _ = HcUtil.get_honeycomb_data(
+ node, "config_vpp_interfaces")
+ status_code_if_oper, _ = HcUtil.get_honeycomb_data(
+ node, "oper_vpp_interfaces")
+ if status_code_if_cfg == HTTPCodes.OK\
+ and status_code_if_cfg == HTTPCodes.OK\
+ and status_code_if_oper == HTTPCodes.OK:
+ logger.info("Check successful, Honeycomb is up and running.")
+ break
+ else:
+ logger.debug(
+ "Attempt ${count} failed on Restconf check. Status codes:\n"
+ "Version: {version}\n"
+ "Interface config: {if_cfg}\n"
+ "Interface operational: {if_oper}".format(
+ count=count,
+ version=status_code_version,
+ if_cfg=status_code_if_cfg,
+ if_oper=status_code_if_oper))
+ sleep(interval)
+ continue
+ else:
+ _, vpp_status, _ = ssh.exec_command("service vpp status")
+ ret_code, hc_log, _ = ssh.exec_command(
+ " | ".join([
+ "sudo tail -n 1000 /var/log/syslog",
+ "grep {pid}".format(pid=pid)]))
+ raise HoneycombError(
+ "Timeout or max retries exceeded. Status of VPP:\n"
+ "{vpp_status}\n"
+ "Syslog entries filtered by Honeycomb's pid:\n"
+ "{hc_log}".format(vpp_status=vpp_status, hc_log=hc_log))