From ee52db86af140da3c919476dcd1441acdc9e7600 Mon Sep 17 00:00:00 2001 From: selias Date: Wed, 9 Aug 2017 10:50:46 +0200 Subject: [PATCH] HC Test: cleanup test failures - have unnumbered interface cases use a real interface instead of local0 - add EXPECTED_FAILING tag and Jira link to known failing test cases Change-Id: I8bc0f4a74ea1949289a3f12dd54168acdf9edc39 Signed-off-by: selias --- .../libraries/python/honeycomb/HoneycombSetup.py | 4 +-- .../mgmt-cfg-int-subint-apihc-apivat-func.robot | 41 ++++++++++++---------- .../mgmt-cfg-intip4-intip6-apihc-apivat-func.robot | 30 ++++++++++------ .../mgmt-cfg-routing-apihc-apivat-func.robot | 4 +++ .../func/honeycomb/mgmt-notif-apihcnc-func.robot | 3 +- 5 files changed, 49 insertions(+), 33 deletions(-) diff --git a/resources/libraries/python/honeycomb/HoneycombSetup.py b/resources/libraries/python/honeycomb/HoneycombSetup.py index bab24aac35..55eb64e8f4 100644 --- a/resources/libraries/python/honeycomb/HoneycombSetup.py +++ b/resources/libraries/python/honeycomb/HoneycombSetup.py @@ -464,7 +464,7 @@ class HoneycombSetup(object): cmd = "cp -r {src}/*karaf_{odl_name}* {dst}".format( src=src_path, odl_name=odl_name, dst=dst_path) - ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=60) + ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=120) if int(ret_code) != 0: raise HoneycombError( "Failed to copy ODL client on node {0}".format(node["host"])) @@ -516,7 +516,7 @@ class HoneycombSetup(object): for feature in features: cmd += " {0}".format(feature) - ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=120) + ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=180) if int(ret_code) != 0: raise HoneycombError("Feature install did not succeed.") diff --git a/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot b/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot index a37f538d2d..b1a4304f87 100644 --- a/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot +++ b/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot @@ -19,9 +19,7 @@ | Resource | resources/libraries/robot/honeycomb/interfaces.robot | Variables | resources/test_data/honeycomb/sub_interfaces.py | ... -| Suite Setup | Run Keywords -| ... | Set Up Honeycomb Functional Test Suite | ${node} | AND -| ... | Add Interface local0 To Topology | ${node} +| Suite Setup | Set Up Honeycomb Functional Test Suite | ${node} | ... | Suite Teardown | Tear Down Honeycomb Functional Test Suite | ${node} | ... @@ -471,20 +469,22 @@ | TC22: Honeycomb can configure unnumbered sub-interface | | [Documentation] | Check if Honeycomb can configure an unnumbered interface\ -| | ... | on a sub-interface, borrowing the IP address of 'local0'. +| | ... | on a sub-interface, borrowing the IP address of a physical interface. | | ... +# CSIT-734: Intermittent failures of all unnumbered interface cases +| | [Tags] | EXPECTED_FAILING | | Given sub-interface ipv4 address from Honeycomb should be empty | | ... | ${node} | ${super_if} | ${sub_if_id} | | And sub-interface ipv4 address from VAT should be empty | | ... | ${node} | ${sub_if_name} | | And Honeycomb sets interface IPv4 address | ${node} -| | ... | local0 | ${ipv4['address']} | ${ipv4['prefix-length']} +| | ... | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']} | | When Honeycomb adds unnumbered configuration to interface -| | ... | ${node} | ${super_if}.${sub_if_id} | local0 +| | ... | ${node} | ${super_if}.${sub_if_id} | ${super_if2} | | Then IPv4 address from Honeycomb should be -| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']} +| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']} | | And IPv4 address from VAT should be -| | ... | ${node} | local0 +| | ... | ${node} | ${super_if2} | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']} | | And sub-interface ipv4 address from Honeycomb should be | | ... | ${node} | ${super_if} | ${sub_if_id} @@ -497,13 +497,15 @@ | | [Documentation] | Check if Honeycomb can remove unnumbered configuration\ | | ... | from a sub-interface. | | ... +# CSIT-734: Intermittent failures of all unnumbered interface cases +| | [Tags] | EXPECTED_FAILING | | [Teardown] | Honeycomb removes interface IPv4 addresses | ${node} -| | ... | local0 +| | ... | ${super_if2} | | ... | | Given IPv4 address from Honeycomb should be -| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']} +| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']} | | And IPv4 address from VAT should be -| | ... | ${node} | local0 +| | ... | ${node} | ${super_if2} | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']} | | And sub-interface ipv4 address from Honeycomb should be | | ... | ${node} | ${super_if} | ${sub_if_id} @@ -514,9 +516,9 @@ | | When Honeycomb removes unnumbered configuration from interface | | ... | ${node} | ${super_if}.${sub_if_id} | | Then IPv4 address from Honeycomb should be -| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']} +| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']} | | And IPv4 address from VAT should be -| | ... | ${node} | local0 +| | ... | ${node} | ${super_if2} | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']} | | And sub-interface ipv4 address from Honeycomb should be empty | | ... | ${node} | ${super_if} | ${sub_if_id} @@ -526,9 +528,10 @@ | TC24: Honeycomb can configure unnumbered interface using a sub-interface | | [Documentation] | Check if Honeycomb can configure an unnumbered interface\ | | ... | on an interface, borrowing the IP address of a sub-interface. -| | ... -| | Given IPv4 address from Honeycomb should be empty | ${node} | local0 -| | And ipv4 address from VAT should be empty | ${node} | local0 +| | ...# CSIT-734: Intermittent failures of all unnumbered interface cases +| | [Tags] | EXPECTED_FAILING +| | Given IPv4 address from Honeycomb should be empty | ${node} | ${super_if2} +| | And ipv4 address from VAT should be empty | ${node} | ${super_if2} | | And sub-interface ipv4 address from Honeycomb should be empty | | ... | ${node} | ${super_if} | ${sub_if_id} | | And sub-interface ipv4 address from VAT should be empty @@ -537,11 +540,11 @@ | | ... | ${node} | ${super_if} | ${sub_if_id} | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | | When Honeycomb adds unnumbered configuration to interface -| | ... | ${node} | local0 | ${super_if}.${sub_if_id} +| | ... | ${node} | ${super_if2} | ${super_if}.${sub_if_id} | | Then IPv4 address from Honeycomb should be -| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']} +| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']} | | And IPv4 address from VAT should be -| | ... | ${node} | local0 +| | ... | ${node} | ${super_if2} | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']} | | And sub-interface ipv4 address from Honeycomb should be | | ... | ${node} | ${super_if} | ${sub_if_id} diff --git a/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot b/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot index f0f6e4e6ad..2266509f61 100644 --- a/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot +++ b/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot @@ -14,6 +14,7 @@ *** Variables *** # Interface to run tests on. | ${interface}= | ${node['interfaces']['port1']['name']} +| ${interface2}= | ${node['interfaces']['port3']['name']} *** Settings *** | Resource | resources/libraries/robot/shared/default.robot @@ -218,16 +219,20 @@ | TC11: Honeycomb can configure unnumbered interface | | [Documentation] | Check if Honeycomb can configure an unnumbered interface\ -| | ... | on a physical interface, borrowing the IP address of 'local0'. +| | ... | on a physical interface, borrowing the IP address of another physical\ +| | ... | interface. | | ... +# CSIT-734: Intermittent failures of all unnumbered interface cases +| | [Tags] | EXPECTED_FAILING | | Given Honeycomb sets interface IPv4 address | ${node} -| | ... | local0 | ${ipv4_address} | ${ipv4_prefix} +| | ... | ${interface2} | ${ipv4_address} | ${ipv4_prefix} | | When Honeycomb adds unnumbered configuration to interface -| | ... | ${node} | ${interface} | local0 -| | Then IPv4 address from Honeycomb should be -| | ... | ${node} | local0 | ${ipv4_address} | ${ipv4_prefix} +| | ... | ${node} | ${interface} | ${interface2} +| | Then Wait until Keyword succeeds | 10s | 2s +| | ... | IPv4 address from Honeycomb should be +| | ... | ${node} | ${interface2} | ${ipv4_address} | ${ipv4_prefix} | | And IPv4 address from VAT should be -| | ... | ${node} | local0 | ${ipv4_address} +| | ... | ${node} | ${interface2} | ${ipv4_address} | | ... | ${ipv4_prefix} | ${ipv4_mask} | | And IPv4 address from Honeycomb should be | | ... | ${node} | ${interface} | ${ipv4_address} | ${ipv4_prefix} @@ -239,10 +244,12 @@ | | [Documentation] | Check if Honeycomb can remove unnumbered configuration\ | | ... | from an interface. | | ... +# CSIT-734: Intermittent failures of all unnumbered interface cases +| | [Tags] | EXPECTED_FAILING | | Given IPv4 address from Honeycomb should be -| | ... | ${node} | local0 | ${ipv4_address} | ${ipv4_prefix} +| | ... | ${node} | ${interface2} | ${ipv4_address} | ${ipv4_prefix} | | And IPv4 address from VAT should be -| | ... | ${node} | local0 | ${ipv4_address} +| | ... | ${node} | ${interface2} | ${ipv4_address} | | ... | ${ipv4_prefix} | ${ipv4_mask} | | And IPv4 address from Honeycomb should be | | ... | ${node} | ${interface} | ${ipv4_address} | ${ipv4_prefix} @@ -251,10 +258,11 @@ | | ... | ${ipv4_prefix} | ${ipv4_mask} | | When Honeycomb removes unnumbered configuration from interface | | ... | ${node} | ${interface} -| | Then IPv4 address from Honeycomb should be -| | ... | ${node} | local0 | ${ipv4_address} | ${ipv4_prefix} +| | Then Wait until Keyword succeeds | 10s | 2s +| | ... | IPv4 address from Honeycomb should be +| | ... | ${node} | ${interface2} | ${ipv4_address} | ${ipv4_prefix} | | And IPv4 address from VAT should be -| | ... | ${node} | local0 | ${ipv4_address} +| | ... | ${node} | ${interface2} | ${ipv4_address} | | ... | ${ipv4_prefix} | ${ipv4_mask} | | And IPv4 address from Honeycomb should be empty | ${node} | ${interface} | | And ipv4 address from VAT should be empty | ${node} | ${interface} diff --git a/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot b/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot index 9f2dbc0ae6..404ba888a3 100644 --- a/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot +++ b/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot @@ -113,6 +113,8 @@ | | ... | [Ver] Send ICMP packet from first TG interface to configured route | | ... | destination. Receive packet on the second TG interface. | | ... +# VPP-930: ip6_fib_dump shows incorrect outgoing interface index +| | [Tags] | EXPECTED_FAILING | | ${table}= | Set Variable | table4 | | Given Setup interfaces and neighbors for IPv6 routing test | | When Honeycomb configures routing table @@ -135,6 +137,8 @@ | | ... | verify that each destination MAC was used by exactly 50 packets. | | ... | Receive packet on the second TG interface. | | ... +# VPP-930: ip6_fib_dump shows incorrect outgoing interface index +| | [Tags] | EXPECTED_FAILING | | ${table}= | Set Variable | table5 | | Given Setup interfaces and neighbors for IPv6 routing test | | And Honeycomb adds interface IPv6 neighbor | ${dut_node} | ${dut_to_tg_if2} diff --git a/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot b/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot index 5320a7fcb7..22a65b2e39 100644 --- a/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot +++ b/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot @@ -31,7 +31,8 @@ | ... | Suite Teardown | Tear Down Honeycomb Functional Test Suite | ${node} | ... -| Force Tags | HC_FUNC +# HC2VPP-203 Notifications are not being sent since VPP 17.10 +| Force Tags | HC_FUNC | EXPECTED_FAILING *** Test Cases *** | TC01: Honeycomb sends notification on interface state change -- 2.16.6