Code Review / csit.git / commitdiff
? search:
summary | shortlog | log | commit | commitdiff | review | tree
raw | patch | inline | side by side (from parent 1: 8cde871)
HC Test: cleanup test failures 53/7953/6
authorselias <samelias@cisco.com>
Wed, 9 Aug 2017 08:50:46 +0000 (10:50 +0200)
committerTibor Frank <tifrank@cisco.com>
Tue, 15 Aug 2017 11:00:41 +0000 (11:00 +0000)
 - have unnumbered interface cases use a real interface instead of local0
 - add EXPECTED_FAILING tag and Jira link to known failing test cases

Change-Id: I8bc0f4a74ea1949289a3f12dd54168acdf9edc39
Signed-off-by: selias <samelias@cisco.com>
resources/libraries/python/honeycomb/HoneycombSetup.py patch | blob | history
tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot patch | blob | history
tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot patch | blob | history
tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot patch | blob | history
tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot patch | blob | history

diff --git a/resources/libraries/python/honeycomb/HoneycombSetup.py b/resources/libraries/python/honeycomb/HoneycombSetup.py
index bab24aa..55eb64e 100644 (file)
--- a/resources/libraries/python/honeycomb/HoneycombSetup.py
+++ b/resources/libraries/python/honeycomb/HoneycombSetup.py
@@ -464,7 +464,7 @@ class HoneycombSetup(object):
         cmd = "cp -r {src}/*karaf_{odl_name}* {dst}".format(
             src=src_path, odl_name=odl_name, dst=dst_path)
 
-        ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=60)
+        ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=120)
         if int(ret_code) != 0:
             raise HoneycombError(
                 "Failed to copy ODL client on node {0}".format(node["host"]))
@@ -516,7 +516,7 @@ class HoneycombSetup(object):
         for feature in features:
             cmd += " {0}".format(feature)
 
-        ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=120)
+        ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=180)
 
         if int(ret_code) != 0:
             raise HoneycombError("Feature install did not succeed.")
diff --git a/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot b/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot
index a37f538..b1a4304 100644 (file)
--- a/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot
+++ b/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot
@@ -19,9 +19,7 @@
 | Resource | resources/libraries/robot/honeycomb/interfaces.robot
 | Variables | resources/test_data/honeycomb/sub_interfaces.py
 | ...
-| Suite Setup | Run Keywords
-| ... | Set Up Honeycomb Functional Test Suite | ${node} | AND
-| ... | Add Interface local0 To Topology | ${node}
+| Suite Setup | Set Up Honeycomb Functional Test Suite | ${node}
 | ...
 | Suite Teardown | Tear Down Honeycomb Functional Test Suite | ${node}
 | ...
@@ -471,20 +469,22 @@
 
 | TC22: Honeycomb can configure unnumbered sub-interface
 | | [Documentation] | Check if Honeycomb can configure an unnumbered interface\
-| | ... | on a sub-interface, borrowing the IP address of 'local0'.
+| | ... | on a sub-interface, borrowing the IP address of a physical interface.
 | | ...
+# CSIT-734: Intermittent failures of all unnumbered interface cases
+| | [Tags] | EXPECTED_FAILING
 | | Given sub-interface ipv4 address from Honeycomb should be empty
 | | ... | ${node} | ${super_if} | ${sub_if_id}
 | | And sub-interface ipv4 address from VAT should be empty
 | | ... | ${node} | ${sub_if_name}
 | | And Honeycomb sets interface IPv4 address | ${node}
-| | ... | local0 | ${ipv4['address']} | ${ipv4['prefix-length']}
+| | ... | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | When Honeycomb adds unnumbered configuration to interface
-| | ... | ${node} | ${super_if}.${sub_if_id} | local0
+| | ... | ${node} | ${super_if}.${sub_if_id} | ${super_if2}
 | | Then IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']}
+| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0
+| | ... | ${node} | ${super_if2}
 | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']}
 | | And sub-interface ipv4 address from Honeycomb should be
 | | ... | ${node} | ${super_if} | ${sub_if_id}
@@ -497,13 +497,15 @@
 | | [Documentation] | Check if Honeycomb can remove unnumbered configuration\
 | | ... | from a sub-interface.
 | | ...
+# CSIT-734: Intermittent failures of all unnumbered interface cases
+| | [Tags] | EXPECTED_FAILING
 | | [Teardown] | Honeycomb removes interface IPv4 addresses | ${node}
-| | ... | local0
+| | ... | ${super_if2}
 | | ...
 | | Given IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']}
+| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0
+| | ... | ${node} | ${super_if2}
 | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']}
 | | And sub-interface ipv4 address from Honeycomb should be
 | | ... | ${node} | ${super_if} | ${sub_if_id}
@@ -514,9 +516,9 @@
 | | When Honeycomb removes unnumbered configuration from interface
 | | ... | ${node} | ${super_if}.${sub_if_id}
 | | Then IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']}
+| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0
+| | ... | ${node} | ${super_if2}
 | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']}
 | | And sub-interface ipv4 address from Honeycomb should be empty
 | | ... | ${node} | ${super_if} | ${sub_if_id}
@@ -526,9 +528,10 @@
 | TC24: Honeycomb can configure unnumbered interface using a sub-interface
 | | [Documentation] | Check if Honeycomb can configure an unnumbered interface\
 | | ... | on an interface, borrowing the IP address of a sub-interface.
-| | ...
-| | Given IPv4 address from Honeycomb should be empty | ${node} | local0
-| | And ipv4 address from VAT should be empty | ${node} | local0
+| | ...# CSIT-734: Intermittent failures of all unnumbered interface cases
+| | [Tags] | EXPECTED_FAILING
+| | Given IPv4 address from Honeycomb should be empty | ${node} | ${super_if2}
+| | And ipv4 address from VAT should be empty | ${node} | ${super_if2}
 | | And sub-interface ipv4 address from Honeycomb should be empty
 | | ... | ${node} | ${super_if} | ${sub_if_id}
 | | And sub-interface ipv4 address from VAT should be empty
@@ -537,11 +540,11 @@
 | | ... | ${node} | ${super_if} | ${sub_if_id}
 | | ... | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | When Honeycomb adds unnumbered configuration to interface
-| | ... | ${node} | local0 | ${super_if}.${sub_if_id}
+| | ... | ${node} | ${super_if2} | ${super_if}.${sub_if_id}
 | | Then IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']}
+| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0
+| | ... | ${node} | ${super_if2}
 | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']}
 | | And sub-interface ipv4 address from Honeycomb should be
 | | ... | ${node} | ${super_if} | ${sub_if_id}
diff --git a/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot b/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot
index f0f6e4e..2266509 100644 (file)
--- a/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot
+++ b/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot
@@ -14,6 +14,7 @@
 *** Variables ***
 # Interface to run tests on.
 | ${interface}= | ${node['interfaces']['port1']['name']}
+| ${interface2}= | ${node['interfaces']['port3']['name']}
 
 *** Settings ***
 | Resource | resources/libraries/robot/shared/default.robot
@@ -218,16 +219,20 @@
 
 | TC11: Honeycomb can configure unnumbered interface
 | | [Documentation] | Check if Honeycomb can configure an unnumbered interface\
-| | ... | on a physical interface, borrowing the IP address of 'local0'.
+| | ... | on a physical interface, borrowing the IP address of another physical\
+| | ... | interface.
 | | ...
+# CSIT-734: Intermittent failures of all unnumbered interface cases
+| | [Tags] | EXPECTED_FAILING
 | | Given Honeycomb sets interface IPv4 address | ${node}
-| | ... | local0 | ${ipv4_address} | ${ipv4_prefix}
+| | ... | ${interface2} | ${ipv4_address} | ${ipv4_prefix}
 | | When Honeycomb adds unnumbered configuration to interface
-| | ... | ${node} | ${interface} | local0
-| | Then IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4_address} | ${ipv4_prefix}
+| | ... | ${node} | ${interface} | ${interface2}
+| | Then Wait until Keyword succeeds | 10s | 2s
+| | ... | IPv4 address from Honeycomb should be
+| | ... | ${node} | ${interface2} | ${ipv4_address} | ${ipv4_prefix}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0 | ${ipv4_address}
+| | ... | ${node} | ${interface2} | ${ipv4_address}
 | | ... | ${ipv4_prefix} | ${ipv4_mask}
 | | And IPv4 address from Honeycomb should be
 | | ... | ${node} | ${interface} | ${ipv4_address} | ${ipv4_prefix}
@@ -239,10 +244,12 @@
 | | [Documentation] | Check if Honeycomb can remove unnumbered configuration\
 | | ... | from an interface.
 | | ...
+# CSIT-734: Intermittent failures of all unnumbered interface cases
+| | [Tags] | EXPECTED_FAILING
 | | Given IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4_address} | ${ipv4_prefix}
+| | ... | ${node} | ${interface2} | ${ipv4_address} | ${ipv4_prefix}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0 | ${ipv4_address}
+| | ... | ${node} | ${interface2} | ${ipv4_address}
 | | ... | ${ipv4_prefix} | ${ipv4_mask}
 | | And IPv4 address from Honeycomb should be
 | | ... | ${node} | ${interface} | ${ipv4_address} | ${ipv4_prefix}
@@ -251,10 +258,11 @@
 | | ... | ${ipv4_prefix} | ${ipv4_mask}
 | | When Honeycomb removes unnumbered configuration from interface
 | | ... | ${node} | ${interface}
-| | Then IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4_address} | ${ipv4_prefix}
+| | Then Wait until Keyword succeeds | 10s | 2s
+| | ... | IPv4 address from Honeycomb should be
+| | ... | ${node} | ${interface2} | ${ipv4_address} | ${ipv4_prefix}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0 | ${ipv4_address}
+| | ... | ${node} | ${interface2} | ${ipv4_address}
 | | ... | ${ipv4_prefix} | ${ipv4_mask}
 | | And IPv4 address from Honeycomb should be empty | ${node} | ${interface}
 | | And ipv4 address from VAT should be empty | ${node} | ${interface}
diff --git a/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot b/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot
index 9f2dbc0..404ba88 100644 (file)
--- a/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot
+++ b/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot
@@ -113,6 +113,8 @@
 | | ... | [Ver] Send ICMP packet from first TG interface to configured route
 | | ... | destination. Receive packet on the second TG interface.
 | | ...
+# VPP-930: ip6_fib_dump shows incorrect outgoing interface index
+| | [Tags] | EXPECTED_FAILING
 | | ${table}= | Set Variable | table4
 | | Given Setup interfaces and neighbors for IPv6 routing test
 | | When Honeycomb configures routing table
@@ -135,6 +137,8 @@
 | | ... | verify that each destination MAC was used by exactly 50 packets.
 | | ... | Receive packet on the second TG interface.
 | | ...
+# VPP-930: ip6_fib_dump shows incorrect outgoing interface index
+| | [Tags] | EXPECTED_FAILING
 | | ${table}= | Set Variable | table5
 | | Given Setup interfaces and neighbors for IPv6 routing test
 | | And Honeycomb adds interface IPv6 neighbor | ${dut_node} | ${dut_to_tg_if2}
diff --git a/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot b/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot
index 5320a7f..22a65b2 100644 (file)
--- a/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot
+++ b/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot
@@ -31,7 +31,8 @@
 | ...
 | Suite Teardown | Tear Down Honeycomb Functional Test Suite | ${node}
 | ...
-| Force Tags | HC_FUNC
+# HC2VPP-203 Notifications are not being sent since VPP 17.10
+| Force Tags | HC_FUNC | EXPECTED_FAILING
 
 *** Test Cases ***
 | TC01: Honeycomb sends notification on interface state change
Integration tests