Honeycomb interface state management test

 - add simple test for interface management through Honeycomb API
 - add status code 500 to honeycomb startup check whitelist,
   due to Honeycomb changes in https://gerrit.fd.io/r/772
 - increase timeout value for Honeycomb startup check
 - modify Honeycomb setup keywords to allow starting Honeycomb
   on individual nodes
 - remove sanity test suite, tests now sorted by functional areas
   such as interfaces, bridge domains, VXlan,...

Change-Id: I87f9cf69577706f3b00e24a8d2c01df52fc707c6
Signed-off-by: selias <samuel.elias@pantheon.tech>

diff --git a/resources/libraries/python/HTTPRequest.py b/resources/libraries/python/HTTPRequest.py
index f94318d..567ac79 100644 (file)
--- a/resources/libraries/python/HTTPRequest.py
+++ b/resources/libraries/python/HTTPRequest.py
@@ -35,6 +35,7 @@ class HTTPCodes(IntEnum):
     UNAUTHORIZED = 401
     FORBIDDEN = 403
     NOT_FOUND = 404
+    INTERNAL_SERVER_ERROR = 500
     SERVICE_UNAVAILABLE = 503
 
 

diff --git a/resources/libraries/python/HoneycombSetup.py b/resources/libraries/python/HoneycombSetup.py
index 384c294..bd2f308 100644 (file)
--- a/resources/libraries/python/HoneycombSetup.py
+++ b/resources/libraries/python/HoneycombSetup.py
@@ -41,24 +41,24 @@ class HoneycombSetup(object):
         pass
 
     @staticmethod
-    def start_honeycomb_on_all_duts(nodes):
-        """Start Honeycomb on all DUT nodes in topology.
+    def start_honeycomb_on_duts(*nodes):
+        """Start Honeycomb on specified DUT nodes.
 
-        This keyword starts the Honeycomb service on all DUTs. The keyword just
-        starts the Honeycomb and does not check its startup state. Use the
-        keyword "Check Honeycomb Startup State" to check if the Honeycomb is up
-        and running.
+        This keyword starts the Honeycomb service on specified DUTs.
+        The keyword just starts the Honeycomb and does not check its startup
+        state. Use the keyword "Check Honeycomb Startup State" to check if the
+        Honeycomb is up and running.
         Honeycomb must be installed in "/opt" directory, otherwise the start
         will fail.
-        :param nodes: All nodes in topology.
-        :type nodes: dict
+        :param nodes: List of nodes to start Honeycomb on.
+        :type nodes: list
         :raises HoneycombError: If Honeycomb fails to start.
         """
         logger.console("Starting Honeycomb service ...")
 
         cmd = "{0}/start".format(Const.REMOTE_HC_DIR)
 
-        for node in nodes.values():
+        for node in nodes:
             if node['type'] == NodeType.DUT:
                 ssh = SSH()
                 ssh.connect(node)
@@ -71,14 +71,15 @@ class HoneycombSetup(object):
                                 "in progress ...".format(node['host']))
 
     @staticmethod
-    def stop_honeycomb_on_all_duts(nodes):
-        """Stop the Honeycomb service on all DUTs.
-
-        This keyword stops the Honeycomb service on all nodes. It just stops the
-        Honeycomb and does not check its shutdown state. Use the keyword "Check
-        Honeycomb Shutdown State" to check if Honeycomb has stopped.
-        :param nodes: Nodes in topology.
-        :type nodes: dict
+    def stop_honeycomb_on_duts(*nodes):
+        """Stop the Honeycomb service on specified DUT nodes.
+
+        This keyword stops the Honeycomb service on specified nodes. It just
+        stops the Honeycomb and does not check its shutdown state. Use the
+        keyword "Check Honeycomb Shutdown State" to check if Honeycomb has
+        stopped.
+        :param nodes: List of nodes to stop Honeycomb on.
+        :type nodes: list
         :raises HoneycombError: If Honeycomb failed to stop.
         """
         logger.console("Shutting down Honeycomb service ...")
@@ -86,7 +87,7 @@ class HoneycombSetup(object):
         cmd = "{0}/stop".format(Const.REMOTE_HC_DIR)
         errors = []
 
-        for node in nodes.values():
+        for node in nodes:
             if node['type'] == NodeType.DUT:
                 ssh = SSH()
                 ssh.connect(node)
@@ -101,27 +102,27 @@ class HoneycombSetup(object):
                                  format(errors))
 
     @staticmethod
-    def check_honeycomb_startup_state(nodes):
-        """Check state of Honeycomb service during startup.
+    def check_honeycomb_startup_state(*nodes):
+        """Check state of Honeycomb service during startup on specified nodes.
 
         Reads html path from template file oper_vpp_version.url.
 
-        Honeycomb node replies with connection refused or the following status
-        codes depending on startup progress: codes 200, 401, 403, 404, 503
+        Honeycomb nodes reply with connection refused or the following status
+        codes depending on startup progress: codes 200, 401, 403, 404, 500, 503
 
-        :param nodes: Nodes in topology.
-        :type nodes: dict
+        :param nodes: List of DUT nodes starting Honeycomb.
+        :type nodes: list
         :return: True if all GETs returned code 200(OK).
         :rtype bool
         """
-
         path = HcUtil.read_path_from_url_file("oper_vpp_version")
         expected_status_codes = (HTTPCodes.UNAUTHORIZED,
                                  HTTPCodes.FORBIDDEN,
                                  HTTPCodes.NOT_FOUND,
-                                 HTTPCodes.SERVICE_UNAVAILABLE)
+                                 HTTPCodes.SERVICE_UNAVAILABLE,
+                                 HTTPCodes.INTERNAL_SERVER_ERROR)
 
-        for node in nodes.values():
+        for node in nodes:
             if node['type'] == NodeType.DUT:
                 status_code, _ = HTTPRequest.get(node, path, timeout=10,
                                                  enable_logging=False)
@@ -142,20 +143,19 @@ class HoneycombSetup(object):
         return True
 
     @staticmethod
-    def check_honeycomb_shutdown_state(nodes):
-        """Check state of Honeycomb service during shutdown.
+    def check_honeycomb_shutdown_state(*nodes):
+        """Check state of Honeycomb service during shutdown on specified nodes.
 
-        Honeycomb node replies with connection refused or the following status
+        Honeycomb nodes reply with connection refused or the following status
         codes depending on shutdown progress: codes 200, 404.
 
-        :param nodes: Nodes in topology.
-        :type nodes: dict
+        :param nodes: List of DUT nodes stopping Honeycomb.
+        :type nodes: list
         :return: True if all GETs fail to connect.
         :rtype bool
         """
-
         cmd = "ps -ef | grep -v grep | grep karaf"
-        for node in nodes.values():
+        for node in nodes:
             if node['type'] == NodeType.DUT:
                 try:
                     status_code, _ = HTTPRequest.get(node, '/index.html',
@@ -237,7 +237,6 @@ class HoneycombSetup(object):
             {passwd}
         MUST be there as they are replaced by correct values.
         """
-
         path = HcUtil.read_path_from_url_file("config_topology_node")
         try:
             xml_data = ET.parse("{0}/add_vpp_to_topology.xml".

diff --git a/resources/libraries/robot/honeycomb.robot b/resources/libraries/robot/honeycomb/honeycomb.robot
similarity index 65%
rename from resources/libraries/robot/honeycomb.robot
rename to resources/libraries/robot/honeycomb/honeycomb.robot
index 98b8e23..7418078 100644 (file)
--- a/resources/libraries/robot/honeycomb.robot
+++ b/resources/libraries/robot/honeycomb/honeycomb.robot
@@ -17,14 +17,15 @@
 | Library | resources/libraries/python/HTTPRequest.py
 
 *** Keywords ***
-| Setup Honeycomb service
+| Setup Honeycomb service on DUTs
 | | [Documentation] | *Setup environment for honeycomb testing*
 | | ...
 | | ... | _Setup steps:_
 | | ... | - 1. Login to each honeycomb node using ssh
 | | ... | - 2. Startup honeycomb service
 | | ... | - 3. Monitor service startup using HTTP GET request loop
-| | ... | Expected sequence of HTTP replies: connection refused -> 404 -> 401 -> 503 -> 200 (pass)
+| | ... | Expected sequence of HTTP replies:
+| | ... | connection refused -> 404 -> 401 -> 503 or 500 -> 200 (pass)
 | | ... | - 4. Configure honeycomb nodes using HTTP PUT request
 | | ...
 | | ... | _Used global constants and variables:_
@@ -32,37 +33,27 @@
 | | ... | - HTTPCodes - HTTP protocol status codes
 | | ... | - ${nodes} - dictionary of all nodes in topology.YAML file
 | | ...
-| | Start Honeycomb on all DUTs | ${nodes}
-| | Wait until keyword succeeds | 3min | 10sec | Check honeycomb startup state | ${nodes}
-| | &{Header}= | Create dictionary | Content-Type=application/xml
-| | Add VPP to honeycomb network topology | ${nodes} | ${header}
+| | [Arguments] | @{duts}
+| | Start honeycomb on DUTs | @{duts}
+| | Wait until keyword succeeds | 4min | 20sec
+| | ... | Check honeycomb startup state | @{duts}
 
-| Stop honeycomb service
+| Stop honeycomb service on DUTs
 | | [Documentation] | *Cleanup environment after honeycomb testing*
 | | ...
 | | ... | _Teardown steps:_
 | | ... | - 1. Login to each honeycomb node using ssh
 | | ... | - 2. Stop honeycomb service
 | | ... | - 3. Monitor service shutdown using HTTP GET request loop
-| | ... | Expected sequence of HTTP replies: 200 -> 404 -> connection refused (pass)
+| | ... | Expected sequence of HTTP replies:
+| | ... | 200 -> 404 -> connection refused (pass)
 | | ...
 | | ... | _Used global constants and variables:_
 | | ... | - RESOURCES_TPL_HC - path to honeycomb templates directory
 | | ... | - HTTPCodes - HTTP protocol status codes
 | | ... | - ${nodes} - dictionary of all nodes in topology.YAML file
 | | ...
-| | Stop honeycomb on all DUTs | ${nodes}
-| | Wait until keyword succeeds | 1m | 5s | Check honeycomb shutdown state | ${nodes}
-
-| Honeycomb checks VPP node configuration
-| | [Documentation] | *Check configuration of honeycomb nodes*
-| | ...
-| | ... | _Arguments:_
-| | ... | - None
-| | ...
-| | ... | _Return value:_
-| | ... | - None
-| | ...
-| | ${reply}= | Get configured topology | ${nodes}
-| | :FOR | ${item} | IN | @{reply}
-| | | Should match regexp | ${item} | ^DUT\\d{1,2}$
\ No newline at end of file
+| | [Arguments] | @{duts}
+| | Stop honeycomb on DUTs | @{duts}
+| | Wait until keyword succeeds | 2m | 10s
+| | ... | Check honeycomb shutdown state | @{duts}

diff --git a/resources/libraries/robot/honeycomb/interfaces.robot b/resources/libraries/robot/honeycomb/interfaces.robot
new file mode 100644 (file)
index 0000000..338c3cf
--- /dev/null
+++ b/resources/libraries/robot/honeycomb/interfaces.robot
@@ -0,0 +1,76 @@
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+| Library | resources/libraries/python/HoneycombUtil.py
+| Library | resources.libraries.python.InterfaceUtil
+| ... | WITH NAME | InterfaceCLI
+| Library | resources.libraries.python.HoneycombAPIKeywords.InterfaceKeywords
+| ... | WITH NAME | interfaceAPI
+
+*** Keywords ***
+| Honeycomb sets interface state
+| | [Arguments] | ${node} | ${interface} | ${state}
+| | [Documentation] | Uses Honeycomb API to change the admin state
+| | ... | of the specified interface.
+| | ...
+| | ... | *Arguments:*
+| | ... | - state - state to set on interface
+| | ... | - node - dictionary of information about a DUT node
+| | ... | - interface - name of an interface on the specified node
+| | ...
+| | interfaceAPI.Set interface state | ${node} | ${interface} | ${state}
+
+| Interface state from Honeycomb should be
+| | [Arguments] | ${node} | ${interface} | ${state}
+| | [Documentation] | Retrieves interface admin state through Honeycomb and
+| | ... | compares with state supplied in argument
+| | ...
+| | ... | *Arguments:*
+| | ... | - state - expected interface state
+| | ... | - node - dictionary of information about a DUT node
+| | ... | - interface - name of an interface on the specified node
+| | ...
+| | ${api_data}= | interfaceAPI.Get interface oper info | ${node} | ${interface}
+| | ${api_state}= | Set Variable | ${api_data['admin-status']}
+| | Should be equal | ${api_state} | ${state}
+
+| Interface state from VAT should be
+| | [Arguments] | ${node} | ${interface} | ${state}
+| | [Documentation] | Retrieves interface admin state through VAT and compares
+| | ... | with state supplied in argument
+| | ...
+| | ... | *Arguments:*
+| | ... | - state - expected interface state
+| | ... | - node - dictionary of information about a DUT node
+| | ... | - interface - name of an interface on the specified node
+| | ...
+| | ... | _NOTE:_ Vat returns state as int (1/0) instead of string (up/down).
+| | ... | This keyword also handles translation.
+| | ...
+| | ${vat_data}= | InterfaceCLI.VPP get interface data | ${node} | ${interface}
+| | ${vat_state}= | Set Variable if
+| | ... | ${vat_data['admin_up_down']} == 1 | up | down
+| | Should be equal | ${vat_state} | ${state}
+
+| Interface state is
+| | [Arguments] | ${node} | ${interface} | ${state}
+| | [Documentation] | Uses VPP binary API to ensure that the interface under
+| | ... | test is in the specified admin state.
+| | ...
+| | ... | *Arguments:*
+| | ... | - state - state to set on interface
+| | ... | - node - dictionary of information about a DUT node
+| | ... | - interface - name of an interface on the specified node
+| | ...
+| | interfaceAPI.Set interface state | ${node} | ${interface} | ${state}

diff --git a/tests/suites/honeycomb/interface_management.robot b/tests/suites/honeycomb/interface_management.robot
new file mode 100644 (file)
index 0000000..37200d0
--- /dev/null
+++ b/tests/suites/honeycomb/interface_management.robot
@@ -0,0 +1,42 @@
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Variables ***
+| ${node}= | ${nodes['DUT1']}
+| ${interface}= | ${node['interfaces'].values()[0]['name']}
+
+*** Settings ***
+| Resource | resources/libraries/robot/default.robot
+| Resource | resources/libraries/robot/honeycomb/honeycomb.robot
+| Resource | resources/libraries/robot/honeycomb/interfaces.robot
+| Suite Setup | Run keywords | Setup all DUTs before test | AND
+| ... | Setup Honeycomb service on DUTs | ${node}
+| Suite Teardown | Stop Honeycomb service on DUTs | ${node}
+| Documentation | *Honeycomb interface management test suite.*
+| ...
+| ... | Test suite uses the first interface of the first DUT node.
+
+*** Test Cases ***
+| Honeycomb modifies interface state
+| | [Documentation] | Check if Honeycomb API can modify the admin state of
+| | ... | VPP interfaces.
+| | [Tags] | honeycomb_sanity
+| | Given Interface state is | ${node} | ${interface} | down
+| | When Honeycomb sets interface state | ${node} | ${interface} | up
+| | Then Interface state from Honeycomb should be
+| | ... | ${node} | ${interface} | up
+| | And Interface state from VAT should be | ${node} | ${interface} | up
+| | When Honeycomb sets interface state | ${node} | ${interface} | down
+| | Then Interface state from Honeycomb should be
+| | ... | ${node} | ${interface} | down
+| | And Interface state from VAT should be | ${node} | ${interface} | down

diff --git a/tests/suites/honeycomb/sanity.robot b/tests/suites/honeycomb/sanity.robot
deleted file mode 100644 (file)
index 5264f99..0000000
--- a/tests/suites/honeycomb/sanity.robot
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-| Resource | resources/libraries/robot/honeycomb.robot
-| Suite Setup | Setup Honeycomb service
-| Suite Teardown | Stop Honeycomb service
-
-*** Test Cases ***
-| Honeycomb reports running configuration
-| | [Documentation] | *Check the contents of honeycomb configuration*
-| | ...
-| | ... | _Test steps:_
-| | ... | - 1. Send HTTP GET request to obtain configured topology from all honeycomb nodes
-| | ... | - 2. Retrieve configuration as JSON object
-| | ... | - 3. Parse JSON for VPP instance ID string
-| | ... | - 4. regex match ID string against expected pattern (vpp1, vpp2, vpp3,...)
-| | ...
-| | ... | _Pass criteria:_
-| | ... | The test passes if the ID strings of VPP instances on each honeycomb node match the expected pattern
-| | ...
-| | ... | _Used global constants and variables:_
-| | ... | - RESOURCES_TPL_HC - path to honeycomb templates directory
-| | ... | - nodes - dictionary of all nodes in topology.YAML file
-| | ...
-| | [Tags] | honeycomb_sanity
-| | Honeycomb checks VPP node configuration
\ No newline at end of file