Honeycomb setup and utils

- re-implement HTTPCodes as IntEnum rather then dictionary
- add methods to manipulate data using honeycomb - get, set, delete
- change the name of url file from vpp_version.url to oper_vpp_version.url
- improve checking of startup and shutdown state of honeycomb
- PEP8 fixes
- add docstrings in all modules and classes
- move logging to the lowest possible level
- improve logging in exceptions
- add method exec_command_sudo_log to resources.libraries.python.ssh module

Change-Id: I54e0c6b45313e3a3c11bafa475488ae2b1e605c2
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/docs/honeycomb_url_files.rst b/docs/honeycomb_url_files.rst
new file mode 100644 (file)
index 0000000..4dfa10c
--- /dev/null
+++ b/docs/honeycomb_url_files.rst
@@ -0,0 +1,22 @@
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Documentation to files used to store URLs to resources in Honeycomb.
+====================================================================
+
+A URL file is a text file encoded in utf-8 with a path to a resource in
+Honeycomb. There is only one line in each file.
+
+The URL is stored without host and port with leading slash. There is no slash at
+the end, e.g.:
+    /restconf/config/v3po:vpp/bridge-domains

diff --git a/resources/libraries/python/HTTPRequest.py b/resources/libraries/python/HTTPRequest.py
index 7b21f5a..fd2925c 100644 (file)
--- a/resources/libraries/python/HTTPRequest.py
+++ b/resources/libraries/python/HTTPRequest.py
@@ -11,69 +11,100 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 # See the License for the specific language governing permissions and
 # limitations under the License.
 

-"""Implements HTTP requests GET, PUT, POST, DELETE used in communication with
-honeycomb.
+"""Implementation of HTTP requests GET, PUT, POST and DELETE used in
+communication with Honeycomb.
+
+The HTTP requests are implemented in the class HTTPRequest which uses
+requests.request.

 """
 
 """
 

+from enum import IntEnum, unique
+
+from robot.api.deco import keyword
+from robot.api import logger
+

 from requests import request, RequestException, Timeout, TooManyRedirects, \
     HTTPError, ConnectionError
 from requests.auth import HTTPBasicAuth
 
 from requests import request, RequestException, Timeout, TooManyRedirects, \
     HTTPError, ConnectionError
 from requests.auth import HTTPBasicAuth
 

-from robot.api import logger
-from robot.api.deco import keyword
-

 
 

-HTTP_CODES = {"OK": 200,
-              "UNAUTHORIZED": 401,
-              "FORBIDDEN": 403,
-              "NOT_FOUND": 404,
-              "SERVICE_UNAVAILABLE": 503}
+@unique
+class HTTPCodes(IntEnum):
+    """HTTP status codes"""
+    OK = 200
+    UNAUTHORIZED = 401
+    FORBIDDEN = 403
+    NOT_FOUND = 404
+    SERVICE_UNAVAILABLE = 503

 
 
 class HTTPRequestError(Exception):
 
 
 class HTTPRequestError(Exception):

-    """Exception raised by HTTPRequest objects."""
+    """Exception raised by HTTPRequest objects.
+
+    When raising this exception, put this information to the message in this
+    order:
+    - short description of the encountered problem,
+    - relevant messages if there are any collected, e.g., from caught
+      exception,
+    - relevant data if there are any collected.
+    The logging is performed on two levels: 1. error - short description of the
+    problem; 2. debug - detailed information.
+    """

 
 

-    def __init__(self, msg, enable_logging=True):
-        """Sets the exception message and enables / disables logging
+    def __init__(self, msg, details='', enable_logging=True):
+        """Sets the exception message and enables / disables logging.

 
         It is not wanted to log errors when using these keywords together
 
         It is not wanted to log errors when using these keywords together

-        with keywords like "Wait until keyword succeeds".
+        with keywords like "Wait until keyword succeeds". So you can disable
+        logging by setting enable_logging to False.

 
 

-        :param msg: Message to be displayed and logged
+        :param msg: Message to be displayed and logged.

         :param enable_logging: When True, logging is enabled, otherwise
         logging is disabled.
         :type msg: str
         :type enable_logging: bool
         """
         super(HTTPRequestError, self).__init__()
         :param enable_logging: When True, logging is enabled, otherwise
         logging is disabled.
         :type msg: str
         :type enable_logging: bool
         """
         super(HTTPRequestError, self).__init__()

-        self._msg = msg
-        self._repr_msg = self.__module__ + '.' + \
-            self.__class__.__name__ + ": " + self._msg
-
+        self._msg = "{0}: {1}".format(self.__class__.__name__, msg)
+        self._details = details

         if enable_logging:
             logger.error(self._msg)
         if enable_logging:
             logger.error(self._msg)

-            logger.debug(self._repr_msg)
+            logger.debug(self._details)

 
     def __repr__(self):
 
     def __repr__(self):

-        return repr(self._repr_msg)
+        return repr(self._msg)

 
     def __str__(self):
 
     def __str__(self):

-        return str(self._repr_msg)
+        return str(self._msg)

 
 
 class HTTPRequest(object):
 
 
 class HTTPRequest(object):

-    """A class implementing HTTP requests."""
+    """A class implementing HTTP requests GET, PUT, POST and DELETE used in
+    communication with Honeycomb.
+
+    The communication with Honeycomb and processing of all exceptions is done in
+    the method _http_request which uses requests.request to send requests and
+    receive responses. The received status code and content of response are
+    logged on the debug level.
+    All possible exceptions raised by requests.request are also processed there.
+
+    The other methods (get, put, post and delete) use _http_request to send
+    corresponding request.
+
+    These methods must not be used as keywords in tests. Use keywords
+    implemented in the module HoneycombAPIKeywords instead.
+    """

 
     def __init__(self):
         pass
 
     @staticmethod
     def create_full_url(ip_addr, port, path):
 
     def __init__(self):
         pass
 
     @staticmethod
     def create_full_url(ip_addr, port, path):

-        """Creates full url including IP, port, and path to data.
+        """Creates full url including host, port, and path to data.

 
 

-        :param ip_addr: Server IP
-        :param port: Communication port
-        :param path: Path to data
+        :param ip_addr: Server IP.
+        :param port: Communication port.
+        :param path: Path to data.

         :type ip_addr: str
         :type port: str or int
         :type path: str
         :type ip_addr: str
         :type port: str or int
         :type path: str


@@ -85,16 +116,16 @@ class HTTPRequest(object):
 
     @staticmethod
     def _http_request(method, node, path, enable_logging=True, **kwargs):
 
     @staticmethod
     def _http_request(method, node, path, enable_logging=True, **kwargs):

-        """Sends specified HTTP request and returns status code and
-        response content
+        """Sends specified HTTP request and returns status code and response
+        content.

 
         :param method: The method to be performed on the resource identified by
 
         :param method: The method to be performed on the resource identified by

-        the given request URI
-        :param node: honeycomb node
-        :param path: URL path, e.g. /index.html
-        :param enable_logging: used to suppress errors when checking
-        honeycomb state during suite setup and teardown
-        :param kwargs: named parameters accepted by request.request:
+        the given request URI.
+        :param node: Honeycomb node.
+        :param path: URL path, e.g. /index.html.
+        :param enable_logging: Used to suppress errors when checking Honeycomb
+        state during suite setup and teardown.
+        :param kwargs: Named parameters accepted by request.request:

             params -- (optional) Dictionary or bytes to be sent in the query
             string for the Request.
             data -- (optional) Dictionary, bytes, or file-like object to
             params -- (optional) Dictionary or bytes to be sent in the query
             string for the Request.
             data -- (optional) Dictionary, bytes, or file-like object to


@@ -127,11 +158,11 @@ class HTTPRequest(object):
         :return: Status code and content of response
         :rtype: tuple
         :raises HTTPRequestError: If
         :return: Status code and content of response
         :rtype: tuple
         :raises HTTPRequestError: If

-        1. it is not possible to connect
-        2. invalid HTTP response comes from server
-        3. request exceeded the configured number of maximum re-directions
-        4. request timed out
-        5. there is any other unexpected HTTP request exception
+        1. it is not possible to connect,
+        2. invalid HTTP response comes from server,
+        3. request exceeded the configured number of maximum re-directions,
+        4. request timed out,
+        5. there is any other unexpected HTTP request exception.

         """
         timeout = kwargs["timeout"]
         url = HTTPRequest.create_full_url(node['host'],
         """
         timeout = kwargs["timeout"]
         url = HTTPRequest.create_full_url(node['host'],


@@ -141,29 +172,30 @@ class HTTPRequest(object):
             auth = HTTPBasicAuth(node['honeycomb']['user'],
                                  node['honeycomb']['passwd'])
             rsp = request(method, url, auth=auth, **kwargs)
             auth = HTTPBasicAuth(node['honeycomb']['user'],
                                  node['honeycomb']['passwd'])
             rsp = request(method, url, auth=auth, **kwargs)

+
+            logger.debug("Status code: {0}".format(rsp.status_code))
+            logger.debug("Response: {0}".format(rsp.content))
+

             return rsp.status_code, rsp.content
 
         except ConnectionError as err:
             # Switching the logging on / off is needed only for
             # "requests.ConnectionError"
             return rsp.status_code, rsp.content
 
         except ConnectionError as err:
             # Switching the logging on / off is needed only for
             # "requests.ConnectionError"

-            if enable_logging:
-                raise HTTPRequestError("Not possible to connect to {0}\n".
-                                       format(url) + repr(err))
-            else:
-                raise HTTPRequestError("Not possible to connect to {0}\n".
-                                       format(url) + repr(err),
-                                       enable_logging=False)
+            raise HTTPRequestError("Not possible to connect to {0}:{1}.".
+                                   format(node['host'],
+                                          node['honeycomb']['port']),
+                                   repr(err), enable_logging=enable_logging)

         except HTTPError as err:
         except HTTPError as err:

-            raise HTTPRequestError("Invalid HTTP response from {0}\n".
-                                   format(url) + repr(err))
+            raise HTTPRequestError("Invalid HTTP response from {0}.".
+                                   format(node['host']), repr(err))

         except TooManyRedirects as err:
             raise HTTPRequestError("Request exceeded the configured number "
         except TooManyRedirects as err:
             raise HTTPRequestError("Request exceeded the configured number "

-                                   "of maximum re-directions\n" + repr(err))
+                                   "of maximum re-directions.", repr(err))

         except Timeout as err:
         except Timeout as err:

-            raise HTTPRequestError("Request timed out. Timeout is set to "
-                                   "{0}\n".format(timeout) + repr(err))
+            raise HTTPRequestError("Request timed out. Timeout is set to {0}.".
+                                   format(timeout), repr(err))

         except RequestException as err:
         except RequestException as err:

-            raise HTTPRequestError("Unexpected HTTP request exception.\n" +
+            raise HTTPRequestError("Unexpected HTTP request exception.",

                                    repr(err))
 
     @staticmethod
                                    repr(err))
 
     @staticmethod


@@ -171,60 +203,64 @@ class HTTPRequest(object):
     def get(node, path, headers=None, timeout=10, enable_logging=True):
         """Sends a GET request and returns the response and status code.
 
     def get(node, path, headers=None, timeout=10, enable_logging=True):
         """Sends a GET request and returns the response and status code.
 

-        :param node: honeycomb node
-        :param path: URL path, e.g. /index.html
+        :param node: Honeycomb node.
+        :param path: URL path, e.g. /index.html.

         :param headers: Dictionary of HTTP Headers to send with the Request.
         :param timeout: How long to wait for the server to send data before
         giving up, as a float, or a (connect timeout, read timeout) tuple.
         :param headers: Dictionary of HTTP Headers to send with the Request.
         :param timeout: How long to wait for the server to send data before
         giving up, as a float, or a (connect timeout, read timeout) tuple.

-        :param enable_logging: Used to suppress errors when checking
-        honeycomb state during suite setup and teardown. When True, logging
-        is enabled, otherwise logging is disabled.
+        :param enable_logging: Used to suppress errors when checking Honeycomb
+        state during suite setup and teardown. When True, logging is enabled,
+        otherwise logging is disabled.

         :type node: dict
         :type path: str
         :type headers: dict
         :type timeout: float or tuple
         :type enable_logging: bool
         :type node: dict
         :type path: str
         :type headers: dict
         :type timeout: float or tuple
         :type enable_logging: bool

-        :return: Status code and content of response
+        :return: Status code and content of response.

         :rtype: tuple
         """
         :rtype: tuple
         """

+

         return HTTPRequest._http_request('GET', node, path,
                                          enable_logging=enable_logging,
                                          headers=headers, timeout=timeout)
 
     @staticmethod
     @keyword(name="HTTP Put")
         return HTTPRequest._http_request('GET', node, path,
                                          enable_logging=enable_logging,
                                          headers=headers, timeout=timeout)
 
     @staticmethod
     @keyword(name="HTTP Put")

-    def put(node, path, headers=None, payload=None, timeout=10):
+    def put(node, path, headers=None, payload=None, json=None, timeout=10):

         """Sends a PUT request and returns the response and status code.
 
         """Sends a PUT request and returns the response and status code.
 

-        :param node: honeycomb node
-        :param path: URL path, e.g. /index.html
+        :param node: Honeycomb node.
+        :param path: URL path, e.g. /index.html.

         :param headers: Dictionary of HTTP Headers to send with the Request.
         :param payload: Dictionary, bytes, or file-like object to send in
         the body of the Request.
         :param headers: Dictionary of HTTP Headers to send with the Request.
         :param payload: Dictionary, bytes, or file-like object to send in
         the body of the Request.

+        :param json: JSON formatted string to send in the body of the Request.

         :param timeout: How long to wait for the server to send data before
         giving up, as a float, or a (connect timeout, read timeout) tuple.
         :type node: dict
         :type path: str
         :type headers: dict
         :type payload: dict, bytes, or file-like object
         :param timeout: How long to wait for the server to send data before
         giving up, as a float, or a (connect timeout, read timeout) tuple.
         :type node: dict
         :type path: str
         :type headers: dict
         :type payload: dict, bytes, or file-like object

+        :type json: str

         :type timeout: float or tuple
         :type timeout: float or tuple

-        :return: Status code and content of response
+        :return: Status code and content of response.

         :rtype: tuple
         """
         return HTTPRequest._http_request('PUT', node, path, headers=headers,
         :rtype: tuple
         """
         return HTTPRequest._http_request('PUT', node, path, headers=headers,

-                                         data=payload, timeout=timeout)
+                                         data=payload, json=json,
+                                         timeout=timeout)

 
     @staticmethod
     @keyword(name="HTTP Post")
     def post(node, path, headers=None, payload=None, json=None, timeout=10):
         """Sends a POST request and returns the response and status code.
 
 
     @staticmethod
     @keyword(name="HTTP Post")
     def post(node, path, headers=None, payload=None, json=None, timeout=10):
         """Sends a POST request and returns the response and status code.
 

-        :param node: honeycomb node
-        :param path: URL path, e.g. /index.html
+        :param node: Honeycomb node.
+        :param path: URL path, e.g. /index.html.

         :param headers: Dictionary of HTTP Headers to send with the Request.
         :param payload: Dictionary, bytes, or file-like object to send in
         the body of the Request.
         :param headers: Dictionary of HTTP Headers to send with the Request.
         :param payload: Dictionary, bytes, or file-like object to send in
         the body of the Request.

-        :param json: json data to send in the body of the Request
+        :param json: JSON formatted string to send in the body of the Request.

         :param timeout: How long to wait for the server to send data before
         giving up, as a float, or a (connect timeout, read timeout) tuple.
         :type node: dict
         :param timeout: How long to wait for the server to send data before
         giving up, as a float, or a (connect timeout, read timeout) tuple.
         :type node: dict


@@ -233,7 +269,7 @@ class HTTPRequest(object):
         :type payload: dict, bytes, or file-like object
         :type json: str
         :type timeout: float or tuple
         :type payload: dict, bytes, or file-like object
         :type json: str
         :type timeout: float or tuple

-        :return: Status code and content of response
+        :return: Status code and content of response.

         :rtype: tuple
         """
         return HTTPRequest._http_request('POST', node, path, headers=headers,
         :rtype: tuple
         """
         return HTTPRequest._http_request('POST', node, path, headers=headers,


@@ -245,14 +281,14 @@ class HTTPRequest(object):
     def delete(node, path, timeout=10):
         """Sends a DELETE request and returns the response and status code.
 
     def delete(node, path, timeout=10):
         """Sends a DELETE request and returns the response and status code.
 

-        :param node: honeycomb node
-        :param path: URL path, e.g. /index.html
+        :param node: Honeycomb node.
+        :param path: URL path, e.g. /index.html.

         :param timeout: How long to wait for the server to send data before
         giving up, as a float, or a (connect timeout, read timeout) tuple.
         :type node: dict
         :type path: str
         :type timeout: float or tuple
         :param timeout: How long to wait for the server to send data before
         giving up, as a float, or a (connect timeout, read timeout) tuple.
         :type node: dict
         :type path: str
         :type timeout: float or tuple

-        :return: Status code and content of response
+        :return: Status code and content of response.

         :rtype: tuple
         """
         return HTTPRequest._http_request('DELETE', node, path, timeout=timeout)
         :rtype: tuple
         """
         return HTTPRequest._http_request('DELETE', node, path, timeout=timeout)

diff --git a/resources/libraries/python/HoneycombSetup.py b/resources/libraries/python/HoneycombSetup.py
index de05eff..384c294 100644 (file)
--- a/resources/libraries/python/HoneycombSetup.py
+++ b/resources/libraries/python/HoneycombSetup.py
@@ -11,220 +11,200 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 # See the License for the specific language governing permissions and
 # limitations under the License.
 

-"""Implements keywords for Honeycomb setup."""
+"""Implementation of keywords for Honeycomb setup."""

 
 

-import os.path

 from xml.etree import ElementTree as ET
 
 from robot.api import logger
 
 from resources.libraries.python.topology import NodeType
 from resources.libraries.python.ssh import SSH
 from xml.etree import ElementTree as ET
 
 from robot.api import logger
 
 from resources.libraries.python.topology import NodeType
 from resources.libraries.python.ssh import SSH

-from resources.libraries.python.HTTPRequest import HTTPRequest, \
-    HTTPRequestError, HTTP_CODES
-from resources.libraries.python.constants import Constants as C
-
-
-class HoneycombError(Exception):
-    """Exception(s) raised by methods working with Honeycomb."""
-
-    def __init__(self, msg, enable_logging=True):
-        """Sets the exception message and enables / disables logging
-
-        It is not wanted to log errors when using these keywords together
-        with keywords like "Wait until keyword succeeds".
-
-        :param msg: Message to be displayed and logged
-        :param enable_logging: When True, logging is enabled, otherwise
-        logging is disabled.
-        :type msg: str
-        :type enable_logging: bool
-        """
-        super(HoneycombError, self).__init__()
-        self._msg = msg
-        self._repr_msg = self.__module__ + '.' + \
-                         self.__class__.__name__ + ": " + self._msg
-        if enable_logging:
-            logger.error(self._msg)
-            logger.debug(self._repr_msg)
-
-    def __repr__(self):
-        return repr(self._repr_msg)
-
-    def __str__(self):
-        return str(self._repr_msg)
+from resources.libraries.python.HTTPRequest import HTTPRequest, HTTPCodes, \
+    HTTPRequestError
+from resources.libraries.python.HoneycombUtil import HoneycombUtil as HcUtil
+from resources.libraries.python.HoneycombUtil import HoneycombError
+from resources.libraries.python.constants import Constants as Const

 
 
 class HoneycombSetup(object):
 
 
 class HoneycombSetup(object):

-    """Implements keywords for Honeycomb setup."""
+    """Implements keywords for Honeycomb setup.
+
+    The keywords implemented in this class make possible to:
+    - start Honeycomb,
+    - stop Honeycomb,
+    - check the Honeycomb start-up state,
+    - check the Honeycomb shutdown state,
+    - add VPP to the topology.
+    """

 
     def __init__(self):
         pass
 
     @staticmethod
     def start_honeycomb_on_all_duts(nodes):
 
     def __init__(self):
         pass
 
     @staticmethod
     def start_honeycomb_on_all_duts(nodes):

-        """Start honeycomb on all DUT nodes in topology.
-
-        :param nodes: all nodes in topology
+        """Start Honeycomb on all DUT nodes in topology.
+
+        This keyword starts the Honeycomb service on all DUTs. The keyword just
+        starts the Honeycomb and does not check its startup state. Use the
+        keyword "Check Honeycomb Startup State" to check if the Honeycomb is up
+        and running.
+        Honeycomb must be installed in "/opt" directory, otherwise the start
+        will fail.
+        :param nodes: All nodes in topology.

         :type nodes: dict
         :type nodes: dict

+        :raises HoneycombError: If Honeycomb fails to start.

         """
         """

-        logger.console("Starting honeycomb service")
+        logger.console("Starting Honeycomb service ...")
+
+        cmd = "{0}/start".format(Const.REMOTE_HC_DIR)

 
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
 
         for node in nodes.values():
             if node['type'] == NodeType.DUT:

-                HoneycombSetup.start_honeycomb(node)
-
-    @staticmethod
-    def start_honeycomb(node):
-        """Start up honeycomb on DUT node.
-
-        :param node: DUT node with honeycomb
-        :type node: dict
-        :return: ret_code, stdout, stderr
-        :rtype: tuple
-        :raises HoneycombError: if Honeycomb fails to start.
-        """
-
-        ssh = SSH()
-        ssh.connect(node)
-        cmd = os.path.join(C.REMOTE_HC_DIR, "start")
-        (ret_code, stdout, stderr) = ssh.exec_command_sudo(cmd)
-        if int(ret_code) != 0:
-            logger.debug('stdout: {0}'.format(stdout))
-            logger.debug('stderr: {0}'.format(stderr))
-            raise HoneycombError('Node {0} failed to start honeycomb'.
-                                 format(node['host']))
-        return ret_code, stdout, stderr
+                ssh = SSH()
+                ssh.connect(node)
+                (ret_code, _, _) = ssh.exec_command_sudo(cmd)
+                if int(ret_code) != 0:
+                    raise HoneycombError('Node {0} failed to start Honeycomb.'.
+                                         format(node['host']))
+                else:
+                    logger.info("Starting the Honeycomb service on node {0} is "
+                                "in progress ...".format(node['host']))

 
     @staticmethod
     def stop_honeycomb_on_all_duts(nodes):
 
     @staticmethod
     def stop_honeycomb_on_all_duts(nodes):

-        """Stop the honeycomb service on all DUTs.
+        """Stop the Honeycomb service on all DUTs.

 
 

-        :param nodes: nodes in topology
+        This keyword stops the Honeycomb service on all nodes. It just stops the
+        Honeycomb and does not check its shutdown state. Use the keyword "Check
+        Honeycomb Shutdown State" to check if Honeycomb has stopped.
+        :param nodes: Nodes in topology.

         :type nodes: dict
         :type nodes: dict

-        :return: ret_code, stdout, stderr
-        :rtype: tuple
-        :raises HoneycombError: if Honeycomb failed to stop.
+        :raises HoneycombError: If Honeycomb failed to stop.

         """
         """

-        logger.console("Shutting down honeycomb service")
+        logger.console("Shutting down Honeycomb service ...")
+
+        cmd = "{0}/stop".format(Const.REMOTE_HC_DIR)

         errors = []
         errors = []

+

         for node in nodes.values():
             if node['type'] == NodeType.DUT:
         for node in nodes.values():
             if node['type'] == NodeType.DUT:

-

                 ssh = SSH()
                 ssh.connect(node)
                 ssh = SSH()
                 ssh.connect(node)

-                cmd = os.path.join(C.REMOTE_HC_DIR, "stop")
-                (ret_code, stdout, stderr) = ssh.exec_command_sudo(cmd)
+                (ret_code, _, _) = ssh.exec_command_sudo(cmd)

                 if int(ret_code) != 0:
                 if int(ret_code) != 0:

-                    logger.debug('stdout: {0}'.format(stdout))
-                    logger.debug('stderr: {0}'.format(stderr))

                     errors.append(node['host'])
                     errors.append(node['host'])

-                    continue
-                logger.info("Honeycomb was successfully stopped on node {0}.".
-                            format(node['host']))
+                else:
+                    logger.info("Stopping the Honeycomb service on node {0} is "
+                                "in progress ...".format(node['host']))

         if errors:
         if errors:

-            raise HoneycombError('Node(s) {0} failed to stop honeycomb.'.
+            raise HoneycombError('Node(s) {0} failed to stop Honeycomb.'.

                                  format(errors))
 
     @staticmethod
     def check_honeycomb_startup_state(nodes):
                                  format(errors))
 
     @staticmethod
     def check_honeycomb_startup_state(nodes):

-        """Check state of honeycomb service during startup.
+        """Check state of Honeycomb service during startup.

 
 

-        Reads html path from template file vpp_version.url
+        Reads html path from template file oper_vpp_version.url.

 
         Honeycomb node replies with connection refused or the following status
         codes depending on startup progress: codes 200, 401, 403, 404, 503
 
 
         Honeycomb node replies with connection refused or the following status
         codes depending on startup progress: codes 200, 401, 403, 404, 503
 

-        :param nodes: nodes in topology
+        :param nodes: Nodes in topology.

         :type nodes: dict
         :type nodes: dict

-        :return: True if all GETs returned code 200(OK)
+        :return: True if all GETs returned code 200(OK).

         :rtype bool
         """
 
         :rtype bool
         """
 

-        url_file = os.path.join(C.RESOURCES_TPL_HC, "vpp_version.url")
-        with open(url_file) as template:
-            data = template.readline()
-
-        expected_status_codes = (HTTP_CODES["UNAUTHORIZED"],
-                                 HTTP_CODES["FORBIDDEN"],
-                                 HTTP_CODES["NOT_FOUND"],
-                                 HTTP_CODES["SERVICE_UNAVAILABLE"])
+        path = HcUtil.read_path_from_url_file("oper_vpp_version")
+        expected_status_codes = (HTTPCodes.UNAUTHORIZED,
+                                 HTTPCodes.FORBIDDEN,
+                                 HTTPCodes.NOT_FOUND,
+                                 HTTPCodes.SERVICE_UNAVAILABLE)

 
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
 
         for node in nodes.values():
             if node['type'] == NodeType.DUT:

-                status_code, _ = HTTPRequest.get(node, data, timeout=10,
+                status_code, _ = HTTPRequest.get(node, path, timeout=10,

                                                  enable_logging=False)
                                                  enable_logging=False)

-                if status_code == HTTP_CODES["OK"]:
-                    pass
+                if status_code == HTTPCodes.OK:
+                    logger.info("Honeycomb on node {0} is up and running".
+                                format(node['host']))

                 elif status_code in expected_status_codes:
                 elif status_code in expected_status_codes:

-                    if status_code == HTTP_CODES["UNAUTHORIZED"]:
+                    if status_code == HTTPCodes.UNAUTHORIZED:

                         logger.info('Unauthorized. If this triggers keyword '
                         logger.info('Unauthorized. If this triggers keyword '

-                                    'timeout, verify honeycomb '
-                                    'username and password')
+                                    'timeout, verify Honeycomb username and '
+                                    'password.')

                     raise HoneycombError('Honeycomb on node {0} running but '
                                          'not yet ready.'.format(node['host']),
                                          enable_logging=False)
                 else:
                     raise HoneycombError('Honeycomb on node {0} running but '
                                          'not yet ready.'.format(node['host']),
                                          enable_logging=False)
                 else:

-                    raise HoneycombError('Unexpected return code: {0}'.
+                    raise HoneycombError('Unexpected return code: {0}.'.

                                          format(status_code))
         return True
 
     @staticmethod
     def check_honeycomb_shutdown_state(nodes):
                                          format(status_code))
         return True
 
     @staticmethod
     def check_honeycomb_shutdown_state(nodes):

-        """Check state of honeycomb service during shutdown.
+        """Check state of Honeycomb service during shutdown.

 
         Honeycomb node replies with connection refused or the following status
 
         Honeycomb node replies with connection refused or the following status

-        codes depending on shutdown progress: codes 200, 404
+        codes depending on shutdown progress: codes 200, 404.

 
 

-        :param nodes: nodes in topology
+        :param nodes: Nodes in topology.

         :type nodes: dict
         :type nodes: dict

-        :return: True if all GETs fail to connect
+        :return: True if all GETs fail to connect.

         :rtype bool
         """
 
         :rtype bool
         """
 

+        cmd = "ps -ef | grep -v grep | grep karaf"

         for node in nodes.values():
             if node['type'] == NodeType.DUT:
                 try:
                     status_code, _ = HTTPRequest.get(node, '/index.html',
                                                      timeout=5,
                                                      enable_logging=False)
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
                 try:
                     status_code, _ = HTTPRequest.get(node, '/index.html',
                                                      timeout=5,
                                                      enable_logging=False)

-                    if status_code == HTTP_CODES["OK"]:
+                    if status_code == HTTPCodes.OK:

                         raise HoneycombError('Honeycomb on node {0} is still '
                         raise HoneycombError('Honeycomb on node {0} is still '

-                                             'running'.format(node['host']),
+                                             'running.'.format(node['host']),

                                              enable_logging=False)
                                              enable_logging=False)

-                    elif status_code == HTTP_CODES["NOT_FOUND"]:
+                    elif status_code == HTTPCodes.NOT_FOUND:

                         raise HoneycombError('Honeycomb on node {0} is shutting'
                         raise HoneycombError('Honeycomb on node {0} is shutting'

-                                             ' down'.format(node['host']),
+                                             ' down.'.format(node['host']),

                                              enable_logging=False)
                     else:
                                              enable_logging=False)
                     else:

-                        raise HoneycombError('Unexpected return code: {'
-                                             '0}'.format(status_code))
+                        raise HoneycombError('Unexpected return code: {0}.'.
+                                             format(status_code))

                 except HTTPRequestError:
                 except HTTPRequestError:

-                    logger.debug('Connection refused')
-
+                    logger.debug('Connection refused, checking the process '
+                                 'state ...')
+                    ssh = SSH()
+                    ssh.connect(node)
+                    (ret_code, _, _) = ssh.exec_command_sudo(cmd)
+                    if ret_code == 0:
+                        raise HoneycombError('Honeycomb on node {0} is still '
+                                             'running.'.format(node['host']),
+                                             enable_logging=False)
+                    else:
+                        logger.info("Honeycomb on node {0} has stopped".
+                                    format(node['host']))

         return True
 
         return True
 

-

     @staticmethod
     @staticmethod

-    def add_vpp_to_honeycomb_network_topology(nodes, headers):
+    def add_vpp_to_honeycomb_network_topology(nodes):

         """Add vpp node to Honeycomb network topology.
 
         """Add vpp node to Honeycomb network topology.
 

-        :param nodes: all nodes in test topology
-        :param headers: headers to be used with PUT requests
+        :param nodes: All nodes in test topology.

         :type nodes: dict
         :type nodes: dict

-        :type headers: dict
-        :return: status code and response from PUT requests
+        :return: Status code and response content from PUT requests.

         :rtype: tuple
         :rtype: tuple

-        :raises HoneycombError: if a node was not added to honeycomb topology
+        :raises HoneycombError: If a node was not added to Honeycomb topology.

 
 

-        Reads HTML path from template file config_topology_node.url
+        Reads HTML path from template file config_topology_node.url.

         Path to the node to be added, e.g.:
         ("/restconf/config/network-topology:network-topology"
          "/topology/topology-netconf/node/")
         Path to the node to be added, e.g.:
         ("/restconf/config/network-topology:network-topology"
          "/topology/topology-netconf/node/")

-        There must be "/" at the end, as generated node name is added
-        at the end.
+        There must be "/" at the end, as generated node name is added at the
+        end.

 
 

-        Reads payload data from template file add_vpp_to_topology.xml
+        Reads payload data from template file add_vpp_to_topology.xml.

         Information about node as XML structure, e.g.:
         <node xmlns="urn:TBD:params:xml:ns:yang:network-topology">
             <node-id>
         Information about node as XML structure, e.g.:
         <node xmlns="urn:TBD:params:xml:ns:yang:network-topology">
             <node-id>


@@ -258,17 +238,16 @@ class HoneycombSetup(object):
         MUST be there as they are replaced by correct values.
         """
 
         MUST be there as they are replaced by correct values.
         """
 

-        with open(os.path.join(C.RESOURCES_TPL_HC, "config_topology_node.url"))\
-                as template:
-            path = template.readline()
-
+        path = HcUtil.read_path_from_url_file("config_topology_node")

         try:
         try:

-            xml_data = ET.parse(os.path.join(C.RESOURCES_TPL_HC,
-                                             "add_vpp_to_topology.xml"))
+            xml_data = ET.parse("{0}/add_vpp_to_topology.xml".
+                                format(Const.RESOURCES_TPL_HC))

         except ET.ParseError as err:
             raise HoneycombError(repr(err))
         data = ET.tostring(xml_data.getroot())
 
         except ET.ParseError as err:
             raise HoneycombError(repr(err))
         data = ET.tostring(xml_data.getroot())
 

+        headers = {"Content-Type": "application/xml"}
+

         status_codes = []
         responses = []
         for node_name, node in nodes.items():
         status_codes = []
         responses = []
         for node_name, node in nodes.items():


@@ -282,20 +261,19 @@ class HoneycombSetup(object):
                         passwd=node['honeycomb']["passwd"])
                     status_code, resp = HTTPRequest.put(
                         node=node,
                         passwd=node['honeycomb']["passwd"])
                     status_code, resp = HTTPRequest.put(
                         node=node,

-                        path=path + '/' + node_name,
+                        path="{0}/{1}".format(path, node_name),

                         headers=headers,
                         payload=payload)
                         headers=headers,
                         payload=payload)

-                    if status_code != HTTP_CODES["OK"]:
+                    if status_code != HTTPCodes.OK:

                         raise HoneycombError(
                             "VPP {0} was not added to topology. "
                         raise HoneycombError(
                             "VPP {0} was not added to topology. "

-                            "Status code: {1}".format(node["host"],
-                                                      status_code))
+                            "Status code: {1}.".format(node["host"],
+                                                       status_code))

 
                     status_codes.append(status_code)
                     responses.append(resp)
 
                 except HTTPRequestError as err:
 
                     status_codes.append(status_code)
                     responses.append(resp)
 
                 except HTTPRequestError as err:

-                    raise HoneycombError("VPP {0} was not added to topology.\n"
-                                         "{1}".format(node["host"], repr(err)))
-
+                    raise HoneycombError("VPP {0} was not added to topology.".
+                                         format(node["host"]), repr(err))

         return status_codes, responses
         return status_codes, responses

diff --git a/resources/libraries/python/HoneycombUtil.py b/resources/libraries/python/HoneycombUtil.py
index c4dc3a0..86c25ad 100644 (file)
--- a/resources/libraries/python/HoneycombUtil.py
+++ b/resources/libraries/python/HoneycombUtil.py
@@ -11,76 +11,137 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 # See the License for the specific language governing permissions and
 # limitations under the License.
 

-"""Implements keywords used with Honeycomb."""
+"""Implementation of low level functionality used in communication with
+Honeycomb.
+
+Exception HoneycombError is used in all methods and in all modules with
+Honeycomb keywords.
+
+Class HoneycombUtil implements methods used by Honeycomb keywords. They must not
+be used directly in tests. Use keywords implemented in the module
+HoneycombAPIKeywords instead.
+"""

 
 

-import os.path

 from json import loads
 
 from robot.api import logger
 
 from json import loads
 
 from robot.api import logger
 

-from resources.libraries.python.topology import NodeType

 from resources.libraries.python.HTTPRequest import HTTPRequest
 from resources.libraries.python.HTTPRequest import HTTPRequest

-from resources.libraries.python.constants import Constants as C
+from resources.libraries.python.constants import Constants as Const
+
+
+class HoneycombError(Exception):
+
+    """Exception(s) raised by methods working with Honeycomb.
+
+    When raising this exception, put this information to the message in this
+    order:
+     - short description of the encountered problem (parameter msg),
+     - relevant messages if there are any collected, e.g., from caught
+       exception (optional parameter details),
+     - relevant data if there are any collected (optional parameter details).
+     The logging is performed on two levels: 1. error - short description of the
+     problem; 2. debug - detailed information.
+    """
+
+    def __init__(self, msg, details='', enable_logging=True):
+        """Sets the exception message and enables / disables logging.
+
+        It is not wanted to log errors when using these keywords together
+        with keywords like "Wait until keyword succeeds". So you can disable
+        logging by setting enable_logging to False.
+
+        :param msg: Message to be displayed and logged
+        :param enable_logging: When True, logging is enabled, otherwise
+        logging is disabled.
+        :type msg: str
+        :type enable_logging: bool
+        """
+        super(HoneycombError, self).__init__()
+        self._msg = "{0}: {1}".format(self.__class__.__name__, msg)
+        self._details = details
+        if enable_logging:
+            logger.error(self._msg)
+            logger.debug(self._details)
+
+    def __repr__(self):
+        return repr(self._msg)
+
+    def __str__(self):
+        return str(self._msg)

 
 
 class HoneycombUtil(object):
 
 
 class HoneycombUtil(object):

-    """Implements keywords used with Honeycomb."""
+    """Implements low level functionality used in communication with Honeycomb.
+
+    There are implemented methods to get, put and delete data to/from Honeycomb.
+    They are based on functionality implemented in the module HTTPRequests which
+    uses HTTP requests GET, PUT, POST and DELETE to communicate with Honeycomb.
+
+    It is possible to PUT the data represented as XML or JSON structures or as
+    plain text.
+    Data received in the response of GET are always represented as a JSON
+    structure.
+
+    There are also two supportive methods implemented:
+    - read_path_from_url_file which reads URL file and returns a path (see
+      docs/honeycomb_url_files.rst).
+    - parse_json_response which parses data from response in JSON representation
+      according to given path.
+    """

 
     def __init__(self):
         pass
 
 
     def __init__(self):
         pass
 

-    def get_configured_topology(self, nodes):
-        """Retrieves topology node IDs from each honeycomb node.
+    @staticmethod
+    def read_path_from_url_file(url_file):
+        """Read path from *.url file.

 
 

-        :param nodes: all nodes in topology
-        :type nodes: dict
-        :return: list of string IDs such as ['vpp1', 'vpp2']
-        :rtype list
+        For more information about *.url file see docs/honeycomb_url_files.rst
+        :param url_file: URL file. The argument contains only the name of file
+        without extension, not the full path.
+        :type url_file: str
+        :return: Requested path.
+        :rtype: str

         """
 
         """
 

-        url_file = os.path.join(C.RESOURCES_TPL_HC, "config_topology.url")
+        url_file = "{0}/{1}.url".format(Const.RESOURCES_TPL_HC, url_file)

         with open(url_file) as template:
             path = template.readline()
         with open(url_file) as template:
             path = template.readline()

+        return path

 
 

-        data = []
-        for node in nodes.values():
-            if node['type'] == NodeType.DUT:
-                _, ret = HTTPRequest.get(node, path)
-                logger.debug('return: {0}'.format(ret))
-                data.append(self.parse_json_response(ret, ("topology",
-                                                           "node", "node-id")))
-
-        return data
-
-    def parse_json_response(self, response, path=None):
+    @staticmethod
+    def parse_json_response(response, path=None):

         """Parse data from response string in JSON format according to given
         path.
 
         """Parse data from response string in JSON format according to given
         path.
 

-        :param response: JSON formatted string
-        :param path: Path to navigate down the data structure
+        :param response: JSON formatted string.
+        :param path: Path to navigate down the data structure.

         :type response: string
         :type path: tuple
         :type response: string
         :type path: tuple

-        :return: JSON dictionary/list tree
-        :rtype: dict
+        :return: JSON dictionary/list tree.
+        :rtype: list

         """
         data = loads(response)
 
         if path:
         """
         data = loads(response)
 
         if path:

-            data = self._parse_json_tree(data, path)
-            while isinstance(data, list) and len(data) == 1:
-                data = data[0]
+            data = HoneycombUtil._parse_json_tree(data, path)
+            if not isinstance(data, list):
+                data = [data, ]

 
         return data
 
 
         return data
 

-    def _parse_json_tree(self, data, path):
-        """Retrieve data from python representation of JSON object.
+    @staticmethod
+    def _parse_json_tree(data, path):
+        """Retrieve data addressed by path from python representation of JSON
+        object.

 
 

-        :param data: parsed JSON dictionary tree
-        :param path: Path to navigate down the dictionary tree
+        :param data: Parsed JSON dictionary tree.
+        :param path: Path to navigate down the dictionary tree.

         :type data: dict
         :type path: tuple
         :type data: dict
         :type path: tuple

-        :return: data from specified path
-        :rtype: list or str
+        :return: Data from specified path.
+        :rtype: list, dict or str

         """
 
         count = 0
         """
 
         count = 0


@@ -91,7 +152,80 @@ class HoneycombUtil(object):
             elif isinstance(data, list):
                 result = []
                 for item in data:
             elif isinstance(data, list):
                 result = []
                 for item in data:

-                    result.append(self._parse_json_tree(item, path[count:]))
+                    result.append(HoneycombUtil._parse_json_tree(item,
+                                                                 path[count:]))

                     return result
                     return result

-

         return data
         return data

+
+    @staticmethod
+    def get_honeycomb_data(node, url_file):
+        """Retrieve data from Honeycomb according to given URL.
+
+        :param node: Honeycomb node.
+        :param url_file: URL file. The argument contains only the name of file
+        without extension, not the full path.
+        :type node: dict
+        :type url_file: str
+        :return: Requested information.
+        :rtype list
+        """
+
+        path = HoneycombUtil.read_path_from_url_file(url_file)
+        status_code, resp = HTTPRequest.get(node, path)
+        return status_code, resp
+
+    @staticmethod
+    def put_honeycomb_data(node, url_file, data, data_representation='json'):
+        """Send configuration data using PUT request and return the status code
+        and response.
+
+        :param node: Honeycomb node.
+        :param url_file: URL file. The argument contains only the name of file
+        without extension, not the full path.
+        :param data: Configuration data to be sent to Honeycomb.
+        :param data_representation: How the data is represented. Supported types
+        of representation are: json, xml and txt.
+        :type node: dict
+        :type url_file: str
+        :type data: str
+        :type data_representation: str
+        :return: Status code and content of response.
+        :rtype: tuple
+        """
+
+        headers = {'json':
+                       {"Content-Type": "application/json",
+                        'Accept': 'text/plain'},
+                   'xml':
+                       {"Content-Type": "application/xml",
+                        'Accept': 'text/plain'},
+                   'txt':
+                       {"Content-Type": "text/plain",
+                        'Accept': 'text/plain'}
+                  }
+        try:
+            header = headers[data_representation]
+        except KeyError as err:
+            raise HoneycombError("Wrong data type: {0}.".
+                                 format(data_representation), repr(err))
+
+        path = HoneycombUtil.read_path_from_url_file(url_file)
+        status_code, resp = HTTPRequest.put(node=node, path=path,
+                                            headers=header, payload=data)
+        return status_code, resp
+
+    @staticmethod
+    def delete_honeycomb_data(node, url_file):
+        """Delete data from Honeycomb according to given URL.
+
+        :param node: Honeycomb node.
+        :param url_file: URL file. The argument contains only the name of file
+        without extension, not the full path.
+        :type node: dict
+        :type url_file: str
+        :return: Status code and response.
+        :rtype tuple
+        """
+
+        path = HoneycombUtil.read_path_from_url_file(url_file)
+        return HTTPRequest.delete(node, path)

diff --git a/resources/templates/honeycomb/vpp_version.url b/resources/templates/honeycomb/oper_vpp_version.url
similarity index 100%
rename from resources/templates/honeycomb/vpp_version.url
rename to resources/templates/honeycomb/oper_vpp_version.url