Add VPP API CRC checking

+ Include both checking at runtime and standalone static quick check.
+ Runtime checking does not look for missing messages,
  as messages belonging to disabled plugins are not visible.
+ Standalone check script has nice loud output.

Change-Id: I1dfc3846d1bcdad0b09017d4ce8edd5028e17e0c
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

diff --git a/resources/libraries/bash/entry/check_crc.sh b/resources/libraries/bash/entry/check_crc.sh
new file mode 100644 (file)
index 0000000..6c91d28
--- /dev/null
+++ b/resources/libraries/bash/entry/check_crc.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+# This entry script does not change CSIT branch,
+# use "with_oper_for_vpp.sh" wrapper for that.
+#
+# Assumptions:
+# + There is a directory holding VPP repo with patch under test checked out.
+# + It contains csit subdirectory with CSIT code to use (this script is there).
+# + Everything needed to generate .api.json files is already installed locally.
+# Consequences:
+# + At the end, VPP repo has parent commit checked out and built.
+# + Directories build_root, build and csit are reset during the run.
+# + The following directory (relative to VPP repo) is (re)created:
+# ++ build-root/install-vpp-native/vpp/share/vpp/api
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/per_patch.sh" || die "Source failed."
+common_dirs || die
+set_perpatch_vpp_dir || die
+activate_virtualenv "${VPP_DIR}" || die
+# TODO: Create bash functions for the following one-liners?
+make json-api-files || die "Generation of .api.json files failed."
+python csit/resources/tools/integrated/check_crc.py
+# RuntimeError causes nonzero return code, to vote -1.

diff --git a/resources/libraries/python/PapiExecutor.py b/resources/libraries/python/PapiExecutor.py
index a3f2479..792fa4c 100644 (file)
--- a/resources/libraries/python/PapiExecutor.py
+++ b/resources/libraries/python/PapiExecutor.py
@@ -33,6 +33,7 @@ from resources.libraries.python.PythonThree import raise_from
 from resources.libraries.python.PapiHistory import PapiHistory
 from resources.libraries.python.ssh import (
     SSH, SSHTimeout, exec_cmd_no_error, scp_node)
 from resources.libraries.python.PapiHistory import PapiHistory
 from resources.libraries.python.ssh import (
     SSH, SSHTimeout, exec_cmd_no_error, scp_node)

+from resources.libraries.python.VppApiCrc import VppApiCrcChecker

 
 
 __all__ = ["PapiExecutor", "PapiSocketExecutor"]
 
 
 __all__ = ["PapiExecutor", "PapiSocketExecutor"]


@@ -67,6 +68,7 @@ def dictize(obj):
     ret.__getitem__ = new_get
     return ret
 
     ret.__getitem__ = new_get
     return ret
 

+

 class PapiSocketExecutor(object):
     """Methods for executing VPP Python API commands on forwarded socket.
 
 class PapiSocketExecutor(object):
     """Methods for executing VPP Python API commands on forwarded socket.
 


@@ -126,6 +128,8 @@ class PapiSocketExecutor(object):
 
     # Class cache for reuse between instances.
     cached_vpp_instance = None
 
     # Class cache for reuse between instances.
     cached_vpp_instance = None

+    api_json_directory = None
+    crc_checker_instance = None

 
     def __init__(self, node, remote_vpp_socket="/run/vpp-api.sock"):
         """Store the given arguments, declare managed variables.
 
     def __init__(self, node, remote_vpp_socket="/run/vpp-api.sock"):
         """Store the given arguments, declare managed variables.


@@ -144,6 +148,20 @@ class PapiSocketExecutor(object):
         self._ssh_control_socket = None
         self._local_vpp_socket = None
 
         self._ssh_control_socket = None
         self._local_vpp_socket = None
 

+    @property
+    def crc_checker(self):
+        """Return the cached instance or create new one from directory.
+
+        It is assumed self.api_json_directory is set, as a class variable.
+
+        :returns: Cached or newly created instance aware of .api.json content.
+        :rtype: VppApiCrc.VppApiCrcChecker
+        """
+        cls = self.__class__
+        if cls.crc_checker_instance is None:
+            cls.crc_checker_instance = VppApiCrcChecker(cls.api_json_directory)
+        return cls.crc_checker_instance
+

     @property
     def vpp_instance(self):
         """Return VPP instance with bindings to all API calls.
     @property
     def vpp_instance(self):
         """Return VPP instance with bindings to all API calls.


@@ -183,13 +201,17 @@ class PapiSocketExecutor(object):
             exec_cmd_no_error(node, ["bash", "-c", "'" + inner_cmd + "'"])
             scp_node(node, tmp_dir + "/papi.txz", "/tmp/papi.txz", get=True)
             run(["tar", "xf", tmp_dir + "/papi.txz", "-C", tmp_dir])
             exec_cmd_no_error(node, ["bash", "-c", "'" + inner_cmd + "'"])
             scp_node(node, tmp_dir + "/papi.txz", "/tmp/papi.txz", get=True)
             run(["tar", "xf", tmp_dir + "/papi.txz", "-C", tmp_dir])

+            cls.api_json_directory = tmp_dir + "/usr/share/vpp/api"
+            # Perform initial checks before .api.json files are gone,
+            # by accessing the property (which also creates its instance).
+            self.crc_checker

             # When present locally, we finally can find the installation path.
             package_path = glob.glob(tmp_dir + installed_papi_glob)[0]
             # Package path has to be one level above the vpp_papi directory.
             package_path = package_path.rsplit('/', 1)[0]
             sys.path.append(package_path)
             from vpp_papi.vpp_papi import VPPApiClient as vpp_class
             # When present locally, we finally can find the installation path.
             package_path = glob.glob(tmp_dir + installed_papi_glob)[0]
             # Package path has to be one level above the vpp_papi directory.
             package_path = package_path.rsplit('/', 1)[0]
             sys.path.append(package_path)
             from vpp_papi.vpp_papi import VPPApiClient as vpp_class

-            vpp_class.apidir = tmp_dir + "/usr/share/vpp/api"
+            vpp_class.apidir = cls.api_json_directory

             # We need to create instance before removing from sys.path.
             cls.cached_vpp_instance = vpp_class(
                 use_socket=True, server_address="TBD", async_thread=False,
             # We need to create instance before removing from sys.path.
             cls.cached_vpp_instance = vpp_class(
                 use_socket=True, server_address="TBD", async_thread=False,


@@ -315,6 +337,14 @@ class PapiSocketExecutor(object):
         be repeated in kwargs.
         Unless disabled, new entry to papi history is also added at this point.
 
         be repeated in kwargs.
         Unless disabled, new entry to papi history is also added at this point.
 

+        Any pending conflicts from .api.json processing are raised.
+        Then the command name is checked for known CRCs.
+        Unsupported commands raise an exception, as CSIT change
+        should not start using messages without making sure which CRCs
+        are supported.
+        Each CRC issue is raised only once, so subsequent tests
+        can raise other issues.
+

         :param csit_papi_command: VPP API command.
         :param history: Enable/disable adding command to PAPI command history.
         :param kwargs: Optional key-value arguments.
         :param csit_papi_command: VPP API command.
         :param history: Enable/disable adding command to PAPI command history.
         :param kwargs: Optional key-value arguments.


@@ -323,10 +353,13 @@ class PapiSocketExecutor(object):
         :type kwargs: dict
         :returns: self, so that method chaining is possible.
         :rtype: PapiSocketExecutor
         :type kwargs: dict
         :returns: self, so that method chaining is possible.
         :rtype: PapiSocketExecutor

+        :raises RuntimeError: If unverified or conflicting CRC is encountered.

         """
         """

+        self.crc_checker.report_initial_conflicts()

         if history:
             PapiHistory.add_to_papi_history(
                 self._node, csit_papi_command, **kwargs)
         if history:
             PapiHistory.add_to_papi_history(
                 self._node, csit_papi_command, **kwargs)

+        self.crc_checker.check_api_name(csit_papi_command)

         self._api_command_list.append(
             dict(api_name=csit_papi_command, api_args=kwargs))
         return self
         self._api_command_list.append(
             dict(api_name=csit_papi_command, api_args=kwargs))
         return self


@@ -486,6 +519,7 @@ class PapiSocketExecutor(object):
             if not isinstance(reply, list):
                 reply = [reply]
             for item in reply:
             if not isinstance(reply, list):
                 reply = [reply]
             for item in reply:

+                self.crc_checker.check_api_name(item.__class__.__name__)

                 dict_item = dictize(item)
                 if "retval" in dict_item.keys():
                     # *_details messages do not contain retval.
                 dict_item = dictize(item)
                 if "retval" in dict_item.keys():
                     # *_details messages do not contain retval.

diff --git a/resources/libraries/python/VppApiCrc.py b/resources/libraries/python/VppApiCrc.py
new file mode 100644 (file)
index 0000000..eba65ac
--- /dev/null
+++ b/resources/libraries/python/VppApiCrc.py
@@ -0,0 +1,422 @@
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module for keeping track of VPP API CRCs relied on by CSIT."""
+
+import json
+import os
+
+from robot.api import logger
+
+
+class VppApiCrcChecker(object):
+    """Holder of data related to tracking VPP API CRCs.
+
+    Each instance of this class starts with same default state,
+    so make sure the calling libraries have appropriate robot library scope.
+    For usual testing, it means "GLOBAL" scope."""
+
+    def __init__(self, directory):
+        """Initialize empty state, then register known collections.
+
+        This also scans directory for .api.json files
+        and performs initial checks, but does not report the findings yet.
+
+        :param directory: Root directory of the search for .api.json files.
+        :type directory: str
+        """
+
+        self._expected = dict()
+        """Mapping from collection name to mapping from API name to CRC string.
+
+        Colection name should be something useful for logging.
+        API name is ordinary Python2 str, CRC is also str.
+
+        Order of addition reflects the order colections should be queried.
+        If an incompatible CRC is found, affected collections are removed.
+        A CRC that would remove all does not, added to _reported instead,
+        while causing a failure in single test.
+        """
+
+        self._missing = dict()
+        """Mapping from collection name to mapping from API name to CRC string.
+
+        Starts the same as _expected, but each time an encountered api,crc pair
+        fits the expectation, the pair is removed from this mapping.
+        Ideally, the active mappings will become empty.
+        If not, it is an error, VPP removed or renamed a message CSIT needs."""
+
+        self._found = dict()
+        """Mapping from API name to CRC string.
+
+        This gets populated with CRCs found in .api.json,
+        to serve as a hint when reporting errors."""
+
+        self._reported = dict()
+        """Mapping from API name to CRC string.
+
+        This gets populated with APIs used, but not found in collections,
+        just before the fact is reported in an exception.
+        The CRC comes from _found mapping (otherwise left as None).
+        The idea is to not report those next time, allowing the job
+        to find more problems in a single run."""
+
+        self._initial_conflicts_reported = False
+        self._register_all()
+        self._check_dir(directory)
+
+    def _register_collection(self, collection_name, collection_dict):
+        """Add a named (copy of) collection of CRCs.
+
+        :param collection_name: Helpful string describing the collection.
+        :param collection_dict: Mapping from API names to CRCs.
+        :type collection_name: str
+        :type collection_dict: dict from str to str
+        """
+        if collection_name in self._expected:
+            raise RuntimeError("Collection {cl} already registered.".format(
+                cl=collection_name))
+        self._expected[collection_name] = collection_dict.copy()
+        self._missing[collection_name] = collection_dict.copy()
+
+    @staticmethod
+    def _get_name(msg_obj):
+        """Utility function to extract API name from an intermediate json.
+
+        :param msg_obj: Loaded json object, item of "messages" list.
+        :type msg_obj: list of various types
+        :returns: Name of the message.
+        :rtype: str or unicode
+        :raises RuntimeError: If no name is found.
+        """
+        for item in msg_obj:
+            if isinstance(item, (dict, list)):
+                continue
+            return item
+        raise RuntimeError("No name found for message: {obj!r}".format(
+            obj=msg_obj))
+
+    @staticmethod
+    def _get_crc(msg_obj):
+        """Utility function to extract API CRC from an intermediate json.
+
+        :param msg_obj: Loaded json object, item of "messages" list.
+        :type msg_obj: list of various types
+        :returns: CRC of the message.
+        :rtype: str or unicode
+        :raises RuntimeError: If no CRC is found.
+        """
+        for item in reversed(msg_obj):
+            if not isinstance(item, dict):
+                continue
+            crc = item.get("crc", None)
+            if crc:
+                return crc
+        raise RuntimeError("No CRC found for message: {obj!r}".format(
+            obj=msg_obj))
+
+    def _process_crc(self, api_name, crc):
+        """Compare API to verified collections, update class state.
+
+        Conflict is NOT when a collection does not recognize the API.
+        Such APIs are merely added to _found for later reporting.
+        Conflict is when a collection recognizes the API under a different CRC.
+        If a partial match happens, only the matching collections are preserved.
+        On no match, all current collections are preserved,
+        but the offending API is added to _reported mapping.
+
+        Note that it is expected that collections are incompatible
+        with each other for some APIs. The removal of collections
+        on partial match is there to help identify the intended collection
+        for the VPP build under test. But if no collection fits perfectly,
+        the last collections to determine the "known" flag
+        depends on the order of api_name submitted,
+        which tends to be fairly random (depends on order of .api.json files).
+        Order of collection registrations does not help much in this regard.
+
+        Attempts to overwrite value in _found or _reported should not happen,
+        so the code does not check for that, simply overwriting.
+
+        The intended usage is to call this method multiple times,
+        and then raise exception listing all _reported.
+
+        :param api_name: API name to check.
+        :param crc: Discovered CRC to check for the name.
+        :type api_name: str
+        :type crc: str or unicode
+        """
+        # Regardless of the result, remember as found.
+        self._found[api_name] = crc
+        old_expected = self._expected
+        new_expected = old_expected.copy()
+        for collection_name, collection_dict in old_expected.items():
+            if api_name not in collection_dict:
+                continue
+            if collection_dict[api_name] == crc:
+                self._missing[collection_name].pop(api_name, None)
+                continue
+            # Remove the offending collection.
+            new_expected.pop(collection_name, None)
+        if new_expected:
+            # Some collections recognized the CRC.
+            self._expected = new_expected
+            self._missing = {name: self._missing[name] for name in new_expected}
+            return
+        # No new_expected means some colections knew the api_name,
+        # but CRC does not match any. This has to be reported.
+        self._reported[api_name] = crc
+
+    def _check_dir(self, directory):
+        """Parse every .api.json found under directory, remember conflicts.
+
+        As several collections are supported, each conflict invalidates
+        one of them, failure happens only when no collections would be left.
+        In that case, set of collections just before the failure is preserved,
+        the _reported mapping is filled with conflicting APIs.
+        The _found mapping is filled with discovered api names and crcs.
+
+        The exception is not thrown here, but from report_initial_conflicts.
+
+        :param directory: Root directory of the search for .api.json files.
+        :type directory: str
+        """
+        for root, _, files in os.walk(directory):
+            for filename in files:
+                if not filename.endswith(".api.json"):
+                    continue
+                with open(root + '/' + filename, "r") as file_in:
+                    json_obj = json.load(file_in)
+                msgs = json_obj["messages"]
+                for msg_obj in msgs:
+                    msg_name = self._get_name(msg_obj)
+                    msg_crc = self._get_crc(msg_obj)
+                    self._process_crc(msg_name, msg_crc)
+        logger.info("Surviving collections: {col}".format(
+            col=self._expected.keys()))
+
+    def report_initial_conflicts(self, report_missing=False):
+        """Report issues discovered by _check_dir, if not done that already.
+
+        Intended use: Call once after init, at a time when throwing exception
+        is convenient.
+
+        Optionally, report also missing messages.
+        Missing reporting is disabled by default, because some messages
+        come from plugins that might not be enabled at runtime.
+
+        :param report_missing: Whether to raise on missing messages.
+        :type report_missing: bool
+        :raises RuntimeError: If CRC mismatch or missing messages are detected.
+        """
+        if self._initial_conflicts_reported:
+            return
+        self._initial_conflicts_reported = True
+        if self._reported:
+            raise RuntimeError("Dir check found incompatible API CRCs: {rep!r}"\
+                .format(rep=self._reported))
+        if not report_missing:
+            return
+        missing = {name: mapp for name, mapp in self._missing.items() if mapp}
+        if missing:
+            raise RuntimeError("Dir check found missing API CRCs: {mis!r}"\
+                .format(mis=missing))
+
+    def check_api_name(self, api_name):
+        """Fail if the api_name has no known CRC associated.
+
+        Do not fail if this particular failure has been already reported.
+
+        Intended use: Call everytime an API call is queued or response received.
+
+        :param api_name: VPP API messagee name to check.
+        :type api_name: str
+        :raises RuntimeError: If no verified CRC for the api_name is found.
+        """
+        if api_name in self._reported:
+            return
+        old_expected = self._expected
+        new_expected = old_expected.copy()
+        for collection_name, collection_dict in old_expected.items():
+            if api_name in collection_dict:
+                continue
+            # Remove the offending collection.
+            new_expected.pop(collection_name, None)
+        if new_expected:
+            # Some collections recognized the message name.
+            self._expected = new_expected
+            return
+        crc = self._found.get(api_name, None)
+        self._reported[api_name] = crc
+        raise RuntimeError("No active collection has API {api} CRC found {crc}"\
+            .format(api=api_name, crc=crc))
+
+    # Moved to the end as this part will be edited frequently.
+    def _register_all(self):
+        """Add all collections this CSIT codebase is tested against."""
+
+        # Rework to read from files?
+        self._register_collection(
+            "19.08-rc0~762-gbb2e5221a", {
+                "acl_add_replace": "0x13bc8539",  # perf
+                "acl_add_replace_reply": "0xac407b0c",  # perf
+                "acl_dump": "0xef34fea4",  # perf teardown
+                "acl_interface_list_dump": "0x529cb13f",  # perf teardown
+                # ^^^^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr
+                "acl_interface_set_acl_list": "0x8baece38",  # perf
+                "acl_interface_set_acl_list_reply": "0xe8d4e804",  # perf
+                "acl_details": "0xf89d7a88",  # perf teardown
+                "acl_interface_list_details": "0xd5e80809",  # perf teardown
+                # ^^^^ tc01-64B-1c-ethip4udp-ip4base-iacl1sl-10kflows-mrr
+                # ^^ ip4fwdANDiaclANDacl10AND100_flows
+                "avf_create": "0xdaab8ae2",  # perf
+                "avf_create_reply": "0xfda5941f",  # perf
+                # ^^ tc01-64B-1c-avf-eth-l2bdbasemaclrn-mrr
+                # ^ l2bdmaclrnANDbaseANDdrv_avf
+                "bridge_domain_add_del": "0xc6360720",  # dev
+                "bridge_domain_add_del_reply": "0xe8d4e804",  # dev
+                "classify_add_del_session": "0x85fd79f4",  # dev
+                "classify_add_del_session_reply": "0xe8d4e804",  # dev
+                "classify_add_del_table": "0x9bd794ae",  # dev
+                "classify_add_del_table_reply": "0x05486349",  # dev
+                "cli_inband": "0xb1ad59b3",  # dev setup
+                "cli_inband_reply": "0x6d3c80a4",  # dev setup
+                "cop_interface_enable_disable": "0x69d24598",  # dev
+                "cop_interface_enable_disable_reply": "0xe8d4e804",  # dev
+                "cop_whitelist_enable_disable": "0x8bb8f6dc",  # dev
+                "cop_whitelist_enable_disable_reply": "0xe8d4e804",  # dev
+                "create_loopback": "0x3b54129c",  # dev
+                "create_loopback_reply": "0xfda5941f",  # dev
+                "create_subif": "0x86cfe408",  # virl
+                "create_subif_reply": "0xfda5941f",  # virl
+                "create_vhost_user_if": "0xbd230b87",  # dev
+                "create_vhost_user_if_reply": "0xfda5941f",  # dev
+                "create_vlan_subif": "0x70cadeda",  # virl
+                "create_vlan_subif_reply": "0xfda5941f",  # virl
+                "gre_tunnel_add_del": "0x04199f47",  # virl
+                "gre_tunnel_add_del_reply": "0x903324db",  # virl
+                "gpe_enable_disable": "0xeb0e943b",  # virl
+                "gpe_enable_disable_reply": "0xe8d4e804",  # virl
+                "hw_interface_set_mtu": "0x132da1e7",  # dev
+                "hw_interface_set_mtu_reply": "0xe8d4e804",  # dev
+                "input_acl_set_interface": "0xe09537b0",  # dev
+                "input_acl_set_interface_reply": "0xe8d4e804",  # dev
+                "ip_address_details": "0x2f1dbc7d",  # dev
+                "ip_address_dump": "0x6b7bcd0a",  # dev
+                "ip_neighbor_add_del": "0x7a68a3c4",  # dev
+                "ip_neighbor_add_del_reply": "0x1992deab",  # dev
+                "ip_probe_neighbor": "0x2736142d",  # virl
+                "ip_route_add_del": "0x83e086ce",  # dev
+                "ip_route_add_del_reply": "0x1992deab",  # dev
+                "ip_source_check_interface_add_del": "0x0a60152a",  # virl
+                "ip_source_check_interface_add_del_reply": "0xe8d4e804",  # virl
+                "ip_table_add_del": "0xe5d378f2",  # dev
+                "ip_table_add_del_reply": "0xe8d4e804",  # dev
+                "ipsec_interface_add_del_spd": "0x1e3b8286",  # dev
+                "ipsec_interface_add_del_spd_reply": "0xe8d4e804",  # dev
+                "ipsec_sad_entry_add_del": "0xa25ab61e",  # dev
+                "ipsec_sad_entry_add_del_reply": "0x9ffac24b",  # dev
+                "ipsec_spd_add_del": "0x9ffdf5da",  # dev
+                "ipsec_spd_add_del_reply": "0xe8d4e804",  # dev
+                "ipsec_spd_entry_add_del": "0x6bc6a3b5",  # dev
+                "ipsec_spd_entry_add_del_reply": "0x9ffac24b",  # dev
+                "l2_interface_vlan_tag_rewrite": "0xb90be6b4",  # virl
+                "l2_interface_vlan_tag_rewrite_reply": "0xe8d4e804",  # virl
+                "l2_patch_add_del": "0x62506e63",  # perf
+                "l2_patch_add_del_reply": "0xe8d4e804",  # perf
+                # ^^ tc01-64B-1c-avf-eth-l2patch-mrr
+                # ^ l2patchANDdrv_avf
+                "lisp_add_del_adjacency": "0xf047390d",  # virl
+                "lisp_add_del_adjacency_reply": "0xe8d4e804",  # virl
+                "lisp_add_del_local_eid": "0xe6d00717",  # virl
+                "lisp_add_del_local_eid_reply": "0xe8d4e804",  # virl
+                "lisp_add_del_locator": "0x006a4240",  # virl
+                "lisp_add_del_locator_reply": "0xe8d4e804",  # virl
+                "lisp_add_del_locator_set": "0x06968e38",  # virl
+                "lisp_add_del_locator_set_reply": "0xb6666db4",  # virl
+                "lisp_add_del_remote_mapping": "0xb879c3a9",  # virl
+                "lisp_add_del_remote_mapping_reply": "0xe8d4e804",  # virl
+                "lisp_eid_table_details": "0xdcd9f414",  # virl
+                "lisp_eid_table_dump": "0xe0df64da",  # virl
+                "lisp_enable_disable": "0xeb0e943b",  # virl
+                "lisp_enable_disable_reply": "0xe8d4e804",  # virl
+                "lisp_locator_set_details": "0x6b846882",  # virl
+                "lisp_locator_set_dump": "0xc79e8ab0",  # virl
+                "lisp_map_resolver_details": "0x60a5f5ca",  # virl
+                "lisp_map_resolver_dump": "0x51077d14",  # virl
+                "memif_create": "0x6597cdb2",  # dev
+                "memif_create_reply": "0xfda5941f",  # dev
+                "memif_details": "0x4f5a3397",  # dev
+                "memif_dump": "0x51077d14",  # dev
+                "memif_socket_filename_add_del": "0x30e3929d",  # dev
+                "memif_socket_filename_add_del_reply": "0xe8d4e804",  # dev
+                "nat_det_add_del_map": "0x04b76549",  # perf
+                "nat_det_add_del_map_reply": "0xe8d4e804",  # perf
+                "nat44_interface_add_del_feature": "0xef3edad1",  # perf
+                "nat44_interface_add_del_feature_reply": "0xe8d4e804",  # perf
+                # ^^^^ tc01-64B-1c-ethip4udp-ip4base-nat44-mrr
+                # ^ nat44NOTscaleNOTsrc_user_1
+                "proxy_arp_intfc_enable_disable": "0x69d24598",  # virl
+                "proxy_arp_intfc_enable_disable_reply": "0xe8d4e804",  # virl
+                "show_lisp_status": "0x51077d14",  # virl
+                "show_lisp_status_reply": "0xddcf48ef",  # virl
+                "show_threads": "0x51077d14",  # dev
+                "show_threads_reply": "0xf5e0b66f",  # dev
+                "show_version": "0x51077d14",  # dev setup
+                "show_version_reply": "0xb9bcf6df",  # dev setup
+                "sw_interface_add_del_address": "0x7b583179",  # dev
+                "sw_interface_add_del_address_reply": "0xe8d4e804",  # dev
+                "sw_interface_details": "0xe4ee7eb6",  # dev setup
+                "sw_interface_dump": "0x052753c5",  # dev setup
+                "sw_interface_ip6nd_ra_config": "0xc3f02daa",  # dev
+                "sw_interface_ip6nd_ra_config_reply": "0xe8d4e804",  # dev
+                "sw_interface_rx_placement_details": "0x0e9e33f4",  # perf
+                "sw_interface_rx_placement_dump": "0x529cb13f",  # perf
+                # ^^ tc01-64B-1c-dot1q-l2bdbasemaclrn-eth-2memif-1dcr-mrr
+                # ^ dot1qANDl2bdmaclrnANDbaseANDmemif
+                "sw_interface_set_flags": "0x555485f5",  # dev
+                "sw_interface_set_flags_reply": "0xe8d4e804",  # dev
+                "sw_interface_set_l2_bridge": "0x5579f809",  # dev
+                "sw_interface_set_l2_bridge_reply": "0xe8d4e804",  # dev
+                "sw_interface_set_l2_xconnect": "0x95de3988",  # dev
+                "sw_interface_set_l2_xconnect_reply": "0xe8d4e804",  # dev
+                "sw_interface_set_rx_placement": "0x4ef4377d",  # perf
+                "sw_interface_set_rx_placement_reply": "0xe8d4e804",  # perf
+                # ^^ tc01-64B-1c-eth-l2xcbase-eth-2memif-1dcr-mrr
+                # ^ l2xcfwdANDbaseANDlxcANDmemif
+                "sw_interface_set_table": "0xacb25d89",  # dev
+                "sw_interface_set_table_reply": "0xe8d4e804",  # dev
+                "sw_interface_set_vxlan_bypass": "0xe74ca095",  # dev
+                "sw_interface_set_vxlan_bypass_reply": "0xe8d4e804",  # dev
+                "sw_interface_vhost_user_details": "0x91ff3307",  # dev
+                "sw_interface_vhost_user_dump": "0x51077d14",  # dev
+                "vxlan_add_del_tunnel": "0x00f4bdd0",  # virl
+                "vxlan_add_del_tunnel_reply": "0xfda5941f",  # virl
+                "vxlan_tunnel_details": "0xce38e127",  # virl
+                "vxlan_tunnel_dump": "0x529cb13f",  # virl
+            }
+            # Perf verify with: csit-3n-skx-perftest
+            # mrrAND1cAND64bANDnic_intel-x710ANDip4fwdANDiaclANDacl10AND100_flows
+            # mrrAND1cAND64bANDnic_intel-x710ANDl2bdmaclrnANDbaseANDdrv_avf
+            # mrrAND1cAND64bANDnic_intel-x710ANDl2patchANDdrv_avf
+            # mrrAND1cAND64bANDnic_intel-x710ANDnat44NOTscaleNOTsrc_user_1
+            # mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDl2bdmaclrnANDbaseANDmemif
+            # mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDbaseANDlxcANDmemif
+
+            # TODO: Add a tag expression for covering those perf tests,
+            # even though any CSIT change can make that outdated.
+            # TODO: Once API coverage job is ready,
+            # add a check to make sure each message was encountered;
+            # failure means we need to add more tests to API coverage job.
+            # Alternatively, add an option to compile messages actually
+            # used or encountered, so CSIT knows what to remove from mapping.
+        )

diff --git a/resources/tools/integrated/__init__.py b/resources/tools/integrated/__init__.py
new file mode 100644 (file)
index 0000000..a708b16
--- /dev/null
+++ b/resources/tools/integrated/__init__.py
@@ -0,0 +1,19 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+__init__ file for directory integrated
+
+Scripts in this directory require PYTHONPATH set to root CSIT directory,
+in order to import resources properly.
+"""

diff --git a/resources/tools/integrated/check_crc.py b/resources/tools/integrated/check_crc.py
new file mode 100644 (file)
index 0000000..3e2836c
--- /dev/null
+++ b/resources/tools/integrated/check_crc.py
@@ -0,0 +1,64 @@
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Script that fails if .api.json files downloaded to a hardcoded place
+do not match CRC checksusms currently supported by CSIT.
+
+No executable flag, nor shebang, as most users do not have .api.json
+files downloaded to the correct place.
+"""
+
+import os.path as op
+import sys
+
+from resources.libraries.python.VppApiCrc import VppApiCrcChecker
+
+# TODO: Read FDIO_VPP_DIR environment variable, or some other input,
+# instead of using hardcoded relative path?
+
+api_dir = op.normpath(op.join(
+    op.dirname(op.abspath(__file__)), "..", "..", "..", "..",
+    "build-root", "install-vpp-native", "vpp", "share", "vpp", "api"))
+checker = VppApiCrcChecker(api_dir)
+try:
+    checker.report_initial_conflicts(report_missing=True)
+except RuntimeError as err:
+    sys.stderr.write("{err!r}\n".format(err=err))
+    sys.stderr.write(
+        "\n"
+        "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n"
+        "\n"
+        "VPP CSIT API CHECK FAIL!\n"
+        "\n"
+        "This means the patch under test has missing messages,\n"
+        "or messages with unexpected CRCs compared to what CSIT needs.\n"
+        "Either this Change and/or its ancestors were editing .api files,\n"
+        "or your chain is not rebased upon the recent enough VPP codebase.\n"
+        "\n"
+        "Please rebase the patch to see if that fixes the problem.\n"
+        "If that fails email csit-dev@lists.fd.io for a new\n"
+        "operational branch supporting the api changes.\n"
+        "\n"
+        "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n"
+    )
+    sys.exit(1)
+else:
+    sys.stderr.write(
+        "\n"
+        "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n"
+        "\n"
+        "VPP CSIT API CHECK PASS!\n"
+        "\n"
+        "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n"
+    )