Add 2048B file size cps rps tests in job specs for http-ldpreload-nginx-1_21_5.

[csit.git] / resources / libraries / python / VppApiCrc.py

diff --git a/resources/libraries/python/VppApiCrc.py b/resources/libraries/python/VppApiCrc.py
index 28d66d3..a8947a1 100644 (file)
--- a/resources/libraries/python/VppApiCrc.py
+++ b/resources/libraries/python/VppApiCrc.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -15,18 +15,37 @@
 
 import json
 import os
 
 import json
 import os

+import yaml

 
 from robot.api import logger
 
 
 from robot.api import logger
 

+from resources.libraries.python.Constants import Constants

 
 

-class VppApiCrcChecker(object):
+
+def _str(text):
+    """Convert from possible bytes without interpreting as number.
+
+    :param text: Input to convert.
+    :type text: str or unicode
+    :returns: Converted text.
+    :rtype: str
+    """
+    return text.decode(u"utf-8") if isinstance(text, bytes) else text
+
+
+class VppApiCrcChecker:

     """Holder of data related to tracking VPP API CRCs.
 
     """Holder of data related to tracking VPP API CRCs.
 

+    Both message names and crc hexa strings are tracked as
+    ordinary Python3 (unicode) string, so _str() is used when input is
+    possibly bytes or otherwise not safe.
+

     Each instance of this class starts with same default state,
     so make sure the calling libraries have appropriate robot library scope.
     For usual testing, it means "GLOBAL" scope."""
 
     Each instance of this class starts with same default state,
     so make sure the calling libraries have appropriate robot library scope.
     For usual testing, it means "GLOBAL" scope."""
 

-    def __init__(self, directory):
+    def __init__(
+            self, directory, fail_on_mismatch=Constants.FAIL_ON_CRC_MISMATCH):

         """Initialize empty state, then register known collections.
 
         This also scans directory for .api.json files
         """Initialize empty state, then register known collections.
 
         This also scans directory for .api.json files


@@ -36,25 +55,29 @@ class VppApiCrcChecker(object):
         :type directory: str
         """
 
         :type directory: str
         """
 

+        self.fail_on_mismatch = fail_on_mismatch
+        """If True, mismatch leads to test failure, by raising exception.
+        If False, the mismatch is logged, but the test is allowed to continue.
+        """
+

         self._expected = dict()
         """Mapping from collection name to mapping from API name to CRC string.
 
         self._expected = dict()
         """Mapping from collection name to mapping from API name to CRC string.
 

-        Colection name should be something useful for logging.
-        API name is ordinary Python2 str, CRC is also str.
+        Collection name should be something useful for logging.

 
 

-        Order of addition reflects the order colections should be queried.
+        Order of addition reflects the order collections should be queried.

         If an incompatible CRC is found, affected collections are removed.
         A CRC that would remove all does not, added to _reported instead,
         If an incompatible CRC is found, affected collections are removed.
         A CRC that would remove all does not, added to _reported instead,

-        while causing a failure in single test.
-        """
+        while causing a failure in single test (if fail_on_mismatch)."""

 
         self._missing = dict()
         """Mapping from collection name to mapping from API name to CRC string.
 
         Starts the same as _expected, but each time an encountered api,crc pair
 
         self._missing = dict()
         """Mapping from collection name to mapping from API name to CRC string.
 
         Starts the same as _expected, but each time an encountered api,crc pair

-        fits the expectation, the pair is removed from this mapping.
-        Ideally, the active mappings will become empty.
-        If not, it is an error, VPP removed or renamed a message CSIT needs."""
+        fits the expectation, the pair is removed from all collections
+        within this mapping. It is fine if an api is missing
+        from some collections, as long as it is not missing from all collections
+        that remained in _expected."""

 
         self._found = dict()
         """Mapping from API name to CRC string.
 
         self._found = dict()
         """Mapping from API name to CRC string.


@@ -62,6 +85,12 @@ class VppApiCrcChecker(object):
         This gets populated with CRCs found in .api.json,
         to serve as a hint when reporting errors."""
 
         This gets populated with CRCs found in .api.json,
         to serve as a hint when reporting errors."""
 

+        self._options = dict()
+        """Mapping from API name to options dictionary.
+
+        This gets populated with options found in .api.json,
+        to serve as a hint when reporting errors."""
+

         self._reported = dict()
         """Mapping from API name to CRC string.
 
         self._reported = dict()
         """Mapping from API name to CRC string.
 


@@ -75,19 +104,45 @@ class VppApiCrcChecker(object):
         self._register_all()
         self._check_dir(directory)
 
         self._register_all()
         self._check_dir(directory)
 

-    def _register_collection(self, collection_name, collection_dict):
+    def log_and_raise(self, exc_msg):
+        """Log to console, on fail_on_mismatch also raise runtime exception.
+
+        :param exc_msg: The message to include in log or exception.
+        :type exc_msg: str
+        :raises RuntimeError: With the message, if fail_on_mismatch.
+        """
+        logger.console("RuntimeError:\n{m}".format(m=exc_msg))
+        if self.fail_on_mismatch:
+            raise RuntimeError(exc_msg)
+
+    def _register_collection(self, collection_name, name_to_crc_mapping):

         """Add a named (copy of) collection of CRCs.
 
         :param collection_name: Helpful string describing the collection.
         """Add a named (copy of) collection of CRCs.
 
         :param collection_name: Helpful string describing the collection.

-        :param collection_dict: Mapping from API names to CRCs.
-        :type collection_name: str
-        :type collection_dict: dict from str to str
+        :param name_to_crc_mapping: Mapping from API names to CRCs.
+        :type collection_name: str or unicode
+        :type name_to_crc_mapping: dict from str/unicode to str/unicode
+        :raises RuntimeError: If the name of a collection is registered already.

         """
         """

+        collection_name = _str(collection_name)

         if collection_name in self._expected:
         if collection_name in self._expected:

-            raise RuntimeError("Collection {cl} already registered.".format(
-                cl=collection_name))
-        self._expected[collection_name] = collection_dict.copy()
-        self._missing[collection_name] = collection_dict.copy()
+            raise RuntimeError(
+                f"Collection {collection_name!r} already registered."
+            )
+        mapping = {_str(k): _str(v) for k, v in name_to_crc_mapping.items()}
+        self._expected[collection_name] = mapping
+        self._missing[collection_name] = mapping.copy()
+
+    def _register_all(self):
+        """Add all collections this CSIT codebase is tested against."""
+
+        file_path = os.path.normpath(os.path.join(
+            os.path.dirname(os.path.abspath(__file__)), u"..", u"..",
+            u"api", u"vpp", u"supported_crcs.yaml"))
+        with open(file_path, u"rt") as file_in:
+            collections_dict = yaml.safe_load(file_in.read())
+        for collection_name, name_to_crc_mapping in collections_dict.items():
+            self._register_collection(collection_name, name_to_crc_mapping)

 
     @staticmethod
     def _get_name(msg_obj):
 
     @staticmethod
     def _get_name(msg_obj):


@@ -102,9 +157,8 @@ class VppApiCrcChecker(object):
         for item in msg_obj:
             if isinstance(item, (dict, list)):
                 continue
         for item in msg_obj:
             if isinstance(item, (dict, list)):
                 continue

-            return item
-        raise RuntimeError("No name found for message: {obj!r}".format(
-            obj=msg_obj))
+            return _str(item)
+        raise RuntimeError(f"No name found for message: {msg_obj!r}")

 
     @staticmethod
     def _get_crc(msg_obj):
 
     @staticmethod
     def _get_crc(msg_obj):


@@ -119,15 +173,43 @@ class VppApiCrcChecker(object):
         for item in reversed(msg_obj):
             if not isinstance(item, dict):
                 continue
         for item in reversed(msg_obj):
             if not isinstance(item, dict):
                 continue

-            crc = item.get("crc", None)
+            crc = item.get(u"crc", None)

             if crc:
             if crc:

-                return crc
-        raise RuntimeError("No CRC found for message: {obj!r}".format(
-            obj=msg_obj))
+                return _str(crc)
+        raise RuntimeError(f"No CRC found for message: {msg_obj!r}")
+
+    @staticmethod
+    def _get_options(msg_obj, version):
+        """Utility function to extract API options from an intermediate json.

 
 

-    def _process_crc(self, api_name, crc):
+        Empty dict is returned if options are not found,
+        so old VPP builds can be tested without spamming.
+        If version starts with "0.", add a fake option,
+        as the message is treated as "in-progress" by the API upgrade process.
+
+        :param msg_obj: Loaded json object, item of "messages" list.
+        :param version: Version string from the .api.json file.
+        :type msg_obj: list of various types
+        :type version: Optional[str]
+        :returns: Object found as value for "options" key.
+        :rtype: dict
+        """
+        options = dict()
+        for item in reversed(msg_obj):
+            if not isinstance(item, dict):
+                continue
+            options = item.get(u"options", dict())
+            if not options:
+                break
+        if version is None or version.startswith(u"0."):
+            options[u"version"] = version
+        return options
+
+    def _process_crc(self, api_name, crc, options):

         """Compare API to verified collections, update class state.
 
         """Compare API to verified collections, update class state.
 

+        Here, API stands for (message name, CRC) pair.
+

         Conflict is NOT when a collection does not recognize the API.
         Such APIs are merely added to _found for later reporting.
         Conflict is when a collection recognizes the API under a different CRC.
         Conflict is NOT when a collection does not recognize the API.
         Such APIs are merely added to _found for later reporting.
         Conflict is when a collection recognizes the API under a different CRC.


@@ -147,22 +229,27 @@ class VppApiCrcChecker(object):
         Attempts to overwrite value in _found or _reported should not happen,
         so the code does not check for that, simply overwriting.
 
         Attempts to overwrite value in _found or _reported should not happen,
         so the code does not check for that, simply overwriting.
 

+        Options are stored, to be examined later.
+

         The intended usage is to call this method multiple times,
         and then raise exception listing all _reported.
 
         :param api_name: API name to check.
         :param crc: Discovered CRC to check for the name.
         The intended usage is to call this method multiple times,
         and then raise exception listing all _reported.
 
         :param api_name: API name to check.
         :param crc: Discovered CRC to check for the name.

+        :param options: Empty dict or options value for in .api.json

         :type api_name: str
         :type api_name: str

-        :type crc: str or unicode
+        :type crc: str
+        :type options: dict

         """
         # Regardless of the result, remember as found.
         self._found[api_name] = crc
         """
         # Regardless of the result, remember as found.
         self._found[api_name] = crc

+        self._options[api_name] = options

         old_expected = self._expected
         new_expected = old_expected.copy()
         old_expected = self._expected
         new_expected = old_expected.copy()

-        for collection_name, collection_dict in old_expected.items():
-            if api_name not in collection_dict:
+        for collection_name, name_to_crc_mapping in old_expected.items():
+            if api_name not in name_to_crc_mapping:

                 continue
                 continue

-            if collection_dict[api_name] == crc:
+            if name_to_crc_mapping[api_name] == crc:

                 self._missing[collection_name].pop(api_name, None)
                 continue
             # Remove the offending collection.
                 self._missing[collection_name].pop(api_name, None)
                 continue
             # Remove the offending collection.


@@ -172,7 +259,7 @@ class VppApiCrcChecker(object):
             self._expected = new_expected
             self._missing = {name: self._missing[name] for name in new_expected}
             return
             self._expected = new_expected
             self._missing = {name: self._missing[name] for name in new_expected}
             return

-        # No new_expected means some colections knew the api_name,
+        # No new_expected means some collections knew the api_name,

         # but CRC does not match any. This has to be reported.
         self._reported[api_name] = crc
 
         # but CRC does not match any. This has to be reported.
         self._reported[api_name] = crc
 


@@ -180,7 +267,7 @@ class VppApiCrcChecker(object):
         """Parse every .api.json found under directory, remember conflicts.
 
         As several collections are supported, each conflict invalidates
         """Parse every .api.json found under directory, remember conflicts.
 
         As several collections are supported, each conflict invalidates

-        one of them, failure happens only when no collections would be left.
+        some of them, failure happens only when no collections would be left.

         In that case, set of collections just before the failure is preserved,
         the _reported mapping is filled with conflicting APIs.
         The _found mapping is filled with discovered api names and crcs.
         In that case, set of collections just before the failure is preserved,
         the _reported mapping is filled with conflicting APIs.
         The _found mapping is filled with discovered api names and crcs.


@@ -192,17 +279,18 @@ class VppApiCrcChecker(object):
         """
         for root, _, files in os.walk(directory):
             for filename in files:
         """
         for root, _, files in os.walk(directory):
             for filename in files:

-                if not filename.endswith(".api.json"):
+                if not filename.endswith(u".api.json"):

                     continue
                     continue

-                with open(root + '/' + filename, "r") as file_in:
+                with open(f"{root}/{filename}", u"rt") as file_in:

                     json_obj = json.load(file_in)
                     json_obj = json.load(file_in)

-                msgs = json_obj["messages"]
+                version = json_obj[u"options"].get(u"version", None)
+                msgs = json_obj[u"messages"]

                 for msg_obj in msgs:
                     msg_name = self._get_name(msg_obj)
                     msg_crc = self._get_crc(msg_obj)
                 for msg_obj in msgs:
                     msg_name = self._get_name(msg_obj)
                     msg_crc = self._get_crc(msg_obj)

-                    self._process_crc(msg_name, msg_crc)
-        logger.info("Surviving collections: {col}".format(
-            col=self._expected.keys()))
+                    msg_options = self._get_options(msg_obj, version)
+                    self._process_crc(msg_name, msg_crc, msg_options)
+        logger.debug(f"Surviving collections: {self._expected.keys()!r}")

 
     def report_initial_conflicts(self, report_missing=False):
         """Report issues discovered by _check_dir, if not done that already.
 
     def report_initial_conflicts(self, report_missing=False):
         """Report issues discovered by _check_dir, if not done that already.


@@ -214,210 +302,102 @@ class VppApiCrcChecker(object):
         Missing reporting is disabled by default, because some messages
         come from plugins that might not be enabled at runtime.
 
         Missing reporting is disabled by default, because some messages
         come from plugins that might not be enabled at runtime.
 

+        After the report, clear _reported, so that test cases report them again,
+        thus tracking which message is actually used (by which test).
+

         :param report_missing: Whether to raise on missing messages.
         :type report_missing: bool
         :param report_missing: Whether to raise on missing messages.
         :type report_missing: bool

-        :raises RuntimeError: If CRC mismatch or missing messages are detected.
+        :raises RuntimeError: If CRC mismatch or missing messages are detected,
+            and fail_on_mismatch is True.

         """
         if self._initial_conflicts_reported:
             return
         self._initial_conflicts_reported = True
         if self._reported:
         """
         if self._initial_conflicts_reported:
             return
         self._initial_conflicts_reported = True
         if self._reported:

-            raise RuntimeError("Dir check found incompatible API CRCs: {rep!r}"\
-                .format(rep=self._reported))
+            reported_indented = json.dumps(
+                self._reported, indent=1, sort_keys=True,
+                separators=[u",", u":"]
+            )
+            self._reported = dict()
+            self.log_and_raise(
+                f"Incompatible API CRCs found in .api.json files:\n"
+                f"{reported_indented}"
+            )

         if not report_missing:
             return
         missing = {name: mapp for name, mapp in self._missing.items() if mapp}
         if not report_missing:
             return
         missing = {name: mapp for name, mapp in self._missing.items() if mapp}

-        if missing:
-            raise RuntimeError("Dir check found missing API CRCs: {mis!r}"\
-                .format(mis=missing))
+        if set(missing.keys()) < set(self._expected.keys()):
+            # There is a collection where nothing is missing.
+            return
+        missing_indented = json.dumps(
+            missing, indent=1, sort_keys=True, separators=[u",", u":"]
+        )
+        self.log_and_raise(
+            f"API CRCs missing from .api.json:\n{missing_indented}"
+        )

 
     def check_api_name(self, api_name):
 
     def check_api_name(self, api_name):

-        """Fail if the api_name has no known CRC associated.
+        """Fail if the api_name has no, or different from known CRC associated.
+
+        Print warning if options contain anything more than vat_help.

 
         Do not fail if this particular failure has been already reported.
 
 
         Do not fail if this particular failure has been already reported.
 

-        Intended use: Call everytime an API call is queued or response received.
+        Intended use: Call during test (not in initialization),
+        every time an API call is queued or response received.

 
 

-        :param api_name: VPP API messagee name to check.
-        :type api_name: str
+        :param api_name: VPP API message name to check.
+        :type api_name: str or unicode

         :raises RuntimeError: If no verified CRC for the api_name is found.
         """
         :raises RuntimeError: If no verified CRC for the api_name is found.
         """

+        api_name = _str(api_name)

         if api_name in self._reported:
             return
         old_expected = self._expected
         new_expected = old_expected.copy()
         if api_name in self._reported:
             return
         old_expected = self._expected
         new_expected = old_expected.copy()

-        for collection_name, collection_dict in old_expected.items():
-            if api_name in collection_dict:
+        for collection_name, name_to_crc_mapping in old_expected.items():
+            if api_name in name_to_crc_mapping:

                 continue
             # Remove the offending collection.
             new_expected.pop(collection_name, None)
         if new_expected:
             # Some collections recognized the message name.
             self._expected = new_expected
                 continue
             # Remove the offending collection.
             new_expected.pop(collection_name, None)
         if new_expected:
             # Some collections recognized the message name.
             self._expected = new_expected

-            return

         crc = self._found.get(api_name, None)
         crc = self._found.get(api_name, None)

-        self._reported[api_name] = crc
-        # Disabled temporarily during CRC mismatch.
-        #raise RuntimeError("No active collection has API {api} CRC found {crc}"\
-        #    .format(api=api_name, crc=crc))
+        matching = False
+        if crc is not None:
+            # Regardless of how many collections are remaining,
+            # verify the known CRC is on one of them.
+            for name_to_crc_mapping in self._expected.values():
+                if api_name not in name_to_crc_mapping:
+                    continue
+                if name_to_crc_mapping[api_name] == crc:
+                    matching = True
+                    break
+        if not matching:
+            self._reported[api_name] = crc
+            self.log_and_raise(
+                f"No active collection has API {api_name!r} with CRC {crc!r}"
+            )
+        options = self._options.get(api_name, None)
+        if not options:
+            # None means CSIT is attempting a new API on an old VPP build.
+            # If that is an issue, the API has been reported as missing already.
+            return
+        options.pop(u"vat_help", None)
+        if options:
+            self._reported[api_name] = crc
+            logger.console(f"{api_name} used but has options {options}")

 
 

-    # Moved to the end as this part will be edited frequently.
-    def _register_all(self):
-        """Add all collections this CSIT codebase is tested against."""
+    def print_warnings(self):
+        """Call check_api_name for API names in surviving collections.

 
 

-        # Rework to read from files?
-        self._register_collection(
-            "19.08-rc0~762-gbb2e5221a", {
-                "acl_add_replace": "0x13bc8539",  # perf
-                "acl_add_replace_reply": "0xac407b0c",  # perf
-                "acl_dump": "0xef34fea4",  # perf teardown
-                "acl_interface_list_dump": "0x529cb13f",  # perf teardown
-                # ^^^^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr
-                "acl_interface_set_acl_list": "0x8baece38",  # perf
-                "acl_interface_set_acl_list_reply": "0xe8d4e804",  # perf
-                "acl_details": "0xf89d7a88",  # perf teardown
-                "acl_interface_list_details": "0xd5e80809",  # perf teardown
-                # ^^^^ tc01-64B-1c-ethip4udp-ip4base-iacl1sl-10kflows-mrr
-                # ^^ ip4fwdANDiaclANDacl10AND100_flows
-                "avf_create": "0xdaab8ae2",  # perf
-                "avf_create_reply": "0xfda5941f",  # perf
-                # ^^ tc01-64B-1c-avf-eth-l2bdbasemaclrn-mrr
-                # ^ l2bdmaclrnANDbaseANDdrv_avf
-                "bridge_domain_add_del": "0xc6360720",  # dev
-                "bridge_domain_add_del_reply": "0xe8d4e804",  # dev
-                "classify_add_del_session": "0x85fd79f4",  # dev
-                "classify_add_del_session_reply": "0xe8d4e804",  # dev
-                "classify_add_del_table": "0x9bd794ae",  # dev
-                "classify_add_del_table_reply": "0x05486349",  # dev
-                "cli_inband": "0xb1ad59b3",  # dev setup
-                "cli_inband_reply": "0x6d3c80a4",  # dev setup
-                "cop_interface_enable_disable": "0x69d24598",  # dev
-                "cop_interface_enable_disable_reply": "0xe8d4e804",  # dev
-                "cop_whitelist_enable_disable": "0x8bb8f6dc",  # dev
-                "cop_whitelist_enable_disable_reply": "0xe8d4e804",  # dev
-                "create_loopback": "0x3b54129c",  # dev
-                "create_loopback_reply": "0xfda5941f",  # dev
-                "create_subif": "0x86cfe408",  # virl
-                "create_subif_reply": "0xfda5941f",  # virl
-                "create_vhost_user_if": "0xbd230b87",  # dev
-                "create_vhost_user_if_reply": "0xfda5941f",  # dev
-                "create_vlan_subif": "0x70cadeda",  # virl
-                "create_vlan_subif_reply": "0xfda5941f",  # virl
-                "gre_tunnel_add_del": "0x04199f47",  # virl
-                "gre_tunnel_add_del_reply": "0x903324db",  # virl
-                "gpe_enable_disable": "0xeb0e943b",  # virl
-                "gpe_enable_disable_reply": "0xe8d4e804",  # virl
-                "hw_interface_set_mtu": "0x132da1e7",  # dev
-                "hw_interface_set_mtu_reply": "0xe8d4e804",  # dev
-                "input_acl_set_interface": "0xe09537b0",  # dev
-                "input_acl_set_interface_reply": "0xe8d4e804",  # dev
-                "ip_address_details": "0x2f1dbc7d",  # dev
-                "ip_address_dump": "0x6b7bcd0a",  # dev
-                "ip_neighbor_add_del": "0x7a68a3c4",  # dev
-                "ip_neighbor_add_del_reply": "0x1992deab",  # dev
-                "ip_probe_neighbor": "0x2736142d",  # virl
-                "ip_route_add_del": "0x83e086ce",  # dev
-                "ip_route_add_del_reply": "0x1992deab",  # dev
-                "ip_source_check_interface_add_del": "0x0a60152a",  # virl
-                "ip_source_check_interface_add_del_reply": "0xe8d4e804",  # virl
-                "ip_table_add_del": "0xe5d378f2",  # dev
-                "ip_table_add_del_reply": "0xe8d4e804",  # dev
-                "ipsec_interface_add_del_spd": "0x1e3b8286",  # dev
-                "ipsec_interface_add_del_spd_reply": "0xe8d4e804",  # dev
-                "ipsec_sad_entry_add_del": "0xa25ab61e",  # dev
-                "ipsec_sad_entry_add_del_reply": "0x9ffac24b",  # dev
-                "ipsec_spd_add_del": "0x9ffdf5da",  # dev
-                "ipsec_spd_add_del_reply": "0xe8d4e804",  # dev
-                "ipsec_spd_entry_add_del": "0x6bc6a3b5",  # dev
-                "ipsec_spd_entry_add_del_reply": "0x9ffac24b",  # dev
-                "l2_interface_vlan_tag_rewrite": "0xb90be6b4",  # virl
-                "l2_interface_vlan_tag_rewrite_reply": "0xe8d4e804",  # virl
-                "l2_patch_add_del": "0x62506e63",  # perf
-                "l2_patch_add_del_reply": "0xe8d4e804",  # perf
-                # ^^ tc01-64B-1c-avf-eth-l2patch-mrr
-                # ^ l2patchANDdrv_avf
-                "lisp_add_del_adjacency": "0xf047390d",  # virl
-                "lisp_add_del_adjacency_reply": "0xe8d4e804",  # virl
-                "lisp_add_del_local_eid": "0xe6d00717",  # virl
-                "lisp_add_del_local_eid_reply": "0xe8d4e804",  # virl
-                "lisp_add_del_locator": "0x006a4240",  # virl
-                "lisp_add_del_locator_reply": "0xe8d4e804",  # virl
-                "lisp_add_del_locator_set": "0x06968e38",  # virl
-                "lisp_add_del_locator_set_reply": "0xb6666db4",  # virl
-                "lisp_add_del_remote_mapping": "0xb879c3a9",  # virl
-                "lisp_add_del_remote_mapping_reply": "0xe8d4e804",  # virl
-                "lisp_eid_table_details": "0xdcd9f414",  # virl
-                "lisp_eid_table_dump": "0xe0df64da",  # virl
-                "lisp_enable_disable": "0xeb0e943b",  # virl
-                "lisp_enable_disable_reply": "0xe8d4e804",  # virl
-                "lisp_locator_set_details": "0x6b846882",  # virl
-                "lisp_locator_set_dump": "0xc79e8ab0",  # virl
-                "lisp_map_resolver_details": "0x60a5f5ca",  # virl
-                "lisp_map_resolver_dump": "0x51077d14",  # virl
-                "memif_create": "0x6597cdb2",  # dev
-                "memif_create_reply": "0xfda5941f",  # dev
-                "memif_details": "0x4f5a3397",  # dev
-                "memif_dump": "0x51077d14",  # dev
-                "memif_socket_filename_add_del": "0x30e3929d",  # dev
-                "memif_socket_filename_add_del_reply": "0xe8d4e804",  # dev
-                "nat_det_add_del_map": "0x04b76549",  # perf
-                "nat_det_add_del_map_reply": "0xe8d4e804",  # perf
-                "nat44_interface_add_del_feature": "0xef3edad1",  # perf
-                "nat44_interface_add_del_feature_reply": "0xe8d4e804",  # perf
-                # ^^^^ tc01-64B-1c-ethip4udp-ip4base-nat44-mrr
-                # ^ nat44NOTscaleNOTsrc_user_1
-                "proxy_arp_intfc_enable_disable": "0x69d24598",  # virl
-                "proxy_arp_intfc_enable_disable_reply": "0xe8d4e804",  # virl
-                "show_lisp_status": "0x51077d14",  # virl
-                "show_lisp_status_reply": "0xddcf48ef",  # virl
-                "show_threads": "0x51077d14",  # dev
-                "show_threads_reply": "0xf5e0b66f",  # dev
-                "show_version": "0x51077d14",  # dev setup
-                "show_version_reply": "0xb9bcf6df",  # dev setup
-                "sw_interface_add_del_address": "0x7b583179",  # dev
-                "sw_interface_add_del_address_reply": "0xe8d4e804",  # dev
-                "sw_interface_details": "0xe4ee7eb6",  # dev setup
-                "sw_interface_dump": "0x052753c5",  # dev setup
-                "sw_interface_ip6nd_ra_config": "0xc3f02daa",  # dev
-                "sw_interface_ip6nd_ra_config_reply": "0xe8d4e804",  # dev
-                "sw_interface_rx_placement_details": "0x0e9e33f4",  # perf
-                "sw_interface_rx_placement_dump": "0x529cb13f",  # perf
-                # ^^ tc01-64B-1c-dot1q-l2bdbasemaclrn-eth-2memif-1dcr-mrr
-                # ^ dot1qANDl2bdmaclrnANDbaseANDmemif
-                "sw_interface_set_flags": "0x555485f5",  # dev
-                "sw_interface_set_flags_reply": "0xe8d4e804",  # dev
-                "sw_interface_set_l2_bridge": "0x5579f809",  # dev
-                "sw_interface_set_l2_bridge_reply": "0xe8d4e804",  # dev
-                "sw_interface_set_l2_xconnect": "0x95de3988",  # dev
-                "sw_interface_set_l2_xconnect_reply": "0xe8d4e804",  # dev
-                "sw_interface_set_rx_placement": "0x4ef4377d",  # perf
-                "sw_interface_set_rx_placement_reply": "0xe8d4e804",  # perf
-                # ^^ tc01-64B-1c-eth-l2xcbase-eth-2memif-1dcr-mrr
-                # ^ l2xcfwdANDbaseANDlxcANDmemif
-                "sw_interface_set_table": "0xacb25d89",  # dev
-                "sw_interface_set_table_reply": "0xe8d4e804",  # dev
-                "sw_interface_set_vxlan_bypass": "0xe74ca095",  # dev
-                "sw_interface_set_vxlan_bypass_reply": "0xe8d4e804",  # dev
-                "sw_interface_vhost_user_details": "0x91ff3307",  # dev
-                "sw_interface_vhost_user_dump": "0x51077d14",  # dev
-                "vxlan_add_del_tunnel": "0x00f4bdd0",  # virl
-                "vxlan_add_del_tunnel_reply": "0xfda5941f",  # virl
-                "vxlan_tunnel_details": "0xce38e127",  # virl
-                "vxlan_tunnel_dump": "0x529cb13f",  # virl
-            }
-            # Perf verify with: csit-3n-skx-perftest
-            # mrrAND1cAND64bANDnic_intel-x710ANDip4fwdANDiaclANDacl10AND100_flows
-            # mrrAND1cAND64bANDnic_intel-x710ANDl2bdmaclrnANDbaseANDdrv_avf
-            # mrrAND1cAND64bANDnic_intel-x710ANDl2patchANDdrv_avf
-            # mrrAND1cAND64bANDnic_intel-x710ANDnat44NOTscaleNOTsrc_user_1
-            # mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDl2bdmaclrnANDbaseANDmemif
-            # mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDbaseANDlxcANDmemif
-
-            # TODO: Add a tag expression for covering those perf tests,
-            # even though any CSIT change can make that outdated.
-            # TODO: Once API coverage job is ready,
-            # add a check to make sure each message was encountered;
-            # failure means we need to add more tests to API coverage job.
-            # Alternatively, add an option to compile messages actually
-            # used or encountered, so CSIT knows what to remove from mapping.
-        )
+        Useful for VPP CRC checking job.
+        The API name is only checked when it appears
+        in all surviving collections.
+        """
+        api_name_to_crc_maps = self._expected.values()
+        api_name_sets = (set(n2c.keys()) for n2c in api_name_to_crc_maps)
+        api_names = set.intersection(*api_name_sets)
+        for api_name in sorted(api_names):
+            self.check_api_name(api_name)