-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
import json
import os
+import yaml
from robot.api import logger
+from resources.libraries.python.Constants import Constants
-class VppApiCrcChecker(object):
+
+def _str(text):
+ """Convert from possible bytes without interpreting as number.
+
+ :param text: Input to convert.
+ :type text: str or unicode
+ :returns: Converted text.
+ :rtype: str
+ """
+ return text.decode(u"utf-8") if isinstance(text, bytes) else text
+
+
+class VppApiCrcChecker:
"""Holder of data related to tracking VPP API CRCs.
+ Both message names and crc hexa strings are tracked as
+ ordinary Python3 (unicode) string, so _str() is used when input is
+ possibly bytes or otherwise not safe.
+
Each instance of this class starts with same default state,
so make sure the calling libraries have appropriate robot library scope.
For usual testing, it means "GLOBAL" scope."""
- def __init__(self, directory):
+ def __init__(
+ self, directory, fail_on_mismatch=Constants.FAIL_ON_CRC_MISMATCH):
"""Initialize empty state, then register known collections.
This also scans directory for .api.json files
:type directory: str
"""
+ self.fail_on_mismatch = fail_on_mismatch
+ """If True, mismatch leads to test failure, by raising exception.
+ If False, the mismatch is logged, but the test is allowed to continue.
+ """
+
self._expected = dict()
"""Mapping from collection name to mapping from API name to CRC string.
- Colection name should be something useful for logging.
- API name is ordinary Python2 str, CRC is also str.
+ Collection name should be something useful for logging.
- Order of addition reflects the order colections should be queried.
+ Order of addition reflects the order collections should be queried.
If an incompatible CRC is found, affected collections are removed.
A CRC that would remove all does not, added to _reported instead,
- while causing a failure in single test.
- """
+ while causing a failure in single test (if fail_on_mismatch)."""
self._missing = dict()
"""Mapping from collection name to mapping from API name to CRC string.
Starts the same as _expected, but each time an encountered api,crc pair
- fits the expectation, the pair is removed from this mapping.
- Ideally, the active mappings will become empty.
- If not, it is an error, VPP removed or renamed a message CSIT needs."""
+ fits the expectation, the pair is removed from all collections
+ within this mapping. It is fine if an api is missing
+ from some collections, as long as it is not missing from all collections
+ that remained in _expected."""
self._found = dict()
"""Mapping from API name to CRC string.
This gets populated with CRCs found in .api.json,
to serve as a hint when reporting errors."""
+ self._options = dict()
+ """Mapping from API name to options dictionary.
+
+ This gets populated with options found in .api.json,
+ to serve as a hint when reporting errors."""
+
self._reported = dict()
"""Mapping from API name to CRC string.
self._register_all()
self._check_dir(directory)
- def _register_collection(self, collection_name, collection_dict):
+ def log_and_raise(self, exc_msg):
+ """Log to console, on fail_on_mismatch also raise runtime exception.
+
+ :param exc_msg: The message to include in log or exception.
+ :type exc_msg: str
+ :raises RuntimeError: With the message, if fail_on_mismatch.
+ """
+ logger.console("RuntimeError:\n{m}".format(m=exc_msg))
+ if self.fail_on_mismatch:
+ raise RuntimeError(exc_msg)
+
+ def _register_collection(self, collection_name, name_to_crc_mapping):
"""Add a named (copy of) collection of CRCs.
:param collection_name: Helpful string describing the collection.
- :param collection_dict: Mapping from API names to CRCs.
- :type collection_name: str
- :type collection_dict: dict from str to str
+ :param name_to_crc_mapping: Mapping from API names to CRCs.
+ :type collection_name: str or unicode
+ :type name_to_crc_mapping: dict from str/unicode to str/unicode
+ :raises RuntimeError: If the name of a collection is registered already.
"""
+ collection_name = _str(collection_name)
if collection_name in self._expected:
- raise RuntimeError("Collection {cl} already registered.".format(
- cl=collection_name))
- self._expected[collection_name] = collection_dict.copy()
- self._missing[collection_name] = collection_dict.copy()
+ raise RuntimeError(
+ f"Collection {collection_name!r} already registered."
+ )
+ mapping = {_str(k): _str(v) for k, v in name_to_crc_mapping.items()}
+ self._expected[collection_name] = mapping
+ self._missing[collection_name] = mapping.copy()
+
+ def _register_all(self):
+ """Add all collections this CSIT codebase is tested against."""
+
+ file_path = os.path.normpath(os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), u"..", u"..",
+ u"api", u"vpp", u"supported_crcs.yaml"))
+ with open(file_path, u"rt") as file_in:
+ collections_dict = yaml.safe_load(file_in.read())
+ for collection_name, name_to_crc_mapping in collections_dict.items():
+ self._register_collection(collection_name, name_to_crc_mapping)
@staticmethod
def _get_name(msg_obj):
for item in msg_obj:
if isinstance(item, (dict, list)):
continue
- return item
- raise RuntimeError("No name found for message: {obj!r}".format(
- obj=msg_obj))
+ return _str(item)
+ raise RuntimeError(f"No name found for message: {msg_obj!r}")
@staticmethod
def _get_crc(msg_obj):
for item in reversed(msg_obj):
if not isinstance(item, dict):
continue
- crc = item.get("crc", None)
+ crc = item.get(u"crc", None)
if crc:
- return crc
- raise RuntimeError("No CRC found for message: {obj!r}".format(
- obj=msg_obj))
+ return _str(crc)
+ raise RuntimeError(f"No CRC found for message: {msg_obj!r}")
+
+ @staticmethod
+ def _get_options(msg_obj, version):
+ """Utility function to extract API options from an intermediate json.
- def _process_crc(self, api_name, crc):
+ Empty dict is returned if options are not found,
+ so old VPP builds can be tested without spamming.
+ If version starts with "0.", add a fake option,
+ as the message is treated as "in-progress" by the API upgrade process.
+
+ :param msg_obj: Loaded json object, item of "messages" list.
+ :param version: Version string from the .api.json file.
+ :type msg_obj: list of various types
+ :type version: Optional[str]
+ :returns: Object found as value for "options" key.
+ :rtype: dict
+ """
+ options = dict()
+ for item in reversed(msg_obj):
+ if not isinstance(item, dict):
+ continue
+ options = item.get(u"options", dict())
+ if not options:
+ break
+ if version is None or version.startswith(u"0."):
+ options[u"version"] = version
+ return options
+
+ def _process_crc(self, api_name, crc, options):
"""Compare API to verified collections, update class state.
+ Here, API stands for (message name, CRC) pair.
+
Conflict is NOT when a collection does not recognize the API.
Such APIs are merely added to _found for later reporting.
Conflict is when a collection recognizes the API under a different CRC.
Attempts to overwrite value in _found or _reported should not happen,
so the code does not check for that, simply overwriting.
+ Options are stored, to be examined later.
+
The intended usage is to call this method multiple times,
and then raise exception listing all _reported.
:param api_name: API name to check.
:param crc: Discovered CRC to check for the name.
+ :param options: Empty dict or options value for in .api.json
:type api_name: str
- :type crc: str or unicode
+ :type crc: str
+ :type options: dict
"""
# Regardless of the result, remember as found.
self._found[api_name] = crc
+ self._options[api_name] = options
old_expected = self._expected
new_expected = old_expected.copy()
- for collection_name, collection_dict in old_expected.items():
- if api_name not in collection_dict:
+ for collection_name, name_to_crc_mapping in old_expected.items():
+ if api_name not in name_to_crc_mapping:
continue
- if collection_dict[api_name] == crc:
+ if name_to_crc_mapping[api_name] == crc:
self._missing[collection_name].pop(api_name, None)
continue
# Remove the offending collection.
self._expected = new_expected
self._missing = {name: self._missing[name] for name in new_expected}
return
- # No new_expected means some colections knew the api_name,
+ # No new_expected means some collections knew the api_name,
# but CRC does not match any. This has to be reported.
self._reported[api_name] = crc
"""Parse every .api.json found under directory, remember conflicts.
As several collections are supported, each conflict invalidates
- one of them, failure happens only when no collections would be left.
+ some of them, failure happens only when no collections would be left.
In that case, set of collections just before the failure is preserved,
the _reported mapping is filled with conflicting APIs.
The _found mapping is filled with discovered api names and crcs.
"""
for root, _, files in os.walk(directory):
for filename in files:
- if not filename.endswith(".api.json"):
+ if not filename.endswith(u".api.json"):
continue
- with open(root + '/' + filename, "r") as file_in:
+ with open(f"{root}/{filename}", u"rt") as file_in:
json_obj = json.load(file_in)
- msgs = json_obj["messages"]
+ version = json_obj[u"options"].get(u"version", None)
+ msgs = json_obj[u"messages"]
for msg_obj in msgs:
msg_name = self._get_name(msg_obj)
msg_crc = self._get_crc(msg_obj)
- self._process_crc(msg_name, msg_crc)
- logger.info("Surviving collections: {col}".format(
- col=self._expected.keys()))
+ msg_options = self._get_options(msg_obj, version)
+ self._process_crc(msg_name, msg_crc, msg_options)
+ logger.debug(f"Surviving collections: {self._expected.keys()!r}")
def report_initial_conflicts(self, report_missing=False):
"""Report issues discovered by _check_dir, if not done that already.
Missing reporting is disabled by default, because some messages
come from plugins that might not be enabled at runtime.
+ After the report, clear _reported, so that test cases report them again,
+ thus tracking which message is actually used (by which test).
+
:param report_missing: Whether to raise on missing messages.
:type report_missing: bool
- :raises RuntimeError: If CRC mismatch or missing messages are detected.
+ :raises RuntimeError: If CRC mismatch or missing messages are detected,
+ and fail_on_mismatch is True.
"""
if self._initial_conflicts_reported:
return
self._initial_conflicts_reported = True
if self._reported:
- raise RuntimeError("Dir check found incompatible API CRCs: {rep!r}"\
- .format(rep=self._reported))
+ reported_indented = json.dumps(
+ self._reported, indent=1, sort_keys=True,
+ separators=[u",", u":"]
+ )
+ self._reported = dict()
+ self.log_and_raise(
+ f"Incompatible API CRCs found in .api.json files:\n"
+ f"{reported_indented}"
+ )
if not report_missing:
return
missing = {name: mapp for name, mapp in self._missing.items() if mapp}
- if missing:
- raise RuntimeError("Dir check found missing API CRCs: {mis!r}"\
- .format(mis=missing))
+ if set(missing.keys()) < set(self._expected.keys()):
+ # There is a collection where nothing is missing.
+ return
+ missing_indented = json.dumps(
+ missing, indent=1, sort_keys=True, separators=[u",", u":"]
+ )
+ self.log_and_raise(
+ f"API CRCs missing from .api.json:\n{missing_indented}"
+ )
def check_api_name(self, api_name):
- """Fail if the api_name has no known CRC associated.
+ """Fail if the api_name has no, or different from known CRC associated.
+
+ Print warning if options contain anything more than vat_help.
Do not fail if this particular failure has been already reported.
- Intended use: Call everytime an API call is queued or response received.
+ Intended use: Call during test (not in initialization),
+ every time an API call is queued or response received.
- :param api_name: VPP API messagee name to check.
- :type api_name: str
+ :param api_name: VPP API message name to check.
+ :type api_name: str or unicode
:raises RuntimeError: If no verified CRC for the api_name is found.
"""
+ api_name = _str(api_name)
if api_name in self._reported:
return
old_expected = self._expected
new_expected = old_expected.copy()
- for collection_name, collection_dict in old_expected.items():
- if api_name in collection_dict:
+ for collection_name, name_to_crc_mapping in old_expected.items():
+ if api_name in name_to_crc_mapping:
continue
# Remove the offending collection.
new_expected.pop(collection_name, None)
if new_expected:
# Some collections recognized the message name.
self._expected = new_expected
- return
crc = self._found.get(api_name, None)
- self._reported[api_name] = crc
- # Disabled temporarily during CRC mismatch.
- #raise RuntimeError("No active collection has API {api} CRC found {crc}"\
- # .format(api=api_name, crc=crc))
+ matching = False
+ if crc is not None:
+ # Regardless of how many collections are remaining,
+ # verify the known CRC is on one of them.
+ for name_to_crc_mapping in self._expected.values():
+ if api_name not in name_to_crc_mapping:
+ continue
+ if name_to_crc_mapping[api_name] == crc:
+ matching = True
+ break
+ if not matching:
+ self._reported[api_name] = crc
+ self.log_and_raise(
+ f"No active collection has API {api_name!r} with CRC {crc!r}"
+ )
+ options = self._options.get(api_name, None)
+ if not options:
+ # None means CSIT is attempting a new API on an old VPP build.
+ # If that is an issue, the API has been reported as missing already.
+ return
+ options.pop(u"vat_help", None)
+ if options:
+ self._reported[api_name] = crc
+ logger.console(f"{api_name} used but has options {options}")
- # Moved to the end as this part will be edited frequently.
- def _register_all(self):
- """Add all collections this CSIT codebase is tested against."""
+ def print_warnings(self):
+ """Call check_api_name for API names in surviving collections.
- # Rework to read from files?
- self._register_collection(
- "19.08-rc0~762-gbb2e5221a", {
- "acl_add_replace": "0x13bc8539", # perf
- "acl_add_replace_reply": "0xac407b0c", # perf
- "acl_dump": "0xef34fea4", # perf teardown
- "acl_interface_list_dump": "0x529cb13f", # perf teardown
- # ^^^^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr
- "acl_interface_set_acl_list": "0x8baece38", # perf
- "acl_interface_set_acl_list_reply": "0xe8d4e804", # perf
- "acl_details": "0xf89d7a88", # perf teardown
- "acl_interface_list_details": "0xd5e80809", # perf teardown
- # ^^^^ tc01-64B-1c-ethip4udp-ip4base-iacl1sl-10kflows-mrr
- # ^^ ip4fwdANDiaclANDacl10AND100_flows
- "avf_create": "0xdaab8ae2", # perf
- "avf_create_reply": "0xfda5941f", # perf
- # ^^ tc01-64B-1c-avf-eth-l2bdbasemaclrn-mrr
- # ^ l2bdmaclrnANDbaseANDdrv_avf
- "bridge_domain_add_del": "0xc6360720", # dev
- "bridge_domain_add_del_reply": "0xe8d4e804", # dev
- "classify_add_del_session": "0x85fd79f4", # dev
- "classify_add_del_session_reply": "0xe8d4e804", # dev
- "classify_add_del_table": "0x9bd794ae", # dev
- "classify_add_del_table_reply": "0x05486349", # dev
- "cli_inband": "0xb1ad59b3", # dev setup
- "cli_inband_reply": "0x6d3c80a4", # dev setup
- "cop_interface_enable_disable": "0x69d24598", # dev
- "cop_interface_enable_disable_reply": "0xe8d4e804", # dev
- "cop_whitelist_enable_disable": "0x8bb8f6dc", # dev
- "cop_whitelist_enable_disable_reply": "0xe8d4e804", # dev
- "create_loopback": "0x3b54129c", # dev
- "create_loopback_reply": "0xfda5941f", # dev
- "create_subif": "0x86cfe408", # virl
- "create_subif_reply": "0xfda5941f", # virl
- "create_vhost_user_if": "0xbd230b87", # dev
- "create_vhost_user_if_reply": "0xfda5941f", # dev
- "create_vlan_subif": "0x70cadeda", # virl
- "create_vlan_subif_reply": "0xfda5941f", # virl
- "gre_tunnel_add_del": "0x04199f47", # virl
- "gre_tunnel_add_del_reply": "0x903324db", # virl
- "gpe_enable_disable": "0xeb0e943b", # virl
- "gpe_enable_disable_reply": "0xe8d4e804", # virl
- "hw_interface_set_mtu": "0x132da1e7", # dev
- "hw_interface_set_mtu_reply": "0xe8d4e804", # dev
- "input_acl_set_interface": "0xe09537b0", # dev
- "input_acl_set_interface_reply": "0xe8d4e804", # dev
- "ip_address_details": "0x2f1dbc7d", # dev
- "ip_address_dump": "0x6b7bcd0a", # dev
- "ip_neighbor_add_del": "0x7a68a3c4", # dev
- "ip_neighbor_add_del_reply": "0x1992deab", # dev
- "ip_probe_neighbor": "0x2736142d", # virl
- "ip_route_add_del": "0x83e086ce", # dev
- "ip_route_add_del_reply": "0x1992deab", # dev
- "ip_source_check_interface_add_del": "0x0a60152a", # virl
- "ip_source_check_interface_add_del_reply": "0xe8d4e804", # virl
- "ip_table_add_del": "0xe5d378f2", # dev
- "ip_table_add_del_reply": "0xe8d4e804", # dev
- "ipsec_interface_add_del_spd": "0x1e3b8286", # dev
- "ipsec_interface_add_del_spd_reply": "0xe8d4e804", # dev
- "ipsec_sad_entry_add_del": "0xa25ab61e", # dev
- "ipsec_sad_entry_add_del_reply": "0x9ffac24b", # dev
- "ipsec_spd_add_del": "0x9ffdf5da", # dev
- "ipsec_spd_add_del_reply": "0xe8d4e804", # dev
- "ipsec_spd_entry_add_del": "0x6bc6a3b5", # dev
- "ipsec_spd_entry_add_del_reply": "0x9ffac24b", # dev
- "l2_interface_vlan_tag_rewrite": "0xb90be6b4", # virl
- "l2_interface_vlan_tag_rewrite_reply": "0xe8d4e804", # virl
- "l2_patch_add_del": "0x62506e63", # perf
- "l2_patch_add_del_reply": "0xe8d4e804", # perf
- # ^^ tc01-64B-1c-avf-eth-l2patch-mrr
- # ^ l2patchANDdrv_avf
- "lisp_add_del_adjacency": "0xf047390d", # virl
- "lisp_add_del_adjacency_reply": "0xe8d4e804", # virl
- "lisp_add_del_local_eid": "0xe6d00717", # virl
- "lisp_add_del_local_eid_reply": "0xe8d4e804", # virl
- "lisp_add_del_locator": "0x006a4240", # virl
- "lisp_add_del_locator_reply": "0xe8d4e804", # virl
- "lisp_add_del_locator_set": "0x06968e38", # virl
- "lisp_add_del_locator_set_reply": "0xb6666db4", # virl
- "lisp_add_del_remote_mapping": "0xb879c3a9", # virl
- "lisp_add_del_remote_mapping_reply": "0xe8d4e804", # virl
- "lisp_eid_table_details": "0xdcd9f414", # virl
- "lisp_eid_table_dump": "0xe0df64da", # virl
- "lisp_enable_disable": "0xeb0e943b", # virl
- "lisp_enable_disable_reply": "0xe8d4e804", # virl
- "lisp_locator_set_details": "0x6b846882", # virl
- "lisp_locator_set_dump": "0xc79e8ab0", # virl
- "lisp_map_resolver_details": "0x60a5f5ca", # virl
- "lisp_map_resolver_dump": "0x51077d14", # virl
- "memif_create": "0x6597cdb2", # dev
- "memif_create_reply": "0xfda5941f", # dev
- "memif_details": "0x4f5a3397", # dev
- "memif_dump": "0x51077d14", # dev
- "memif_socket_filename_add_del": "0x30e3929d", # dev
- "memif_socket_filename_add_del_reply": "0xe8d4e804", # dev
- "nat_det_add_del_map": "0x04b76549", # perf
- "nat_det_add_del_map_reply": "0xe8d4e804", # perf
- "nat44_interface_add_del_feature": "0xef3edad1", # perf
- "nat44_interface_add_del_feature_reply": "0xe8d4e804", # perf
- # ^^^^ tc01-64B-1c-ethip4udp-ip4base-nat44-mrr
- # ^ nat44NOTscaleNOTsrc_user_1
- "proxy_arp_intfc_enable_disable": "0x69d24598", # virl
- "proxy_arp_intfc_enable_disable_reply": "0xe8d4e804", # virl
- "show_lisp_status": "0x51077d14", # virl
- "show_lisp_status_reply": "0xddcf48ef", # virl
- "show_threads": "0x51077d14", # dev
- "show_threads_reply": "0xf5e0b66f", # dev
- "show_version": "0x51077d14", # dev setup
- "show_version_reply": "0xb9bcf6df", # dev setup
- "sw_interface_add_del_address": "0x7b583179", # dev
- "sw_interface_add_del_address_reply": "0xe8d4e804", # dev
- "sw_interface_details": "0xe4ee7eb6", # dev setup
- "sw_interface_dump": "0x052753c5", # dev setup
- "sw_interface_ip6nd_ra_config": "0xc3f02daa", # dev
- "sw_interface_ip6nd_ra_config_reply": "0xe8d4e804", # dev
- "sw_interface_rx_placement_details": "0x0e9e33f4", # perf
- "sw_interface_rx_placement_dump": "0x529cb13f", # perf
- # ^^ tc01-64B-1c-dot1q-l2bdbasemaclrn-eth-2memif-1dcr-mrr
- # ^ dot1qANDl2bdmaclrnANDbaseANDmemif
- "sw_interface_set_flags": "0x555485f5", # dev
- "sw_interface_set_flags_reply": "0xe8d4e804", # dev
- "sw_interface_set_l2_bridge": "0x5579f809", # dev
- "sw_interface_set_l2_bridge_reply": "0xe8d4e804", # dev
- "sw_interface_set_l2_xconnect": "0x95de3988", # dev
- "sw_interface_set_l2_xconnect_reply": "0xe8d4e804", # dev
- "sw_interface_set_rx_placement": "0x4ef4377d", # perf
- "sw_interface_set_rx_placement_reply": "0xe8d4e804", # perf
- # ^^ tc01-64B-1c-eth-l2xcbase-eth-2memif-1dcr-mrr
- # ^ l2xcfwdANDbaseANDlxcANDmemif
- "sw_interface_set_table": "0xacb25d89", # dev
- "sw_interface_set_table_reply": "0xe8d4e804", # dev
- "sw_interface_set_vxlan_bypass": "0xe74ca095", # dev
- "sw_interface_set_vxlan_bypass_reply": "0xe8d4e804", # dev
- "sw_interface_vhost_user_details": "0x91ff3307", # dev
- "sw_interface_vhost_user_dump": "0x51077d14", # dev
- "vxlan_add_del_tunnel": "0x00f4bdd0", # virl
- "vxlan_add_del_tunnel_reply": "0xfda5941f", # virl
- "vxlan_tunnel_details": "0xce38e127", # virl
- "vxlan_tunnel_dump": "0x529cb13f", # virl
- }
- # Perf verify with: csit-3n-skx-perftest
- # mrrAND1cAND64bANDnic_intel-x710ANDip4fwdANDiaclANDacl10AND100_flows
- # mrrAND1cAND64bANDnic_intel-x710ANDl2bdmaclrnANDbaseANDdrv_avf
- # mrrAND1cAND64bANDnic_intel-x710ANDl2patchANDdrv_avf
- # mrrAND1cAND64bANDnic_intel-x710ANDnat44NOTscaleNOTsrc_user_1
- # mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDl2bdmaclrnANDbaseANDmemif
- # mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDbaseANDlxcANDmemif
-
- # TODO: Add a tag expression for covering those perf tests,
- # even though any CSIT change can make that outdated.
- # TODO: Once API coverage job is ready,
- # add a check to make sure each message was encountered;
- # failure means we need to add more tests to API coverage job.
- # Alternatively, add an option to compile messages actually
- # used or encountered, so CSIT knows what to remove from mapping.
- )
+ Useful for VPP CRC checking job.
+ The API name is only checked when it appears
+ in all surviving collections.
+ """
+ api_name_to_crc_maps = self._expected.values()
+ api_name_sets = (set(n2c.keys()) for n2c in api_name_to_crc_maps)
+ api_names = set.intersection(*api_name_sets)
+ for api_name in sorted(api_names):
+ self.check_api_name(api_name)
Code Review / csit.git / blobdiff