-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
from resources.libraries.python.Constants import Constants
+
def _str(text):
- """Convert from possible unicode without interpreting as number.
+ """Convert from possible bytes without interpreting as number.
:param text: Input to convert.
:type text: str or unicode
:returns: Converted text.
:rtype: str
"""
- return text.encode("utf-8") if isinstance(text, unicode) else text
+ return text.decode(u"utf-8") if isinstance(text, bytes) else text
-class VppApiCrcChecker(object):
+class VppApiCrcChecker:
"""Holder of data related to tracking VPP API CRCs.
Both message names and crc hexa strings are tracked as
- ordinary Python2 (bytes) string, so _str() is used when input is
- possibly unicode or otherwise not safe.
+ ordinary Python3 (unicode) string, so _str() is used when input is
+ possibly bytes or otherwise not safe.
Each instance of this class starts with same default state,
so make sure the calling libraries have appropriate robot library scope.
self._expected = dict()
"""Mapping from collection name to mapping from API name to CRC string.
- Colection name should be something useful for logging.
+ Collection name should be something useful for logging.
- Order of addition reflects the order colections should be queried.
+ Order of addition reflects the order collections should be queried.
If an incompatible CRC is found, affected collections are removed.
A CRC that would remove all does not, added to _reported instead,
while causing a failure in single test (if fail_on_mismatch)."""
This gets populated with CRCs found in .api.json,
to serve as a hint when reporting errors."""
+ self._options = dict()
+ """Mapping from API name to options dictionary.
+
+ This gets populated with options found in .api.json,
+ to serve as a hint when reporting errors."""
+
self._reported = dict()
"""Mapping from API name to CRC string.
"""Log to console, on fail_on_mismatch also raise runtime exception.
:param exc_msg: The message to include in log or exception.
- :type exception: str
+ :type exc_msg: str
:raises RuntimeError: With the message, if fail_on_mismatch.
"""
logger.console("RuntimeError:\n{m}".format(m=exc_msg))
"""
collection_name = _str(collection_name)
if collection_name in self._expected:
- raise RuntimeError("Collection {cn!r} already registered.".format(
- cn=collection_name))
+ raise RuntimeError(
+ f"Collection {collection_name!r} already registered."
+ )
mapping = {_str(k): _str(v) for k, v in name_to_crc_mapping.items()}
self._expected[collection_name] = mapping
self._missing[collection_name] = mapping.copy()
"""Add all collections this CSIT codebase is tested against."""
file_path = os.path.normpath(os.path.join(
- os.path.dirname(os.path.abspath(__file__)), "..", "..",
- "api", "vpp", "supported_crcs.yaml"))
- with open(file_path, "r") as file_in:
- collections_dict = yaml.load(file_in.read())
+ os.path.dirname(os.path.abspath(__file__)), u"..", u"..",
+ u"api", u"vpp", u"supported_crcs.yaml"))
+ with open(file_path, u"rt") as file_in:
+ collections_dict = yaml.safe_load(file_in.read())
for collection_name, name_to_crc_mapping in collections_dict.items():
self._register_collection(collection_name, name_to_crc_mapping)
if isinstance(item, (dict, list)):
continue
return _str(item)
- raise RuntimeError("No name found for message: {obj!r}".format(
- obj=msg_obj))
+ raise RuntimeError(f"No name found for message: {msg_obj!r}")
@staticmethod
def _get_crc(msg_obj):
for item in reversed(msg_obj):
if not isinstance(item, dict):
continue
- crc = item.get("crc", None)
+ crc = item.get(u"crc", None)
if crc:
return _str(crc)
- raise RuntimeError("No CRC found for message: {obj!r}".format(
- obj=msg_obj))
+ raise RuntimeError(f"No CRC found for message: {msg_obj!r}")
+
+ @staticmethod
+ def _get_options(msg_obj, version):
+ """Utility function to extract API options from an intermediate json.
+
+ Empty dict is returned if options are not found,
+ so old VPP builds can be tested without spamming.
+ If version starts with "0.", add a fake option,
+ as the message is treated as "in-progress" by the API upgrade process.
- def _process_crc(self, api_name, crc):
+ :param msg_obj: Loaded json object, item of "messages" list.
+ :param version: Version string from the .api.json file.
+ :type msg_obj: list of various types
+ :type version: Optional[str]
+ :returns: Object found as value for "options" key.
+ :rtype: dict
+ """
+ options = dict()
+ for item in reversed(msg_obj):
+ if not isinstance(item, dict):
+ continue
+ options = item.get(u"options", dict())
+ if not options:
+ break
+ if version is None or version.startswith(u"0."):
+ options[u"version"] = version
+ return options
+
+ def _process_crc(self, api_name, crc, options):
"""Compare API to verified collections, update class state.
Here, API stands for (message name, CRC) pair.
Attempts to overwrite value in _found or _reported should not happen,
so the code does not check for that, simply overwriting.
+ Options are stored, to be examined later.
+
The intended usage is to call this method multiple times,
and then raise exception listing all _reported.
:param api_name: API name to check.
:param crc: Discovered CRC to check for the name.
+ :param options: Empty dict or options value for in .api.json
:type api_name: str
:type crc: str
+ :type options: dict
"""
# Regardless of the result, remember as found.
self._found[api_name] = crc
+ self._options[api_name] = options
old_expected = self._expected
new_expected = old_expected.copy()
for collection_name, name_to_crc_mapping in old_expected.items():
self._expected = new_expected
self._missing = {name: self._missing[name] for name in new_expected}
return
- # No new_expected means some colections knew the api_name,
+ # No new_expected means some collections knew the api_name,
# but CRC does not match any. This has to be reported.
self._reported[api_name] = crc
"""
for root, _, files in os.walk(directory):
for filename in files:
- if not filename.endswith(".api.json"):
+ if not filename.endswith(u".api.json"):
continue
- with open(root + '/' + filename, "r") as file_in:
+ with open(f"{root}/{filename}", u"rt") as file_in:
json_obj = json.load(file_in)
- msgs = json_obj["messages"]
+ version = json_obj[u"options"].get(u"version", None)
+ msgs = json_obj[u"messages"]
for msg_obj in msgs:
msg_name = self._get_name(msg_obj)
msg_crc = self._get_crc(msg_obj)
- self._process_crc(msg_name, msg_crc)
- logger.debug("Surviving collections: {col!r}".format(
- col=self._expected.keys()))
+ msg_options = self._get_options(msg_obj, version)
+ self._process_crc(msg_name, msg_crc, msg_options)
+ logger.debug(f"Surviving collections: {self._expected.keys()!r}")
def report_initial_conflicts(self, report_missing=False):
"""Report issues discovered by _check_dir, if not done that already.
self._initial_conflicts_reported = True
if self._reported:
reported_indented = json.dumps(
- self._reported, indent=1, sort_keys=True, separators=[",", ":"])
+ self._reported, indent=1, sort_keys=True,
+ separators=[u",", u":"]
+ )
self._reported = dict()
self.log_and_raise(
- "Incompatible API CRCs found in .api.json files:\n{ri}".format(
- ri=reported_indented))
+ f"Incompatible API CRCs found in .api.json files:\n"
+ f"{reported_indented}"
+ )
if not report_missing:
return
missing = {name: mapp for name, mapp in self._missing.items() if mapp}
if missing:
missing_indented = json.dumps(
- missing, indent=1, sort_keys=True, separators=[",", ":"])
- self.log_and_raise("API CRCs missing from .api.json:\n{mi}".format(
- mi=missing_indented))
+ missing, indent=1, sort_keys=True, separators=[u",", u":"])
+ self.log_and_raise(
+ f"API CRCs missing from .api.json:\n{missing_indented}"
+ )
def check_api_name(self, api_name):
"""Fail if the api_name has no, or different from known CRC associated.
+ Print warning if options contain anything more than vat_help.
+
Do not fail if this particular failure has been already reported.
Intended use: Call during test (not in initialization),
- everytime an API call is queued or response received.
+ every time an API call is queued or response received.
- :param api_name: VPP API messagee name to check.
+ :param api_name: VPP API message name to check.
:type api_name: str or unicode
:raises RuntimeError: If no verified CRC for the api_name is found.
"""
if name_to_crc_mapping[api_name] == crc:
matching = True
break
- if matching:
- return
- self._reported[api_name] = crc
- self.log_and_raise("No active collection contains API {api!r}"
- " with CRC {crc!r}".format(api=api_name, crc=crc))
+ if not matching:
+ self._reported[api_name] = crc
+ self.log_and_raise(
+ f"No active collection has API {api_name!r} with CRC {crc!r}"
+ )
+ options = self._options[api_name]
+ options.pop(u"vat_help", None)
+ if options:
+ self._reported[api_name] = crc
+ logger.console(f"{api_name} used but has options {options}")
+
+ def print_warnings(self):
+ """Call check_api_name for every API name in surviving collections.
+
+ Useful for VPP CRC checking job.
+
+ Even though there usually is only one surviving collection,
+ the implementation has to be prepared for multiple collections,
+ and it should de-duplicate api names.
+ """
+ api_name_to_crc_maps = self._expected.values()
+ api_name_sets = [set(n2c.keys()) for n2c in api_name_to_crc_maps]
+ api_names = set().union(*api_name_sets)
+ for api_name in sorted(api_names):
+ self.check_api_name(api_name)