X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2FVppApiCrc.py;h=693dac064a93f182839b74424fc897a1cd22f33c;hp=cb6f5b0c60bb2be7101104685a57aad79b2c6224;hb=HEAD;hpb=e310a40eab90bb5ecd8471dbbccc1d02daf2dea3 diff --git a/resources/libraries/python/VppApiCrc.py b/resources/libraries/python/VppApiCrc.py index cb6f5b0c60..a8947a18cb 100644 --- a/resources/libraries/python/VppApiCrc.py +++ b/resources/libraries/python/VppApiCrc.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2023 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -21,23 +21,24 @@ from robot.api import logger from resources.libraries.python.Constants import Constants + def _str(text): - """Convert from possible unicode without interpreting as number. + """Convert from possible bytes without interpreting as number. :param text: Input to convert. :type text: str or unicode :returns: Converted text. :rtype: str """ - return text.encode("utf-8") if isinstance(text, unicode) else text + return text.decode(u"utf-8") if isinstance(text, bytes) else text -class VppApiCrcChecker(object): +class VppApiCrcChecker: """Holder of data related to tracking VPP API CRCs. Both message names and crc hexa strings are tracked as - ordinary Python2 (bytes) str, so _str() is used when input is - possibly unicode or otherwise not safe. + ordinary Python3 (unicode) string, so _str() is used when input is + possibly bytes or otherwise not safe. Each instance of this class starts with same default state, so make sure the calling libraries have appropriate robot library scope. @@ -62,20 +63,21 @@ class VppApiCrcChecker(object): self._expected = dict() """Mapping from collection name to mapping from API name to CRC string. - Colection name should be something useful for logging. + Collection name should be something useful for logging. - Order of addition reflects the order colections should be queried. + Order of addition reflects the order collections should be queried. If an incompatible CRC is found, affected collections are removed. A CRC that would remove all does not, added to _reported instead, - while causing a failure in single test.""" + while causing a failure in single test (if fail_on_mismatch).""" self._missing = dict() """Mapping from collection name to mapping from API name to CRC string. Starts the same as _expected, but each time an encountered api,crc pair - fits the expectation, the pair is removed from this mapping. - Ideally, the active mappings will become empty. - If not, it is an error, VPP removed or renamed a message CSIT needs.""" + fits the expectation, the pair is removed from all collections + within this mapping. It is fine if an api is missing + from some collections, as long as it is not missing from all collections + that remained in _expected.""" self._found = dict() """Mapping from API name to CRC string. @@ -83,6 +85,12 @@ class VppApiCrcChecker(object): This gets populated with CRCs found in .api.json, to serve as a hint when reporting errors.""" + self._options = dict() + """Mapping from API name to options dictionary. + + This gets populated with options found in .api.json, + to serve as a hint when reporting errors.""" + self._reported = dict() """Mapping from API name to CRC string. @@ -96,15 +104,16 @@ class VppApiCrcChecker(object): self._register_all() self._check_dir(directory) - def raise_or_log(self, exception): - """If fail_on_mismatch, raise, else log to console the exception. + def log_and_raise(self, exc_msg): + """Log to console, on fail_on_mismatch also raise runtime exception. - :param exception: The exception to raise or log. - :type exception: RuntimeError + :param exc_msg: The message to include in log or exception. + :type exc_msg: str + :raises RuntimeError: With the message, if fail_on_mismatch. """ + logger.console("RuntimeError:\n{m}".format(m=exc_msg)) if self.fail_on_mismatch: - raise exception - logger.console("{exc!r}".format(exc=exception)) + raise RuntimeError(exc_msg) def _register_collection(self, collection_name, name_to_crc_mapping): """Add a named (copy of) collection of CRCs. @@ -113,11 +122,13 @@ class VppApiCrcChecker(object): :param name_to_crc_mapping: Mapping from API names to CRCs. :type collection_name: str or unicode :type name_to_crc_mapping: dict from str/unicode to str/unicode + :raises RuntimeError: If the name of a collection is registered already. """ collection_name = _str(collection_name) if collection_name in self._expected: - raise RuntimeError("Collection {cl!r} already registered.".format( - cl=collection_name)) + raise RuntimeError( + f"Collection {collection_name!r} already registered." + ) mapping = {_str(k): _str(v) for k, v in name_to_crc_mapping.items()} self._expected[collection_name] = mapping self._missing[collection_name] = mapping.copy() @@ -126,10 +137,10 @@ class VppApiCrcChecker(object): """Add all collections this CSIT codebase is tested against.""" file_path = os.path.normpath(os.path.join( - os.path.dirname(os.path.abspath(__file__)), "..", "..", - "api", "vpp", "supported_crcs.yaml")) - with open(file_path, "r") as file_in: - collections_dict = yaml.load(file_in.read()) + os.path.dirname(os.path.abspath(__file__)), u"..", u"..", + u"api", u"vpp", u"supported_crcs.yaml")) + with open(file_path, u"rt") as file_in: + collections_dict = yaml.safe_load(file_in.read()) for collection_name, name_to_crc_mapping in collections_dict.items(): self._register_collection(collection_name, name_to_crc_mapping) @@ -147,8 +158,7 @@ class VppApiCrcChecker(object): if isinstance(item, (dict, list)): continue return _str(item) - raise RuntimeError("No name found for message: {obj!r}".format( - obj=msg_obj)) + raise RuntimeError(f"No name found for message: {msg_obj!r}") @staticmethod def _get_crc(msg_obj): @@ -163,15 +173,43 @@ class VppApiCrcChecker(object): for item in reversed(msg_obj): if not isinstance(item, dict): continue - crc = item.get("crc", None) + crc = item.get(u"crc", None) if crc: return _str(crc) - raise RuntimeError("No CRC found for message: {obj!r}".format( - obj=msg_obj)) + raise RuntimeError(f"No CRC found for message: {msg_obj!r}") + + @staticmethod + def _get_options(msg_obj, version): + """Utility function to extract API options from an intermediate json. + + Empty dict is returned if options are not found, + so old VPP builds can be tested without spamming. + If version starts with "0.", add a fake option, + as the message is treated as "in-progress" by the API upgrade process. - def _process_crc(self, api_name, crc): + :param msg_obj: Loaded json object, item of "messages" list. + :param version: Version string from the .api.json file. + :type msg_obj: list of various types + :type version: Optional[str] + :returns: Object found as value for "options" key. + :rtype: dict + """ + options = dict() + for item in reversed(msg_obj): + if not isinstance(item, dict): + continue + options = item.get(u"options", dict()) + if not options: + break + if version is None or version.startswith(u"0."): + options[u"version"] = version + return options + + def _process_crc(self, api_name, crc, options): """Compare API to verified collections, update class state. + Here, API stands for (message name, CRC) pair. + Conflict is NOT when a collection does not recognize the API. Such APIs are merely added to _found for later reporting. Conflict is when a collection recognizes the API under a different CRC. @@ -191,16 +229,21 @@ class VppApiCrcChecker(object): Attempts to overwrite value in _found or _reported should not happen, so the code does not check for that, simply overwriting. + Options are stored, to be examined later. + The intended usage is to call this method multiple times, and then raise exception listing all _reported. :param api_name: API name to check. :param crc: Discovered CRC to check for the name. + :param options: Empty dict or options value for in .api.json :type api_name: str :type crc: str + :type options: dict """ # Regardless of the result, remember as found. self._found[api_name] = crc + self._options[api_name] = options old_expected = self._expected new_expected = old_expected.copy() for collection_name, name_to_crc_mapping in old_expected.items(): @@ -216,7 +259,7 @@ class VppApiCrcChecker(object): self._expected = new_expected self._missing = {name: self._missing[name] for name in new_expected} return - # No new_expected means some colections knew the api_name, + # No new_expected means some collections knew the api_name, # but CRC does not match any. This has to be reported. self._reported[api_name] = crc @@ -224,7 +267,7 @@ class VppApiCrcChecker(object): """Parse every .api.json found under directory, remember conflicts. As several collections are supported, each conflict invalidates - one of them, failure happens only when no collections would be left. + some of them, failure happens only when no collections would be left. In that case, set of collections just before the failure is preserved, the _reported mapping is filled with conflicting APIs. The _found mapping is filled with discovered api names and crcs. @@ -236,17 +279,18 @@ class VppApiCrcChecker(object): """ for root, _, files in os.walk(directory): for filename in files: - if not filename.endswith(".api.json"): + if not filename.endswith(u".api.json"): continue - with open(root + '/' + filename, "r") as file_in: + with open(f"{root}/{filename}", u"rt") as file_in: json_obj = json.load(file_in) - msgs = json_obj["messages"] + version = json_obj[u"options"].get(u"version", None) + msgs = json_obj[u"messages"] for msg_obj in msgs: msg_name = self._get_name(msg_obj) msg_crc = self._get_crc(msg_obj) - self._process_crc(msg_name, msg_crc) - logger.debug("Surviving collections: {col!r}".format( - col=self._expected.keys())) + msg_options = self._get_options(msg_obj, version) + self._process_crc(msg_name, msg_crc, msg_options) + logger.debug(f"Surviving collections: {self._expected.keys()!r}") def report_initial_conflicts(self, report_missing=False): """Report issues discovered by _check_dir, if not done that already. @@ -258,6 +302,9 @@ class VppApiCrcChecker(object): Missing reporting is disabled by default, because some messages come from plugins that might not be enabled at runtime. + After the report, clear _reported, so that test cases report them again, + thus tracking which message is actually used (by which test). + :param report_missing: Whether to raise on missing messages. :type report_missing: bool :raises RuntimeError: If CRC mismatch or missing messages are detected, @@ -267,25 +314,39 @@ class VppApiCrcChecker(object): return self._initial_conflicts_reported = True if self._reported: - self.raise_or_log( - RuntimeError("Dir check found incompatible API CRCs: {rep!r}"\ - .format(rep=self._reported))) + reported_indented = json.dumps( + self._reported, indent=1, sort_keys=True, + separators=[u",", u":"] + ) + self._reported = dict() + self.log_and_raise( + f"Incompatible API CRCs found in .api.json files:\n" + f"{reported_indented}" + ) if not report_missing: return missing = {name: mapp for name, mapp in self._missing.items() if mapp} - if missing: - self.raise_or_log( - RuntimeError("Dir check found missing API CRCs: {mis!r}"\ - .format(mis=missing))) + if set(missing.keys()) < set(self._expected.keys()): + # There is a collection where nothing is missing. + return + missing_indented = json.dumps( + missing, indent=1, sort_keys=True, separators=[u",", u":"] + ) + self.log_and_raise( + f"API CRCs missing from .api.json:\n{missing_indented}" + ) def check_api_name(self, api_name): - """Fail if the api_name has no known CRC associated. + """Fail if the api_name has no, or different from known CRC associated. + + Print warning if options contain anything more than vat_help. Do not fail if this particular failure has been already reported. - Intended use: Call everytime an API call is queued or response received. + Intended use: Call during test (not in initialization), + every time an API call is queued or response received. - :param api_name: VPP API messagee name to check. + :param api_name: VPP API message name to check. :type api_name: str or unicode :raises RuntimeError: If no verified CRC for the api_name is found. """ @@ -302,9 +363,41 @@ class VppApiCrcChecker(object): if new_expected: # Some collections recognized the message name. self._expected = new_expected - return crc = self._found.get(api_name, None) - self._reported[api_name] = crc - self.raise_or_log( - RuntimeError("No active collection has API {api!r}" - " CRC found {crc!r}".format(api=api_name, crc=crc))) + matching = False + if crc is not None: + # Regardless of how many collections are remaining, + # verify the known CRC is on one of them. + for name_to_crc_mapping in self._expected.values(): + if api_name not in name_to_crc_mapping: + continue + if name_to_crc_mapping[api_name] == crc: + matching = True + break + if not matching: + self._reported[api_name] = crc + self.log_and_raise( + f"No active collection has API {api_name!r} with CRC {crc!r}" + ) + options = self._options.get(api_name, None) + if not options: + # None means CSIT is attempting a new API on an old VPP build. + # If that is an issue, the API has been reported as missing already. + return + options.pop(u"vat_help", None) + if options: + self._reported[api_name] = crc + logger.console(f"{api_name} used but has options {options}") + + def print_warnings(self): + """Call check_api_name for API names in surviving collections. + + Useful for VPP CRC checking job. + The API name is only checked when it appears + in all surviving collections. + """ + api_name_to_crc_maps = self._expected.values() + api_name_sets = (set(n2c.keys()) for n2c in api_name_to_crc_maps) + api_names = set.intersection(*api_name_sets) + for api_name in sorted(api_names): + self.check_api_name(api_name)