1 # Copyright (c) 2023 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Module for keeping track of VPP API CRCs relied on by CSIT."""
20 from robot.api import logger
22 from resources.libraries.python.Constants import Constants
26 """Convert from possible bytes without interpreting as number.
28 :param text: Input to convert.
29 :type text: str or unicode
30 :returns: Converted text.
33 return text.decode(u"utf-8") if isinstance(text, bytes) else text
36 class VppApiCrcChecker:
37 """Holder of data related to tracking VPP API CRCs.
39 Both message names and crc hexa strings are tracked as
40 ordinary Python3 (unicode) string, so _str() is used when input is
41 possibly bytes or otherwise not safe.
43 Each instance of this class starts with same default state,
44 so make sure the calling libraries have appropriate robot library scope.
45 For usual testing, it means "GLOBAL" scope."""
48 self, directory, fail_on_mismatch=Constants.FAIL_ON_CRC_MISMATCH):
49 """Initialize empty state, then register known collections.
51 This also scans directory for .api.json files
52 and performs initial checks, but does not report the findings yet.
54 :param directory: Root directory of the search for .api.json files.
58 self.fail_on_mismatch = fail_on_mismatch
59 """If True, mismatch leads to test failure, by raising exception.
60 If False, the mismatch is logged, but the test is allowed to continue.
63 self._expected = dict()
64 """Mapping from collection name to mapping from API name to CRC string.
66 Collection name should be something useful for logging.
68 Order of addition reflects the order collections should be queried.
69 If an incompatible CRC is found, affected collections are removed.
70 A CRC that would remove all does not, added to _reported instead,
71 while causing a failure in single test (if fail_on_mismatch)."""
73 self._missing = dict()
74 """Mapping from collection name to mapping from API name to CRC string.
76 Starts the same as _expected, but each time an encountered api,crc pair
77 fits the expectation, the pair is removed from all collections
78 within this mapping. It is fine if an api is missing
79 from some collections, as long as it is not missing from all collections
80 that remained in _expected."""
83 """Mapping from API name to CRC string.
85 This gets populated with CRCs found in .api.json,
86 to serve as a hint when reporting errors."""
88 self._options = dict()
89 """Mapping from API name to options dictionary.
91 This gets populated with options found in .api.json,
92 to serve as a hint when reporting errors."""
94 self._reported = dict()
95 """Mapping from API name to CRC string.
97 This gets populated with APIs used, but not found in collections,
98 just before the fact is reported in an exception.
99 The CRC comes from _found mapping (otherwise left as None).
100 The idea is to not report those next time, allowing the job
101 to find more problems in a single run."""
103 self._initial_conflicts_reported = False
105 self._check_dir(directory)
107 def log_and_raise(self, exc_msg):
108 """Log to console, on fail_on_mismatch also raise runtime exception.
110 :param exc_msg: The message to include in log or exception.
112 :raises RuntimeError: With the message, if fail_on_mismatch.
114 logger.console("RuntimeError:\n{m}".format(m=exc_msg))
115 if self.fail_on_mismatch:
116 raise RuntimeError(exc_msg)
118 def _register_collection(self, collection_name, name_to_crc_mapping):
119 """Add a named (copy of) collection of CRCs.
121 :param collection_name: Helpful string describing the collection.
122 :param name_to_crc_mapping: Mapping from API names to CRCs.
123 :type collection_name: str or unicode
124 :type name_to_crc_mapping: dict from str/unicode to str/unicode
125 :raises RuntimeError: If the name of a collection is registered already.
127 collection_name = _str(collection_name)
128 if collection_name in self._expected:
130 f"Collection {collection_name!r} already registered."
132 mapping = {_str(k): _str(v) for k, v in name_to_crc_mapping.items()}
133 self._expected[collection_name] = mapping
134 self._missing[collection_name] = mapping.copy()
136 def _register_all(self):
137 """Add all collections this CSIT codebase is tested against."""
139 file_path = os.path.normpath(os.path.join(
140 os.path.dirname(os.path.abspath(__file__)), u"..", u"..",
141 u"api", u"vpp", u"supported_crcs.yaml"))
142 with open(file_path, u"rt") as file_in:
143 collections_dict = yaml.safe_load(file_in.read())
144 for collection_name, name_to_crc_mapping in collections_dict.items():
145 self._register_collection(collection_name, name_to_crc_mapping)
148 def _get_name(msg_obj):
149 """Utility function to extract API name from an intermediate json.
151 :param msg_obj: Loaded json object, item of "messages" list.
152 :type msg_obj: list of various types
153 :returns: Name of the message.
154 :rtype: str or unicode
155 :raises RuntimeError: If no name is found.
158 if isinstance(item, (dict, list)):
161 raise RuntimeError(f"No name found for message: {msg_obj!r}")
164 def _get_crc(msg_obj):
165 """Utility function to extract API CRC from an intermediate json.
167 :param msg_obj: Loaded json object, item of "messages" list.
168 :type msg_obj: list of various types
169 :returns: CRC of the message.
170 :rtype: str or unicode
171 :raises RuntimeError: If no CRC is found.
173 for item in reversed(msg_obj):
174 if not isinstance(item, dict):
176 crc = item.get(u"crc", None)
179 raise RuntimeError(f"No CRC found for message: {msg_obj!r}")
182 def _get_options(msg_obj, version):
183 """Utility function to extract API options from an intermediate json.
185 Empty dict is returned if options are not found,
186 so old VPP builds can be tested without spamming.
187 If version starts with "0.", add a fake option,
188 as the message is treated as "in-progress" by the API upgrade process.
190 :param msg_obj: Loaded json object, item of "messages" list.
191 :param version: Version string from the .api.json file.
192 :type msg_obj: list of various types
193 :type version: Optional[str]
194 :returns: Object found as value for "options" key.
198 for item in reversed(msg_obj):
199 if not isinstance(item, dict):
201 options = item.get(u"options", dict())
204 if version is None or version.startswith(u"0."):
205 options[u"version"] = version
208 def _process_crc(self, api_name, crc, options):
209 """Compare API to verified collections, update class state.
211 Here, API stands for (message name, CRC) pair.
213 Conflict is NOT when a collection does not recognize the API.
214 Such APIs are merely added to _found for later reporting.
215 Conflict is when a collection recognizes the API under a different CRC.
216 If a partial match happens, only the matching collections are preserved.
217 On no match, all current collections are preserved,
218 but the offending API is added to _reported mapping.
220 Note that it is expected that collections are incompatible
221 with each other for some APIs. The removal of collections
222 on partial match is there to help identify the intended collection
223 for the VPP build under test. But if no collection fits perfectly,
224 the last collections to determine the "known" flag
225 depends on the order of api_name submitted,
226 which tends to be fairly random (depends on order of .api.json files).
227 Order of collection registrations does not help much in this regard.
229 Attempts to overwrite value in _found or _reported should not happen,
230 so the code does not check for that, simply overwriting.
232 Options are stored, to be examined later.
234 The intended usage is to call this method multiple times,
235 and then raise exception listing all _reported.
237 :param api_name: API name to check.
238 :param crc: Discovered CRC to check for the name.
239 :param options: Empty dict or options value for in .api.json
244 # Regardless of the result, remember as found.
245 self._found[api_name] = crc
246 self._options[api_name] = options
247 old_expected = self._expected
248 new_expected = old_expected.copy()
249 for collection_name, name_to_crc_mapping in old_expected.items():
250 if api_name not in name_to_crc_mapping:
252 if name_to_crc_mapping[api_name] == crc:
253 self._missing[collection_name].pop(api_name, None)
255 # Remove the offending collection.
256 new_expected.pop(collection_name, None)
258 # Some collections recognized the CRC.
259 self._expected = new_expected
260 self._missing = {name: self._missing[name] for name in new_expected}
262 # No new_expected means some collections knew the api_name,
263 # but CRC does not match any. This has to be reported.
264 self._reported[api_name] = crc
266 def _check_dir(self, directory):
267 """Parse every .api.json found under directory, remember conflicts.
269 As several collections are supported, each conflict invalidates
270 some of them, failure happens only when no collections would be left.
271 In that case, set of collections just before the failure is preserved,
272 the _reported mapping is filled with conflicting APIs.
273 The _found mapping is filled with discovered api names and crcs.
275 The exception is not thrown here, but from report_initial_conflicts.
277 :param directory: Root directory of the search for .api.json files.
280 for root, _, files in os.walk(directory):
281 for filename in files:
282 if not filename.endswith(u".api.json"):
284 with open(f"{root}/{filename}", u"rt") as file_in:
285 json_obj = json.load(file_in)
286 version = json_obj[u"options"].get(u"version", None)
287 msgs = json_obj[u"messages"]
289 msg_name = self._get_name(msg_obj)
290 msg_crc = self._get_crc(msg_obj)
291 msg_options = self._get_options(msg_obj, version)
292 self._process_crc(msg_name, msg_crc, msg_options)
293 logger.debug(f"Surviving collections: {self._expected.keys()!r}")
295 def report_initial_conflicts(self, report_missing=False):
296 """Report issues discovered by _check_dir, if not done that already.
298 Intended use: Call once after init, at a time when throwing exception
301 Optionally, report also missing messages.
302 Missing reporting is disabled by default, because some messages
303 come from plugins that might not be enabled at runtime.
305 After the report, clear _reported, so that test cases report them again,
306 thus tracking which message is actually used (by which test).
308 :param report_missing: Whether to raise on missing messages.
309 :type report_missing: bool
310 :raises RuntimeError: If CRC mismatch or missing messages are detected,
311 and fail_on_mismatch is True.
313 if self._initial_conflicts_reported:
315 self._initial_conflicts_reported = True
317 reported_indented = json.dumps(
318 self._reported, indent=1, sort_keys=True,
319 separators=[u",", u":"]
321 self._reported = dict()
323 f"Incompatible API CRCs found in .api.json files:\n"
324 f"{reported_indented}"
326 if not report_missing:
328 missing = {name: mapp for name, mapp in self._missing.items() if mapp}
329 if set(missing.keys()) < set(self._expected.keys()):
330 # There is a collection where nothing is missing.
332 missing_indented = json.dumps(
333 missing, indent=1, sort_keys=True, separators=[u",", u":"]
336 f"API CRCs missing from .api.json:\n{missing_indented}"
339 def check_api_name(self, api_name):
340 """Fail if the api_name has no, or different from known CRC associated.
342 Print warning if options contain anything more than vat_help.
344 Do not fail if this particular failure has been already reported.
346 Intended use: Call during test (not in initialization),
347 every time an API call is queued or response received.
349 :param api_name: VPP API message name to check.
350 :type api_name: str or unicode
351 :raises RuntimeError: If no verified CRC for the api_name is found.
353 api_name = _str(api_name)
354 if api_name in self._reported:
356 old_expected = self._expected
357 new_expected = old_expected.copy()
358 for collection_name, name_to_crc_mapping in old_expected.items():
359 if api_name in name_to_crc_mapping:
361 # Remove the offending collection.
362 new_expected.pop(collection_name, None)
364 # Some collections recognized the message name.
365 self._expected = new_expected
366 crc = self._found.get(api_name, None)
369 # Regardless of how many collections are remaining,
370 # verify the known CRC is on one of them.
371 for name_to_crc_mapping in self._expected.values():
372 if api_name not in name_to_crc_mapping:
374 if name_to_crc_mapping[api_name] == crc:
378 self._reported[api_name] = crc
380 f"No active collection has API {api_name!r} with CRC {crc!r}"
382 options = self._options.get(api_name, None)
384 # None means CSIT is attempting a new API on an old VPP build.
385 # If that is an issue, the API has been reported as missing already.
387 options.pop(u"vat_help", None)
389 self._reported[api_name] = crc
390 logger.console(f"{api_name} used but has options {options}")
392 def print_warnings(self):
393 """Call check_api_name for API names in surviving collections.
395 Useful for VPP CRC checking job.
396 The API name is only checked when it appears
397 in all surviving collections.
399 api_name_to_crc_maps = self._expected.values()
400 api_name_sets = (set(n2c.keys()) for n2c in api_name_to_crc_maps)
401 api_names = set.intersection(*api_name_sets)
402 for api_name in sorted(api_names):
403 self.check_api_name(api_name)