1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Module for keeping track of VPP API CRCs relied on by CSIT."""
19 from robot.api import logger
22 class VppApiCrcChecker(object):
23 """Holder of data related to tracking VPP API CRCs.
25 Each instance of this class starts with same default state,
26 so make sure the calling libraries have appropriate robot library scope.
27 For usual testing, it means "GLOBAL" scope."""
29 def __init__(self, directory):
30 """Initialize empty state, then register known collections.
32 This also scans directory for .api.json files
33 and performs initial checks, but does not report the findings yet.
35 :param directory: Root directory of the search for .api.json files.
39 self._expected = dict()
40 """Mapping from collection name to mapping from API name to CRC string.
42 Colection name should be something useful for logging.
43 API name is ordinary Python2 str, CRC is also str.
45 Order of addition reflects the order colections should be queried.
46 If an incompatible CRC is found, affected collections are removed.
47 A CRC that would remove all does not, added to _reported instead,
48 while causing a failure in single test.
51 self._missing = dict()
52 """Mapping from collection name to mapping from API name to CRC string.
54 Starts the same as _expected, but each time an encountered api,crc pair
55 fits the expectation, the pair is removed from this mapping.
56 Ideally, the active mappings will become empty.
57 If not, it is an error, VPP removed or renamed a message CSIT needs."""
60 """Mapping from API name to CRC string.
62 This gets populated with CRCs found in .api.json,
63 to serve as a hint when reporting errors."""
65 self._reported = dict()
66 """Mapping from API name to CRC string.
68 This gets populated with APIs used, but not found in collections,
69 just before the fact is reported in an exception.
70 The CRC comes from _found mapping (otherwise left as None).
71 The idea is to not report those next time, allowing the job
72 to find more problems in a single run."""
74 self._initial_conflicts_reported = False
76 self._check_dir(directory)
78 def _register_collection(self, collection_name, collection_dict):
79 """Add a named (copy of) collection of CRCs.
81 :param collection_name: Helpful string describing the collection.
82 :param collection_dict: Mapping from API names to CRCs.
83 :type collection_name: str
84 :type collection_dict: dict from str to str
86 if collection_name in self._expected:
87 raise RuntimeError("Collection {cl} already registered.".format(
89 self._expected[collection_name] = collection_dict.copy()
90 self._missing[collection_name] = collection_dict.copy()
93 def _get_name(msg_obj):
94 """Utility function to extract API name from an intermediate json.
96 :param msg_obj: Loaded json object, item of "messages" list.
97 :type msg_obj: list of various types
98 :returns: Name of the message.
99 :rtype: str or unicode
100 :raises RuntimeError: If no name is found.
103 if isinstance(item, (dict, list)):
106 raise RuntimeError("No name found for message: {obj!r}".format(
110 def _get_crc(msg_obj):
111 """Utility function to extract API CRC from an intermediate json.
113 :param msg_obj: Loaded json object, item of "messages" list.
114 :type msg_obj: list of various types
115 :returns: CRC of the message.
116 :rtype: str or unicode
117 :raises RuntimeError: If no CRC is found.
119 for item in reversed(msg_obj):
120 if not isinstance(item, dict):
122 crc = item.get("crc", None)
125 raise RuntimeError("No CRC found for message: {obj!r}".format(
128 def _process_crc(self, api_name, crc):
129 """Compare API to verified collections, update class state.
131 Conflict is NOT when a collection does not recognize the API.
132 Such APIs are merely added to _found for later reporting.
133 Conflict is when a collection recognizes the API under a different CRC.
134 If a partial match happens, only the matching collections are preserved.
135 On no match, all current collections are preserved,
136 but the offending API is added to _reported mapping.
138 Note that it is expected that collections are incompatible
139 with each other for some APIs. The removal of collections
140 on partial match is there to help identify the intended collection
141 for the VPP build under test. But if no collection fits perfectly,
142 the last collections to determine the "known" flag
143 depends on the order of api_name submitted,
144 which tends to be fairly random (depends on order of .api.json files).
145 Order of collection registrations does not help much in this regard.
147 Attempts to overwrite value in _found or _reported should not happen,
148 so the code does not check for that, simply overwriting.
150 The intended usage is to call this method multiple times,
151 and then raise exception listing all _reported.
153 :param api_name: API name to check.
154 :param crc: Discovered CRC to check for the name.
156 :type crc: str or unicode
158 # Regardless of the result, remember as found.
159 self._found[api_name] = crc
160 old_expected = self._expected
161 new_expected = old_expected.copy()
162 for collection_name, collection_dict in old_expected.items():
163 if api_name not in collection_dict:
165 if collection_dict[api_name] == crc:
166 self._missing[collection_name].pop(api_name, None)
168 # Remove the offending collection.
169 new_expected.pop(collection_name, None)
171 # Some collections recognized the CRC.
172 self._expected = new_expected
173 self._missing = {name: self._missing[name] for name in new_expected}
175 # No new_expected means some colections knew the api_name,
176 # but CRC does not match any. This has to be reported.
177 self._reported[api_name] = crc
179 def _check_dir(self, directory):
180 """Parse every .api.json found under directory, remember conflicts.
182 As several collections are supported, each conflict invalidates
183 one of them, failure happens only when no collections would be left.
184 In that case, set of collections just before the failure is preserved,
185 the _reported mapping is filled with conflicting APIs.
186 The _found mapping is filled with discovered api names and crcs.
188 The exception is not thrown here, but from report_initial_conflicts.
190 :param directory: Root directory of the search for .api.json files.
193 for root, _, files in os.walk(directory):
194 for filename in files:
195 if not filename.endswith(".api.json"):
197 with open(root + '/' + filename, "r") as file_in:
198 json_obj = json.load(file_in)
199 msgs = json_obj["messages"]
201 msg_name = self._get_name(msg_obj)
202 msg_crc = self._get_crc(msg_obj)
203 self._process_crc(msg_name, msg_crc)
204 logger.info("Surviving collections: {col}".format(
205 col=self._expected.keys()))
207 def report_initial_conflicts(self, report_missing=False):
208 """Report issues discovered by _check_dir, if not done that already.
210 Intended use: Call once after init, at a time when throwing exception
213 Optionally, report also missing messages.
214 Missing reporting is disabled by default, because some messages
215 come from plugins that might not be enabled at runtime.
217 :param report_missing: Whether to raise on missing messages.
218 :type report_missing: bool
219 :raises RuntimeError: If CRC mismatch or missing messages are detected.
221 if self._initial_conflicts_reported:
223 self._initial_conflicts_reported = True
225 raise RuntimeError("Dir check found incompatible API CRCs: {rep!r}"\
226 .format(rep=self._reported))
227 if not report_missing:
229 missing = {name: mapp for name, mapp in self._missing.items() if mapp}
231 raise RuntimeError("Dir check found missing API CRCs: {mis!r}"\
232 .format(mis=missing))
234 def check_api_name(self, api_name):
235 """Fail if the api_name has no known CRC associated.
237 Do not fail if this particular failure has been already reported.
239 Intended use: Call everytime an API call is queued or response received.
241 :param api_name: VPP API messagee name to check.
243 :raises RuntimeError: If no verified CRC for the api_name is found.
245 if api_name in self._reported:
247 old_expected = self._expected
248 new_expected = old_expected.copy()
249 for collection_name, collection_dict in old_expected.items():
250 if api_name in collection_dict:
252 # Remove the offending collection.
253 new_expected.pop(collection_name, None)
255 # Some collections recognized the message name.
256 self._expected = new_expected
258 crc = self._found.get(api_name, None)
259 self._reported[api_name] = crc
260 # Disabled temporarily during CRC mismatch.
261 #raise RuntimeError("No active collection has API {api} CRC found {crc}"\
262 # .format(api=api_name, crc=crc))
264 # Moved to the end as this part will be edited frequently.
265 def _register_all(self):
266 """Add all collections this CSIT codebase is tested against."""
268 # Rework to read from files?
269 self._register_collection(
270 "19.08-rc0~762-gbb2e5221a", {
271 "acl_add_replace": "0x13bc8539", # perf
272 "acl_add_replace_reply": "0xac407b0c", # perf
273 "acl_dump": "0xef34fea4", # perf teardown
274 "acl_interface_list_dump": "0x529cb13f", # perf teardown
275 # ^^^^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr
276 "acl_interface_set_acl_list": "0x8baece38", # perf
277 "acl_interface_set_acl_list_reply": "0xe8d4e804", # perf
278 "acl_details": "0xf89d7a88", # perf teardown
279 "acl_interface_list_details": "0xd5e80809", # perf teardown
280 # ^^^^ tc01-64B-1c-ethip4udp-ip4base-iacl1sl-10kflows-mrr
281 # ^^ ip4fwdANDiaclANDacl10AND100_flows
282 "avf_create": "0xdaab8ae2", # perf
283 "avf_create_reply": "0xfda5941f", # perf
284 # ^^ tc01-64B-1c-avf-eth-l2bdbasemaclrn-mrr
285 # ^ l2bdmaclrnANDbaseANDdrv_avf
286 "bridge_domain_add_del": "0xc6360720", # dev
287 "bridge_domain_add_del_reply": "0xe8d4e804", # dev
288 "classify_add_del_session": "0x85fd79f4", # dev
289 "classify_add_del_session_reply": "0xe8d4e804", # dev
290 "classify_add_del_table": "0x9bd794ae", # dev
291 "classify_add_del_table_reply": "0x05486349", # dev
292 "cli_inband": "0xb1ad59b3", # dev setup
293 "cli_inband_reply": "0x6d3c80a4", # dev setup
294 "cop_interface_enable_disable": "0x69d24598", # dev
295 "cop_interface_enable_disable_reply": "0xe8d4e804", # dev
296 "cop_whitelist_enable_disable": "0x8bb8f6dc", # dev
297 "cop_whitelist_enable_disable_reply": "0xe8d4e804", # dev
298 "create_loopback": "0x3b54129c", # dev
299 "create_loopback_reply": "0xfda5941f", # dev
300 "create_subif": "0x86cfe408", # virl
301 "create_subif_reply": "0xfda5941f", # virl
302 "create_vhost_user_if": "0xbd230b87", # dev
303 "create_vhost_user_if_reply": "0xfda5941f", # dev
304 "create_vlan_subif": "0x70cadeda", # virl
305 "create_vlan_subif_reply": "0xfda5941f", # virl
306 "gre_tunnel_add_del": "0x04199f47", # virl
307 "gre_tunnel_add_del_reply": "0x903324db", # virl
308 "gpe_enable_disable": "0xeb0e943b", # virl
309 "gpe_enable_disable_reply": "0xe8d4e804", # virl
310 "hw_interface_set_mtu": "0x132da1e7", # dev
311 "hw_interface_set_mtu_reply": "0xe8d4e804", # dev
312 "input_acl_set_interface": "0xe09537b0", # dev
313 "input_acl_set_interface_reply": "0xe8d4e804", # dev
314 "ip_address_details": "0x2f1dbc7d", # dev
315 "ip_address_dump": "0x6b7bcd0a", # dev
316 "ip_neighbor_add_del": "0x7a68a3c4", # dev
317 "ip_neighbor_add_del_reply": "0x1992deab", # dev
318 "ip_probe_neighbor": "0x2736142d", # virl
319 "ip_route_add_del": "0x83e086ce", # dev
320 "ip_route_add_del_reply": "0x1992deab", # dev
321 "ip_source_check_interface_add_del": "0x0a60152a", # virl
322 "ip_source_check_interface_add_del_reply": "0xe8d4e804", # virl
323 "ip_table_add_del": "0xe5d378f2", # dev
324 "ip_table_add_del_reply": "0xe8d4e804", # dev
325 "ipsec_interface_add_del_spd": "0x1e3b8286", # dev
326 "ipsec_interface_add_del_spd_reply": "0xe8d4e804", # dev
327 "ipsec_sad_entry_add_del": "0xa25ab61e", # dev
328 "ipsec_sad_entry_add_del_reply": "0x9ffac24b", # dev
329 "ipsec_spd_add_del": "0x9ffdf5da", # dev
330 "ipsec_spd_add_del_reply": "0xe8d4e804", # dev
331 "ipsec_spd_entry_add_del": "0x6bc6a3b5", # dev
332 "ipsec_spd_entry_add_del_reply": "0x9ffac24b", # dev
333 "l2_interface_vlan_tag_rewrite": "0xb90be6b4", # virl
334 "l2_interface_vlan_tag_rewrite_reply": "0xe8d4e804", # virl
335 "l2_patch_add_del": "0x62506e63", # perf
336 "l2_patch_add_del_reply": "0xe8d4e804", # perf
337 # ^^ tc01-64B-1c-avf-eth-l2patch-mrr
338 # ^ l2patchANDdrv_avf
339 "lisp_add_del_adjacency": "0xf047390d", # virl
340 "lisp_add_del_adjacency_reply": "0xe8d4e804", # virl
341 "lisp_add_del_local_eid": "0xe6d00717", # virl
342 "lisp_add_del_local_eid_reply": "0xe8d4e804", # virl
343 "lisp_add_del_locator": "0x006a4240", # virl
344 "lisp_add_del_locator_reply": "0xe8d4e804", # virl
345 "lisp_add_del_locator_set": "0x06968e38", # virl
346 "lisp_add_del_locator_set_reply": "0xb6666db4", # virl
347 "lisp_add_del_remote_mapping": "0xb879c3a9", # virl
348 "lisp_add_del_remote_mapping_reply": "0xe8d4e804", # virl
349 "lisp_eid_table_details": "0xdcd9f414", # virl
350 "lisp_eid_table_dump": "0xe0df64da", # virl
351 "lisp_enable_disable": "0xeb0e943b", # virl
352 "lisp_enable_disable_reply": "0xe8d4e804", # virl
353 "lisp_locator_set_details": "0x6b846882", # virl
354 "lisp_locator_set_dump": "0xc79e8ab0", # virl
355 "lisp_map_resolver_details": "0x60a5f5ca", # virl
356 "lisp_map_resolver_dump": "0x51077d14", # virl
357 "memif_create": "0x6597cdb2", # dev
358 "memif_create_reply": "0xfda5941f", # dev
359 "memif_details": "0x4f5a3397", # dev
360 "memif_dump": "0x51077d14", # dev
361 "memif_socket_filename_add_del": "0x30e3929d", # dev
362 "memif_socket_filename_add_del_reply": "0xe8d4e804", # dev
363 "nat_det_add_del_map": "0x04b76549", # perf
364 "nat_det_add_del_map_reply": "0xe8d4e804", # perf
365 "nat44_interface_add_del_feature": "0xef3edad1", # perf
366 "nat44_interface_add_del_feature_reply": "0xe8d4e804", # perf
367 # ^^^^ tc01-64B-1c-ethip4udp-ip4base-nat44-mrr
368 # ^ nat44NOTscaleNOTsrc_user_1
369 "proxy_arp_intfc_enable_disable": "0x69d24598", # virl
370 "proxy_arp_intfc_enable_disable_reply": "0xe8d4e804", # virl
371 "show_lisp_status": "0x51077d14", # virl
372 "show_lisp_status_reply": "0xddcf48ef", # virl
373 "show_threads": "0x51077d14", # dev
374 "show_threads_reply": "0xf5e0b66f", # dev
375 "show_version": "0x51077d14", # dev setup
376 "show_version_reply": "0xb9bcf6df", # dev setup
377 "sw_interface_add_del_address": "0x7b583179", # dev
378 "sw_interface_add_del_address_reply": "0xe8d4e804", # dev
379 "sw_interface_details": "0xe4ee7eb6", # dev setup
380 "sw_interface_dump": "0x052753c5", # dev setup
381 "sw_interface_ip6nd_ra_config": "0xc3f02daa", # dev
382 "sw_interface_ip6nd_ra_config_reply": "0xe8d4e804", # dev
383 "sw_interface_rx_placement_details": "0x0e9e33f4", # perf
384 "sw_interface_rx_placement_dump": "0x529cb13f", # perf
385 # ^^ tc01-64B-1c-dot1q-l2bdbasemaclrn-eth-2memif-1dcr-mrr
386 # ^ dot1qANDl2bdmaclrnANDbaseANDmemif
387 "sw_interface_set_flags": "0x555485f5", # dev
388 "sw_interface_set_flags_reply": "0xe8d4e804", # dev
389 "sw_interface_set_l2_bridge": "0x5579f809", # dev
390 "sw_interface_set_l2_bridge_reply": "0xe8d4e804", # dev
391 "sw_interface_set_l2_xconnect": "0x95de3988", # dev
392 "sw_interface_set_l2_xconnect_reply": "0xe8d4e804", # dev
393 "sw_interface_set_rx_placement": "0x4ef4377d", # perf
394 "sw_interface_set_rx_placement_reply": "0xe8d4e804", # perf
395 # ^^ tc01-64B-1c-eth-l2xcbase-eth-2memif-1dcr-mrr
396 # ^ l2xcfwdANDbaseANDlxcANDmemif
397 "sw_interface_set_table": "0xacb25d89", # dev
398 "sw_interface_set_table_reply": "0xe8d4e804", # dev
399 "sw_interface_set_vxlan_bypass": "0xe74ca095", # dev
400 "sw_interface_set_vxlan_bypass_reply": "0xe8d4e804", # dev
401 "sw_interface_vhost_user_details": "0x91ff3307", # dev
402 "sw_interface_vhost_user_dump": "0x51077d14", # dev
403 "vxlan_add_del_tunnel": "0x00f4bdd0", # virl
404 "vxlan_add_del_tunnel_reply": "0xfda5941f", # virl
405 "vxlan_tunnel_details": "0xce38e127", # virl
406 "vxlan_tunnel_dump": "0x529cb13f", # virl
408 # Perf verify with: csit-3n-skx-perftest
409 # mrrAND1cAND64bANDnic_intel-x710ANDip4fwdANDiaclANDacl10AND100_flows
410 # mrrAND1cAND64bANDnic_intel-x710ANDl2bdmaclrnANDbaseANDdrv_avf
411 # mrrAND1cAND64bANDnic_intel-x710ANDl2patchANDdrv_avf
412 # mrrAND1cAND64bANDnic_intel-x710ANDnat44NOTscaleNOTsrc_user_1
413 # mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDl2bdmaclrnANDbaseANDmemif
414 # mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDbaseANDlxcANDmemif
416 # TODO: Add a tag expression for covering those perf tests,
417 # even though any CSIT change can make that outdated.
418 # TODO: Once API coverage job is ready,
419 # add a check to make sure each message was encountered;
420 # failure means we need to add more tests to API coverage job.
421 # Alternatively, add an option to compile messages actually
422 # used or encountered, so CSIT knows what to remove from mapping.