CRCs: Update comments and improve CSIT checks.

+ Restore alphabetic order in yaml.
+ Mark the lisp messages verified to be covered by dev instead of virl.
+ Mark the GBP messages verified by devicetest now.
+ Mark acl_add_replace as dev and create_subif as perf.
+ Mark ip_probe_neighbor as unused L1.
+ Improve loadbalancer comments.
  - Nat4 testcase is described, even if not merged yet.
+ Forget reported CRC after initial dir report,
  and report them again (once) in test cases (if hit).
  With this, single run (with fail on crc off) gives full info.
+ Small improvements to CRC library docstrings and messages.

Change-Id: Ic1cfad45e10d437be827af2a89eac49e25267b59
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

diff --git a/resources/api/vpp/supported_crcs.yaml b/resources/api/vpp/supported_crcs.yaml
index c397373..0370500 100644 (file)
--- a/resources/api/vpp/supported_crcs.yaml
+++ b/resources/api/vpp/supported_crcs.yaml
@@ -30,17 +30,17 @@
 # Use bash command "env LC_COLLATE=C sort -u" if not clear.
 
 # https://logs.fd.io/production/vex-yul-rot-jenkins-1
 # Use bash command "env LC_COLLATE=C sort -u" if not clear.
 
 # https://logs.fd.io/production/vex-yul-rot-jenkins-1

-# /vpp-beta-merge-master-ubuntu1804/3788/archives/build-root/
-20.01-rc0~375:
-    acl_add_replace: '0x13bc8539'  # perf
-    acl_add_replace_reply: '0xac407b0c'  # perf
+# /vpp-beta-merge-master-ubuntu1804/3866/archives/build-root/
+20.01-rc0~453:
+    acl_add_replace: '0x13bc8539'  # dev
+    acl_add_replace_reply: '0xac407b0c'  # dev

     acl_details: '0xf89d7a88'  # perf teardown
     acl_dump: '0xef34fea4'  # perf teardown
     acl_interface_list_details: '0xd5e80809'  # perf teardown
     acl_interface_list_dump: '0x529cb13f'  # perf teardown
     acl_interface_set_acl_list: '0x8baece38'  # perf
     acl_interface_set_acl_list_reply: '0xe8d4e804'  # perf
     acl_details: '0xf89d7a88'  # perf teardown
     acl_dump: '0xef34fea4'  # perf teardown
     acl_interface_list_details: '0xd5e80809'  # perf teardown
     acl_interface_list_dump: '0x529cb13f'  # perf teardown
     acl_interface_set_acl_list: '0x8baece38'  # perf
     acl_interface_set_acl_list_reply: '0xe8d4e804'  # perf

-    # 8x^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr
+    # 6x^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr

     #     tc01-64B-1c-ethip4udp-ip4base-iacl1sl-10kflows-mrr
     # ^^ ip4fwdANDiaclANDacl10AND100_flows
     avf_create: '0xdaab8ae2'  # dev
     #     tc01-64B-1c-ethip4udp-ip4base-iacl1sl-10kflows-mrr
     # ^^ ip4fwdANDiaclANDacl10AND100_flows
     avf_create: '0xdaab8ae2'  # dev


@@ -50,7 +50,7 @@
     bond_enslave: '0x076ecfa7'  # perf
     bond_enslave_reply: '0xe8d4e804'  # perf
     # 4x^ tc01-64B-1c-1lbvpplacp-dot1q-l2xcbase-eth-2vhostvr1024-1vm-mrr
     bond_enslave: '0x076ecfa7'  # perf
     bond_enslave_reply: '0xe8d4e804'  # perf
     # 4x^ tc01-64B-1c-1lbvpplacp-dot1q-l2xcbase-eth-2vhostvr1024-1vm-mrr

-    # ^ l2xcfwdANDlbond_1l
+    # ^ l2xcfwdANDlbond_1lANDnf_testpmd

     bridge_domain_add_del: '0xc6360720'  # dev
     bridge_domain_add_del_reply: '0xe8d4e804'  # dev
     # bridge_domain_dump / details # honeycomb
     bridge_domain_add_del: '0xc6360720'  # dev
     bridge_domain_add_del_reply: '0xe8d4e804'  # dev
     # bridge_domain_dump / details # honeycomb


@@ -69,31 +69,31 @@
     cop_whitelist_enable_disable_reply: '0xe8d4e804'  # dev
     create_loopback: '0x42bb5d22'  # dev
     create_loopback_reply: '0x5383d31f'  # dev
     cop_whitelist_enable_disable_reply: '0xe8d4e804'  # dev
     create_loopback: '0x42bb5d22'  # dev
     create_loopback_reply: '0x5383d31f'  # dev

-    create_subif: '0xcb371063'  # virl
-    create_subif_reply: '0x5383d31f'  # virl
+    create_subif: '0xcb371063'  # perf
+    create_subif_reply: '0x5383d31f'  # perf
+    # ^^ tc01-64B-1c-dot1ad-l2xcbase-mrr
+    # ^ 3n: dot1adANDl2xcfwd

     create_vhost_user_if: '0xa3438cd4'  # dev
     create_vhost_user_if_reply: '0xfda5941f'  # dev
     create_vlan_subif: '0xaf34ac8b'  # dev
     create_vlan_subif_reply: '0x5383d31f'  # dev
     # dhcp_proxy_dump / details # honeycomb
     create_vhost_user_if: '0xa3438cd4'  # dev
     create_vhost_user_if_reply: '0xfda5941f'  # dev
     create_vlan_subif: '0xaf34ac8b'  # dev
     create_vlan_subif_reply: '0x5383d31f'  # dev
     # dhcp_proxy_dump / details # honeycomb

-    gbp_bridge_domain_add: '0x70f1069c'  # perf
-    gbp_bridge_domain_add_reply: '0xe8d4e804'  # perf
-    gbp_contract_add_del: '0x5b63d90a'  # perf
-    gbp_contract_add_del_reply: '0x1992deab'  # perf
-    gbp_endpoint_add: '0xf0efa120'  # perf
-    gbp_endpoint_add_reply: '0x1dd3ff3e'  # perf
-    gbp_endpoint_group_add: '0x1031b376'  # perf
-    gbp_endpoint_group_add_reply: '0xe8d4e804'  # perf
-    gbp_ext_itf_add_del: '0x6995e85f'  # perf
-    gbp_ext_itf_add_del_reply: '0xe8d4e804'  # perf
-    gbp_route_domain_add: '0x1560adc7'  # perf
-    gbp_route_domain_add_reply: '0xe8d4e804'  # perf
-    gbp_subnet_add_del: '0xf42b9430'  # perf
-    gbp_subnet_add_del_reply: '0xe8d4e804'  # perf
-    # 14x^ tc01-64B-1c-avf-dot1q-l2bdbasemaclrn-gbp-mrr
-    # ^ dot1qANDdrv_avfANDgbp
-    gpe_enable_disable: '0xeb0e943b'  # virl
-    gpe_enable_disable_reply: '0xe8d4e804'  # virl
+    gbp_bridge_domain_add: '0x70f1069c'  # dev
+    gbp_bridge_domain_add_reply: '0xe8d4e804'  # dev
+    gbp_contract_add_del: '0x5b63d90a'  # dev
+    gbp_contract_add_del_reply: '0x1992deab'  # dev
+    gbp_endpoint_add: '0xf0efa120'  # dev
+    gbp_endpoint_add_reply: '0x1dd3ff3e'  # dev
+    gbp_endpoint_group_add: '0x1031b376'  # dev
+    gbp_endpoint_group_add_reply: '0xe8d4e804'  # dev
+    gbp_ext_itf_add_del: '0x6995e85f'  # dev
+    gbp_ext_itf_add_del_reply: '0xe8d4e804'  # dev
+    gbp_route_domain_add: '0x1560adc7'  # dev
+    gbp_route_domain_add_reply: '0xe8d4e804'  # dev
+    gbp_subnet_add_del: '0xf42b9430'  # dev
+    gbp_subnet_add_del_reply: '0xe8d4e804'  # dev
+    gpe_enable_disable: '0xeb0e943b'  # dev
+    gpe_enable_disable_reply: '0xe8d4e804'  # dev

     # gre_tunnel_add_del / reply # unused L1 keyword: create_gre_tunnel_interface
     hw_interface_set_mtu: '0xe6746899'  # dev
     hw_interface_set_mtu_reply: '0xe8d4e804'  # dev
     # gre_tunnel_add_del / reply # unused L1 keyword: create_gre_tunnel_interface
     hw_interface_set_mtu: '0xe6746899'  # dev
     hw_interface_set_mtu_reply: '0xe8d4e804'  # dev


@@ -103,8 +103,7 @@
     ip_address_dump: '0x6b7bcd0a'  # dev
     ip_neighbor_add_del: '0x029dad44'  # dev
     ip_neighbor_add_del_reply: '0x1992deab'  # dev
     ip_address_dump: '0x6b7bcd0a'  # dev
     ip_neighbor_add_del: '0x029dad44'  # dev
     ip_neighbor_add_del_reply: '0x1992deab'  # dev

-    ip_probe_neighbor: '0x37bc128d'  # virl
-    ip_probe_neighbor_reply: '0xe8d4e804'  # virl
+    # ip_probe_neighbor / reply # unused L1 keyword vpp_ip_probe

     ip_route_add_del: '0x5ceee41c'  # dev
     ip_route_add_del_reply: '0x1992deab'  # dev
     # ip_source_check_interface_add_del / reply # unused L1 keyword vpp_ip_source_check_setup
     ip_route_add_del: '0x5ceee41c'  # dev
     ip_route_add_del_reply: '0x1992deab'  # dev
     # ip_source_check_interface_add_del / reply # unused L1 keyword vpp_ip_source_check_setup


@@ -116,8 +115,8 @@
     ipsec_sad_entry_add_del_reply: '0x9ffac24b'  # dev
     ipsec_select_backend: '0x4fd24836'  # perf
     ipsec_select_backend_reply: '0xe8d4e804'  # perf
     ipsec_sad_entry_add_del_reply: '0x9ffac24b'  # dev
     ipsec_select_backend: '0x4fd24836'  # perf
     ipsec_select_backend_reply: '0xe8d4e804'  # perf

-    # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-policy-aes256gcm-mrr
-    # ^ nic_intel-xl710ANDipsechwANDtnl_1ANDaes_256_gcm
+    # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes256gcm-mrr
+    # ^ nic_intel-xl710ANDipsechwANDipsecintANDtnl_1ANDaes_256_gcm

     ipsec_spd_add_del: '0x9ffdf5da'  # dev
     ipsec_spd_add_del_reply: '0xe8d4e804'  # dev
     ipsec_spd_entry_add_del: '0xdb217840'  # dev
     ipsec_spd_add_del: '0x9ffdf5da'  # dev
     ipsec_spd_add_del_reply: '0xe8d4e804'  # dev
     ipsec_spd_entry_add_del: '0xdb217840'  # dev


@@ -125,33 +124,45 @@
     ipsec_tunnel_if_add_del: '0xd5a98274'  # perf
     ipsec_tunnel_if_add_del_reply: '0xfda5941f'  # perf
     # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes256gcm-mrr
     ipsec_tunnel_if_add_del: '0xd5a98274'  # perf
     ipsec_tunnel_if_add_del_reply: '0xfda5941f'  # perf
     # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes256gcm-mrr

-    # ^ See select_backend, the tag expression selects both -int- and -policy-.
+    # ^ See select_backend.

     # l2_fib_table_dump / details # honeycomb
     l2_interface_vlan_tag_rewrite: '0xb90be6b4'  # dev
     l2_interface_vlan_tag_rewrite_reply: '0xe8d4e804'  # dev
     l2_patch_add_del: '0x62506e63'  # perf
     l2_patch_add_del_reply: '0xe8d4e804'  # perf
     # l2_fib_table_dump / details # honeycomb
     l2_interface_vlan_tag_rewrite: '0xb90be6b4'  # dev
     l2_interface_vlan_tag_rewrite_reply: '0xe8d4e804'  # dev
     l2_patch_add_del: '0x62506e63'  # perf
     l2_patch_add_del_reply: '0xe8d4e804'  # perf

-    # ^^ tc01-64B-1c-avf-eth-l2patch-mrr
+    # ^^ tc01-64B-1c-avf-eth-l2patch-mrr # currently failing after calls

     # ^ l2patchANDdrv_avf
     # l2fib_add_del / reply # unused L1 keyword: vpp_add_l2fib_entry
     # ^ l2patchANDdrv_avf
     # l2fib_add_del / reply # unused L1 keyword: vpp_add_l2fib_entry

-    lisp_add_del_adjacency: '0xf047390d'  # virl
-    lisp_add_del_adjacency_reply: '0xe8d4e804'  # virl
-    lisp_add_del_local_eid: '0xe6d00717'  # virl
-    lisp_add_del_local_eid_reply: '0xe8d4e804'  # virl
-    lisp_add_del_locator: '0x006a4240'  # virl
-    lisp_add_del_locator_reply: '0xe8d4e804'  # virl
-    lisp_add_del_locator_set: '0x06968e38'  # virl
-    lisp_add_del_locator_set_reply: '0xb6666db4'  # virl
+    lb_conf: '0x22ddb739'  # perf
+    lb_conf_reply: '0xe8d4e804'  # perf
+    lb_add_del_vip: '0xd15b7ddc'  # perf
+    lb_add_del_vip_reply: '0xe8d4e804'  # perf
+    lb_add_del_as: '0x78628987'  # perf
+    lb_add_del_as_reply: '0xe8d4e804'  # perf
+    # 6x^ tc01-64B-1c-ethip4-loadbalancer-maglev
+    # ^ 2n: loadbalancer
+    lb_add_del_intf_nat4: '0x47d6e753' # perf
+    lb_add_del_intf_nat4_reply: '0xe8d4e804' # perf
+    # ^^ tc01-64B-1c-ethip4-loadbalancer-nat4
+    # ^^ loadbalancer-nat4
+    lisp_add_del_adjacency: '0xf047390d'  # dev
+    lisp_add_del_adjacency_reply: '0xe8d4e804'  # dev
+    lisp_add_del_local_eid: '0xe6d00717'  # dev
+    lisp_add_del_local_eid_reply: '0xe8d4e804'  # dev
+    lisp_add_del_locator: '0x006a4240'  # dev
+    lisp_add_del_locator_reply: '0xe8d4e804'  # dev
+    lisp_add_del_locator_set: '0x06968e38'  # dev
+    lisp_add_del_locator_set_reply: '0xb6666db4'  # dev

     # lisp_add_del_map_resolver / reply # unused L2 keyword: Configure LISP map resolver address
     # lisp_add_del_map_resolver / reply # unused L2 keyword: Configure LISP map resolver address

-    lisp_add_del_remote_mapping: '0xb879c3a9'  # virl
-    lisp_add_del_remote_mapping_reply: '0xe8d4e804'  # virl
+    lisp_add_del_remote_mapping: '0xb879c3a9'  # dev
+    lisp_add_del_remote_mapping_reply: '0xe8d4e804'  # dev

     lisp_eid_table_add_del_map: '0x59e9975e'  # perf
     lisp_eid_table_add_del_map_reply: '0xe8d4e804'  # perf
     lisp_eid_table_add_del_map: '0x59e9975e'  # perf
     lisp_eid_table_add_del_map_reply: '0xe8d4e804'  # perf

-    # ^^ tc01-64B-1c-ethip4ipsectptlispgpe-ip4base-aes128cbc-hmac256sha-ndrpdr
+    # ^^ tc01-64B-1c-ethip4ipsectptlispgpe-ip4base-aes128cbc-hmac256sha-ndrpdr # currently failing before calls

     # ^ ipsecANDlispgpe
     # lisp_eid_table_dump / details # unused L2 keyword: LISP eid address should be set correctly to eid table
     # ^ ipsecANDlispgpe
     # lisp_eid_table_dump / details # unused L2 keyword: LISP eid address should be set correctly to eid table

-    lisp_enable_disable: '0xeb0e943b'  # virl
-    lisp_enable_disable_reply: '0xe8d4e804'  # virl
+    lisp_enable_disable: '0xeb0e943b'  # dev
+    lisp_enable_disable_reply: '0xe8d4e804'  # dev

     # lisp_locator_set_dump / details # unused L2 keyword: LISP locator_set shpuld be configured correctly
     # lisp_map_server_dump / details # honeycomb
     # lisp_map_resolver_dump / details # unused L2 keyword: LISP map resolver address should be configured correctly
     # lisp_locator_set_dump / details # unused L2 keyword: LISP locator_set shpuld be configured correctly
     # lisp_map_server_dump / details # honeycomb
     # lisp_map_resolver_dump / details # unused L2 keyword: LISP map resolver address should be configured correctly


@@ -179,6 +190,11 @@
     nat_show_config_reply: '0x006a0786'  # perf teardown
     # 6x^ tc01-64B-1c-ethip4udp-ip4base-nat44-mrr
     # ^ nat44NOTscaleNOTsrc_user_1
     nat_show_config_reply: '0x006a0786'  # perf teardown
     # 6x^ tc01-64B-1c-ethip4udp-ip4base-nat44-mrr
     # ^ nat44NOTscaleNOTsrc_user_1

+    policer_add_del: '0xdfea2be8'  # dev
+    policer_add_del_reply: '0xa177cef2'  # dev
+    policer_classify_set_interface: '0xe09537b0' # dev
+    policer_classify_set_interface_reply: '0xe8d4e804' # dev
+    # 4x^ tc01-64B-ethip4-ip4base-ipolicemarkbase-dev

     # show_lisp_map_register_state / reply # honeycomb
     # show_lisp_map_request_mode / reply # honeycomb
     # show_lisp_pitr / reply # honeycomb
     # show_lisp_map_register_state / reply # honeycomb
     # show_lisp_map_request_mode / reply # honeycomb
     # show_lisp_pitr / reply # honeycomb


@@ -203,7 +219,7 @@
     sr_steering_add_del_reply: '0xe8d4e804'  # perf
     sr_steering_pol_details: '0x5627d483'  # perf teardown
     sr_steering_pol_dump: '0x51077d14'  # perf teardown
     sr_steering_add_del_reply: '0xe8d4e804'  # perf
     sr_steering_pol_details: '0x5627d483'  # perf teardown
     sr_steering_pol_dump: '0x51077d14'  # perf teardown

-    # x^ tc01-78B-1c-ethip6ip6-ip6base-srv6enc1sid-mrr
+    # 14x^ tc01-78B-1c-ethip6ip6-ip6base-srv6enc1sid-mrr

     # ^ srv6_1sid
     sw_interface_add_del_address: '0x5803d5c4'  # dev
     sw_interface_add_del_address_reply: '0xe8d4e804'  # dev
     # ^ srv6_1sid
     sw_interface_add_del_address: '0x5803d5c4'  # dev
     sw_interface_add_del_address_reply: '0xe8d4e804'  # dev


@@ -249,20 +265,8 @@
     vxlan_add_del_tunnel_reply: '0xfda5941f'  # dev
     # vxlan_gpe_tunnel_dump / details # honeycomb
     # vxlan_tunnel_dump /details # unused L2 keyword: Get VXLAN dump
     vxlan_add_del_tunnel_reply: '0xfda5941f'  # dev
     # vxlan_gpe_tunnel_dump / details # honeycomb
     # vxlan_tunnel_dump /details # unused L2 keyword: Get VXLAN dump

-    policer_add_del: '0xdfea2be8'  # dev
-    policer_add_del_reply: '0xa177cef2'  # dev
-    policer_classify_set_interface: '0xe09537b0' # dev
-    policer_classify_set_interface_reply: '0xe8d4e804' # dev
-    # 4x^ tc01-64B-ethip4-ip4base-ipolicemarkbase-dev
-    # ^^ tc01-64B-1c-ethip4-loadbalancer-maglev/l3dsr/nat4-mrr
-    lb_conf: '0x22ddb739'  # perf
-    lb_conf_reply: '0xe8d4e804'  # perf
-    lb_add_del_vip: '0xd15b7ddc'  # perf
-    lb_add_del_vip_reply: '0xe8d4e804'  # perf
-    lb_add_del_as: '0x78628987'  # perf
-    lb_add_del_as_reply: '0xe8d4e804'  # perf
-    lb_add_del_intf_nat4: '0x47d6e753' # perf
-    lb_add_del_intf_nat4_reply: '0xe8d4e804' # perf
+# Please keep alphabetic order.
+# Use bash command "env LC_COLLATE=C sort -u" if not clear.

 
 
 # Hint to see the currently used command messages:
 
 
 # Hint to see the currently used command messages:


@@ -278,13 +282,14 @@
 # mrrAND1cAND64bANDnic_intel-x710ANDip4fwdANDiaclANDacl10AND100_flows
 # mrrAND1cAND64bANDnic_intel-x710ANDnat44NOTscaleNOTsrc_user_1
 # mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDl2bdmaclrnANDbaseANDmemif
 # mrrAND1cAND64bANDnic_intel-x710ANDip4fwdANDiaclANDacl10AND100_flows
 # mrrAND1cAND64bANDnic_intel-x710ANDnat44NOTscaleNOTsrc_user_1
 # mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDl2bdmaclrnANDbaseANDmemif

-# mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDlbond_1l
+# mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDlbond_1lANDnf_testpmd

 # mrrAND1cAND64bANDnic_intel-x710ANDmacipANDacl1AND100_flows
 # mrrAND1cAND78bANDnic_intel-x710ANDsrv6_1sid
 # mrrAND1cAND64bANDnic_intel-xl710ANDipsechwANDtnl_1ANDaes_256_gcm
 # mrrAND1cAND64bANDnic_intel-x710ANDmacipANDacl1AND100_flows
 # mrrAND1cAND78bANDnic_intel-x710ANDsrv6_1sid
 # mrrAND1cAND64bANDnic_intel-xl710ANDipsechwANDtnl_1ANDaes_256_gcm

-# mrrAND1cAND64bANDnic_intel-xl710ANDipsecANDlispgpe
+# mrrAND1cAND64bANDnic_intel-x710ANDdot1adANDl2xcfwd
+# mrrAND1cAND64bANDnic_intel-x710ANDipsecANDlispgpe

 # csit-2n-skx-perftest
 # csit-2n-skx-perftest

-# mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDbaseANDdrv_avfANDgbp
+# mrrAND1cAND64bANDnic_intel-x710ANDloadbalancer

 # mrrAND1cAND64bANDnic_intel-x710ANDl2patchANDdrv_avf
 
 # TODO: Once device job has complete API coverage,
 # mrrAND1cAND64bANDnic_intel-x710ANDl2patchANDdrv_avf
 
 # TODO: Once device job has complete API coverage,

diff --git a/resources/libraries/python/VppApiCrc.py b/resources/libraries/python/VppApiCrc.py
index 9a5247a..8f2979e 100644 (file)
--- a/resources/libraries/python/VppApiCrc.py
+++ b/resources/libraries/python/VppApiCrc.py
@@ -36,7 +36,7 @@ class VppApiCrcChecker(object):
     """Holder of data related to tracking VPP API CRCs.
 
     Both message names and crc hexa strings are tracked as
     """Holder of data related to tracking VPP API CRCs.
 
     Both message names and crc hexa strings are tracked as

-    ordinary Python2 (bytes) str, so _str() is used when input is
+    ordinary Python2 (bytes) string, so _str() is used when input is

     possibly unicode or otherwise not safe.
 
     Each instance of this class starts with same default state,
     possibly unicode or otherwise not safe.
 
     Each instance of this class starts with same default state,


@@ -67,14 +67,14 @@ class VppApiCrcChecker(object):
         Order of addition reflects the order colections should be queried.
         If an incompatible CRC is found, affected collections are removed.
         A CRC that would remove all does not, added to _reported instead,
         Order of addition reflects the order colections should be queried.
         If an incompatible CRC is found, affected collections are removed.
         A CRC that would remove all does not, added to _reported instead,

-        while causing a failure in single test."""
+        while causing a failure in single test (if fail_on_mismatch)."""

 
         self._missing = dict()
         """Mapping from collection name to mapping from API name to CRC string.
 
         Starts the same as _expected, but each time an encountered api,crc pair
 
         self._missing = dict()
         """Mapping from collection name to mapping from API name to CRC string.
 
         Starts the same as _expected, but each time an encountered api,crc pair

-        fits the expectation, the pair is removed from this mapping.
-        Ideally, the active mappings will become empty.
+        fits the expectation, the pair is removed from all collections
+        within this mapping. Ideally, the active mappings will become empty.

         If not, it is an error, VPP removed or renamed a message CSIT needs."""
 
         self._found = dict()
         If not, it is an error, VPP removed or renamed a message CSIT needs."""
 
         self._found = dict()


@@ -114,11 +114,12 @@ class VppApiCrcChecker(object):
         :param name_to_crc_mapping: Mapping from API names to CRCs.
         :type collection_name: str or unicode
         :type name_to_crc_mapping: dict from str/unicode to str/unicode
         :param name_to_crc_mapping: Mapping from API names to CRCs.
         :type collection_name: str or unicode
         :type name_to_crc_mapping: dict from str/unicode to str/unicode

+        :raises RuntimeError: If the name of a collection is registered already.

         """
         collection_name = _str(collection_name)
         if collection_name in self._expected:
         """
         collection_name = _str(collection_name)
         if collection_name in self._expected:

-            raise RuntimeError("Collection {cl!r} already registered.".format(
-                cl=collection_name))
+            raise RuntimeError("Collection {cn!r} already registered.".format(
+                cn=collection_name))

         mapping = {_str(k): _str(v) for k, v in name_to_crc_mapping.items()}
         self._expected[collection_name] = mapping
         self._missing[collection_name] = mapping.copy()
         mapping = {_str(k): _str(v) for k, v in name_to_crc_mapping.items()}
         self._expected[collection_name] = mapping
         self._missing[collection_name] = mapping.copy()


@@ -173,6 +174,8 @@ class VppApiCrcChecker(object):
     def _process_crc(self, api_name, crc):
         """Compare API to verified collections, update class state.
 
     def _process_crc(self, api_name, crc):
         """Compare API to verified collections, update class state.
 

+        Here, API stands for (message name, CRC) pair.
+

         Conflict is NOT when a collection does not recognize the API.
         Such APIs are merely added to _found for later reporting.
         Conflict is when a collection recognizes the API under a different CRC.
         Conflict is NOT when a collection does not recognize the API.
         Such APIs are merely added to _found for later reporting.
         Conflict is when a collection recognizes the API under a different CRC.


@@ -225,7 +228,7 @@ class VppApiCrcChecker(object):
         """Parse every .api.json found under directory, remember conflicts.
 
         As several collections are supported, each conflict invalidates
         """Parse every .api.json found under directory, remember conflicts.
 
         As several collections are supported, each conflict invalidates

-        one of them, failure happens only when no collections would be left.
+        some of them, failure happens only when no collections would be left.

         In that case, set of collections just before the failure is preserved,
         the _reported mapping is filled with conflicting APIs.
         The _found mapping is filled with discovered api names and crcs.
         In that case, set of collections just before the failure is preserved,
         the _reported mapping is filled with conflicting APIs.
         The _found mapping is filled with discovered api names and crcs.


@@ -259,6 +262,9 @@ class VppApiCrcChecker(object):
         Missing reporting is disabled by default, because some messages
         come from plugins that might not be enabled at runtime.
 
         Missing reporting is disabled by default, because some messages
         come from plugins that might not be enabled at runtime.
 

+        After the report, clear _reported, so that test cases report them again,
+        thus tracking which message is actually used (by which test).
+

         :param report_missing: Whether to raise on missing messages.
         :type report_missing: bool
         :raises RuntimeError: If CRC mismatch or missing messages are detected,
         :param report_missing: Whether to raise on missing messages.
         :type report_missing: bool
         :raises RuntimeError: If CRC mismatch or missing messages are detected,


@@ -270,8 +276,9 @@ class VppApiCrcChecker(object):
         if self._reported:
             reported_indented = json.dumps(
                 self._reported, indent=1, sort_keys=True, separators=[",", ":"])
         if self._reported:
             reported_indented = json.dumps(
                 self._reported, indent=1, sort_keys=True, separators=[",", ":"])

+            self._reported = dict()

             self.log_and_raise(
             self.log_and_raise(

-                "Dir check found incompatible API CRCs:\n{ri}".format(
+                "Incompatible API CRCs found in .api.json files:\n{ri}".format(

                     ri=reported_indented))
         if not report_missing:
             return
                     ri=reported_indented))
         if not report_missing:
             return


@@ -279,15 +286,16 @@ class VppApiCrcChecker(object):
         if missing:
             missing_indented = json.dumps(
                 missing, indent=1, sort_keys=True, separators=[",", ":"])
         if missing:
             missing_indented = json.dumps(
                 missing, indent=1, sort_keys=True, separators=[",", ":"])

-            self.log_and_raise("Dir check found missing API CRCs:\n{mi}".format(
+            self.log_and_raise("API CRCs missing from .api.json:\n{mi}".format(

                 mi=missing_indented))
 
     def check_api_name(self, api_name):
                 mi=missing_indented))
 
     def check_api_name(self, api_name):

-        """Fail if the api_name has no known CRC associated.
+        """Fail if the api_name has no, or different from known CRC associated.

 
         Do not fail if this particular failure has been already reported.
 
 
         Do not fail if this particular failure has been already reported.
 

-        Intended use: Call everytime an API call is queued or response received.
+        Intended use: Call during test (not in initialization),
+        everytime an API call is queued or response received.

 
         :param api_name: VPP API messagee name to check.
         :type api_name: str or unicode
 
         :param api_name: VPP API messagee name to check.
         :type api_name: str or unicode


@@ -306,8 +314,19 @@ class VppApiCrcChecker(object):
         if new_expected:
             # Some collections recognized the message name.
             self._expected = new_expected
         if new_expected:
             # Some collections recognized the message name.
             self._expected = new_expected

-            return

         crc = self._found.get(api_name, None)
         crc = self._found.get(api_name, None)

+        matching = False
+        if crc is not None:
+            # Regardless of how many collections are remaining,
+            # verify the known CRC is on one of them.
+            for name_to_crc_mapping in self._expected.values():
+                if api_name not in name_to_crc_mapping:
+                    continue
+                if name_to_crc_mapping[api_name] == crc:
+                    matching = True
+                    break
+        if matching:
+            return

         self._reported[api_name] = crc
         self._reported[api_name] = crc

-        self.log_and_raise("No active collection has API {api!r}"
-                           " CRC found {crc!r}".format(api=api_name, crc=crc))
+        self.log_and_raise("No active collection contains API {api!r}"
+                           " with CRC {crc!r}".format(api=api_name, crc=crc))