X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Facl%2Fpublic_inlines.h;h=ba174c9b2a63ab277639dbd8a4e6fde1f446bc03;hb=87ee947d0b053b33571c5e33617b138236bada59;hp=f7d7abbec0a7cf6836c2d3186978cd6367502ad8;hpb=22f9fb1286d2469819cfcef68ffdc258f4d52c24;p=vpp.git diff --git a/src/plugins/acl/public_inlines.h b/src/plugins/acl/public_inlines.h index f7d7abbec0a..ba174c9b2a6 100644 --- a/src/plugins/acl/public_inlines.h +++ b/src/plugins/acl/public_inlines.h @@ -58,6 +58,13 @@ offset_within_packet (vlib_buffer_t * b0, int offset) return (offset <= (b0->current_length - 8)); } +always_inline int +offset_beyond_packet (vlib_buffer_t * b0, int offset) +{ + /* For the purposes of this code, "within" means we have at least 8 bytes after it */ + return (offset > (b0->current_length - 8)); +} + always_inline void acl_fill_5tuple_l3_data (acl_main_t * am, vlib_buffer_t * b0, int is_ip6, @@ -65,20 +72,19 @@ acl_fill_5tuple_l3_data (acl_main_t * am, vlib_buffer_t * b0, int is_ip6, { if (is_ip6) { - clib_memcpy (&p5tuple_pkt->ip6_addr, - get_ptr_to_offset (b0, - offsetof (ip6_header_t, - src_address) + l3_offset), - sizeof (p5tuple_pkt->ip6_addr)); + ip6_header_t *ip6 = vlib_buffer_get_current (b0) + l3_offset; + p5tuple_pkt->ip6_addr[0] = ip6->src_address; + p5tuple_pkt->ip6_addr[1] = ip6->dst_address; } else { - memset(p5tuple_pkt->l3_zero_pad, 0, sizeof(p5tuple_pkt->l3_zero_pad)); - clib_memcpy (&p5tuple_pkt->ip4_addr, - get_ptr_to_offset (b0, - offsetof (ip4_header_t, - src_address) + l3_offset), - sizeof (p5tuple_pkt->ip4_addr)); + int ii; + for(ii=0; ii<6; ii++) { + p5tuple_pkt->l3_zero_pad[ii] = 0; + } + ip4_header_t *ip4 = vlib_buffer_get_current (b0) + l3_offset; + p5tuple_pkt->ip4_addr[0] = ip4->src_address; + p5tuple_pkt->ip4_addr[1] = ip4->dst_address; } } @@ -90,23 +96,19 @@ acl_fill_5tuple_l4_and_pkt_data (acl_main_t * am, u32 sw_if_index0, vlib_buffer_ static u8 icmp_protos_v4v6[] = { IP_PROTOCOL_ICMP, IP_PROTOCOL_ICMP6 }; int l4_offset; - u16 ports[2]; + u16 ports[2] = { 0 }; u8 proto; - fa_session_l4_key_t tmp_l4 = { .lsb_of_sw_if_index = sw_if_index0 & 0xffff }; + u8 tmp_l4_flags = 0; fa_packet_info_t tmp_pkt = { .is_ip6 = is_ip6, .mask_type_index_lsb = ~0 }; if (is_ip6) { - proto = - *(u8 *) get_ptr_to_offset (b0, - offsetof (ip6_header_t, - protocol) + l3_offset); + ip6_header_t *ip6 = vlib_buffer_get_current (b0) + l3_offset; + proto = ip6->protocol; + l4_offset = l3_offset + sizeof (ip6_header_t); -#ifdef FA_NODE_VERBOSE_DEBUG - clib_warning ("ACL_FA_NODE_DBG: proto: %d, l4_offset: %d", proto, - l4_offset); -#endif + /* IP6 EH handling is here, increment l4_offset if needs to, update the proto */ int need_skip_eh = clib_bitmap_get (am->fa_ipv6_known_eh_bitmap, proto); if (PREDICT_FALSE (need_skip_eh)) @@ -117,8 +119,7 @@ acl_fill_5tuple_l4_and_pkt_data (acl_main_t * am, u32 sw_if_index0, vlib_buffer_ if (PREDICT_FALSE(ACL_EH_FRAGMENT == proto)) { proto = *(u8 *) get_ptr_to_offset (b0, l4_offset); - u16 frag_offset; - clib_memcpy (&frag_offset, get_ptr_to_offset (b0, 2 + l4_offset), sizeof(frag_offset)); + u16 frag_offset = *(u16 *) get_ptr_to_offset (b0, 2 + l4_offset); frag_offset = clib_net_to_host_u16(frag_offset) >> 3; if (frag_offset) { @@ -138,10 +139,6 @@ acl_fill_5tuple_l4_and_pkt_data (acl_main_t * am, u32 sw_if_index0, vlib_buffer_ proto = *(u8 *) get_ptr_to_offset (b0, l4_offset); l4_offset += 8 * (1 + (u16) nwords); } -#ifdef FA_NODE_VERBOSE_DEBUG - clib_warning ("ACL_FA_NODE_DBG: new proto: %d, new offset: %d", - proto, l4_offset); -#endif need_skip_eh = clib_bitmap_get (am->fa_ipv6_known_eh_bitmap, proto); } @@ -149,21 +146,12 @@ acl_fill_5tuple_l4_and_pkt_data (acl_main_t * am, u32 sw_if_index0, vlib_buffer_ } else { - proto = - *(u8 *) get_ptr_to_offset (b0, - offsetof (ip4_header_t, - protocol) + l3_offset); - l4_offset = l3_offset + sizeof (ip4_header_t); - u16 flags_and_fragment_offset; - clib_memcpy (&flags_and_fragment_offset, - get_ptr_to_offset (b0, - offsetof (ip4_header_t, - flags_and_fragment_offset)) + l3_offset, - sizeof(flags_and_fragment_offset)); - flags_and_fragment_offset = clib_net_to_host_u16 (flags_and_fragment_offset); + ip4_header_t *ip4 = vlib_buffer_get_current (b0) + l3_offset; + proto = ip4->protocol; + l4_offset = l3_offset + ip4_header_bytes(ip4); /* non-initial fragments have non-zero offset */ - if ((PREDICT_FALSE(0xfff & flags_and_fragment_offset))) + if (PREDICT_FALSE(ip4_get_fragment_offset(ip4))) { tmp_pkt.is_nonfirst_fragment = 1; /* invalidate L4 offset so we don't try to find L4 info */ @@ -171,50 +159,47 @@ acl_fill_5tuple_l4_and_pkt_data (acl_main_t * am, u32 sw_if_index0, vlib_buffer_ } } - tmp_l4.proto = proto; - tmp_l4.is_input = is_input; + tmp_l4_flags |= is_input ? FA_SK_L4_FLAG_IS_INPUT : 0; if (PREDICT_TRUE (offset_within_packet (b0, l4_offset))) { + tcp_header_t *tcph = vlib_buffer_get_current (b0) + l4_offset; + udp_header_t *udph = vlib_buffer_get_current (b0) + l4_offset; tmp_pkt.l4_valid = 1; - if (icmp_protos_v4v6[is_ip6] == proto) + + if (PREDICT_FALSE(icmp_protos_v4v6[is_ip6] == proto)) { - /* type */ - tmp_l4.port[0] = - *(u8 *) get_ptr_to_offset (b0, - l4_offset + offsetof (icmp46_header_t, - type)); - /* code */ - tmp_l4.port[1] = - *(u8 *) get_ptr_to_offset (b0, - l4_offset + offsetof (icmp46_header_t, - code)); - tmp_l4.is_slowpath = 1; + icmp46_header_t *icmph = vlib_buffer_get_current (b0) + l4_offset; + ports[0] = icmph->type; + ports[1] = icmph->code; + /* ICMP needs special handling */ + tmp_l4_flags |= FA_SK_L4_FLAG_IS_SLOWPATH; } - else if ((IP_PROTOCOL_TCP == proto) || (IP_PROTOCOL_UDP == proto)) + else if (IP_PROTOCOL_TCP == proto) { - clib_memcpy (&ports, - get_ptr_to_offset (b0, - l4_offset + offsetof (tcp_header_t, - src_port)), - sizeof (ports)); - tmp_l4.port[0] = clib_net_to_host_u16 (ports[0]); - tmp_l4.port[1] = clib_net_to_host_u16 (ports[1]); - - tmp_pkt.tcp_flags = - *(u8 *) get_ptr_to_offset (b0, - l4_offset + offsetof (tcp_header_t, - flags)); - tmp_pkt.tcp_flags_valid = (proto == IP_PROTOCOL_TCP); - tmp_l4.is_slowpath = 0; + ports[0] = clib_net_to_host_u16(tcph->src_port); + ports[1] = clib_net_to_host_u16(tcph->dst_port); + tmp_pkt.tcp_flags = tcph->flags; + tmp_pkt.tcp_flags_valid = 1; } + else if (IP_PROTOCOL_UDP == proto) + { + ports[0] = clib_net_to_host_u16(udph->src_port); + ports[1] = clib_net_to_host_u16(udph->dst_port); + } else { - tmp_l4.is_slowpath = 1; + tmp_l4_flags |= FA_SK_L4_FLAG_IS_SLOWPATH; } } p5tuple_pkt->as_u64 = tmp_pkt.as_u64; + + fa_session_l4_key_t tmp_l4 = { .port = { ports[0], ports[1] }, + .proto = proto, + .l4_flags = tmp_l4_flags, + .lsb_of_sw_if_index = sw_if_index0 & 0xffff }; + p5tuple_l4->as_u64 = tmp_l4.as_u64; } @@ -489,91 +474,156 @@ match_portranges(acl_main_t *am, fa_5tuple_t *match, u32 index) ((r->dst_port_or_code_first <= match->l4.port[1]) && r->dst_port_or_code_last >= match->l4.port[1]) ); } +always_inline int +single_rule_match_5tuple (acl_rule_t * r, int is_ip6, fa_5tuple_t * pkt_5tuple) +{ + if (is_ip6 != r->is_ipv6) + { + return 0; + } + + if (is_ip6) + { + if (!fa_acl_match_ip6_addr + (&pkt_5tuple->ip6_addr[1], &r->dst.ip6, r->dst_prefixlen)) + return 0; + if (!fa_acl_match_ip6_addr + (&pkt_5tuple->ip6_addr[0], &r->src.ip6, r->src_prefixlen)) + return 0; + } + else + { + if (!fa_acl_match_ip4_addr + (&pkt_5tuple->ip4_addr[1], &r->dst.ip4, r->dst_prefixlen)) + return 0; + if (!fa_acl_match_ip4_addr + (&pkt_5tuple->ip4_addr[0], &r->src.ip4, r->src_prefixlen)) + return 0; + } + + if (r->proto) + { + if (pkt_5tuple->l4.proto != r->proto) + return 0; + + /* A sanity check just to ensure we are about to match the ports extracted from the packet */ + if (PREDICT_FALSE (!pkt_5tuple->pkt.l4_valid)) + return 0; + + + if (!fa_acl_match_port + (pkt_5tuple->l4.port[0], r->src_port_or_type_first, + r->src_port_or_type_last, pkt_5tuple->pkt.is_ip6)) + return 0; + + + if (!fa_acl_match_port + (pkt_5tuple->l4.port[1], r->dst_port_or_code_first, + r->dst_port_or_code_last, pkt_5tuple->pkt.is_ip6)) + return 0; + + if (pkt_5tuple->pkt.tcp_flags_valid + && ((pkt_5tuple->pkt.tcp_flags & r->tcp_flags_mask) != + r->tcp_flags_value)) + return 0; + } + /* everything matches! */ + return 1; +} + always_inline u32 -multi_acl_match_get_applied_ace_index(acl_main_t *am, fa_5tuple_t *match) +multi_acl_match_get_applied_ace_index (acl_main_t * am, int is_ip6, fa_5tuple_t * match) { clib_bihash_kv_48_8_t kv; clib_bihash_kv_48_8_t result; - fa_5tuple_t *kv_key = (fa_5tuple_t *)kv.key; - hash_acl_lookup_value_t *result_val = (hash_acl_lookup_value_t *)&result.value; - u64 *pmatch = (u64 *)match; + fa_5tuple_t *kv_key = (fa_5tuple_t *) kv.key; + hash_acl_lookup_value_t *result_val = + (hash_acl_lookup_value_t *) & result.value; + u64 *pmatch = (u64 *) match; u64 *pmask; u64 *pkey; - int mask_type_index; - u32 curr_match_index = ~0; + int mask_type_index, order_index; + u32 curr_match_index = (~0 - 1); + + u32 lc_index = match->pkt.lc_index; - applied_hash_ace_entry_t **applied_hash_aces = vec_elt_at_index(am->hash_entry_vec_by_lc_index, match->pkt.lc_index); - applied_hash_acl_info_t **applied_hash_acls = &am->applied_hash_acl_info_by_lc_index; + applied_hash_ace_entry_t **applied_hash_aces = + vec_elt_at_index (am->hash_entry_vec_by_lc_index, lc_index); - DBG("TRYING TO MATCH: %016llx %016llx %016llx %016llx %016llx %016llx", - pmatch[0], pmatch[1], pmatch[2], pmatch[3], pmatch[4], pmatch[5]); + hash_applied_mask_info_t **hash_applied_mask_info_vec = + vec_elt_at_index (am->hash_applied_mask_info_vec_by_lc_index, lc_index); - for(mask_type_index=0; mask_type_index < pool_len(am->ace_mask_type_pool); mask_type_index++) { - if (!clib_bitmap_get(vec_elt_at_index((*applied_hash_acls), lc_index)->mask_type_index_bitmap, mask_type_index)) { - /* This bit is not set. Avoid trying to match */ - continue; - } - ace_mask_type_entry_t *mte = vec_elt_at_index(am->ace_mask_type_pool, mask_type_index); - pmatch = (u64 *)match; - pmask = (u64 *)&mte->mask; - pkey = (u64 *)kv.key; - /* - * unrolling the below loop results in a noticeable performance increase. - int i; - for(i=0; i<6; i++) { - kv.key[i] = pmatch[i] & pmask[i]; - } - */ - - *pkey++ = *pmatch++ & *pmask++; - *pkey++ = *pmatch++ & *pmask++; - *pkey++ = *pmatch++ & *pmask++; - *pkey++ = *pmatch++ & *pmask++; - *pkey++ = *pmatch++ & *pmask++; - *pkey++ = *pmatch++ & *pmask++; - - kv_key->pkt.mask_type_index_lsb = mask_type_index; - DBG(" KEY %3d: %016llx %016llx %016llx %016llx %016llx %016llx", mask_type_index, - kv.key[0], kv.key[1], kv.key[2], kv.key[3], kv.key[4], kv.key[5]); - int res = clib_bihash_search_48_8 (&am->acl_lookup_hash, &kv, &result); - if (res == 0) { - DBG("ACL-MATCH! result_val: %016llx", result_val->as_u64); - if (result_val->applied_entry_index < curr_match_index) { - if (PREDICT_FALSE(result_val->need_portrange_check)) { - /* - * This is going to be slow, since we can have multiple superset - * entries for narrow-ish portranges, e.g.: - * 0..42 100..400, 230..60000, - * so we need to walk linearly and check if they match. - */ - - u32 curr_index = result_val->applied_entry_index; - while ((curr_index != ~0) && !match_portranges(am, match, curr_index)) { - /* while no match and there are more entries, walk... */ - applied_hash_ace_entry_t *pae = vec_elt_at_index((*applied_hash_aces),curr_index); - DBG("entry %d did not portmatch, advancing to %d", curr_index, pae->next_applied_entry_index); - curr_index = pae->next_applied_entry_index; - } - if (curr_index < curr_match_index) { - DBG("The index %d is the new candidate in portrange matches.", curr_index); - curr_match_index = curr_index; - } else { - DBG("Curr portmatch index %d is too big vs. current matched one %d", curr_index, curr_match_index); - } - } else { - /* The usual path is here. Found an entry in front of the current candiate - so it's a new one */ - DBG("This match is the new candidate"); - curr_match_index = result_val->applied_entry_index; - if (!result_val->shadowed) { - /* new result is known to not be shadowed, so no point to look up further */ - break; - } - } - } + hash_applied_mask_info_t *minfo; + + DBG ("TRYING TO MATCH: %016llx %016llx %016llx %016llx %016llx %016llx", + pmatch[0], pmatch[1], pmatch[2], pmatch[3], pmatch[4], pmatch[5]); + + for (order_index = 0; order_index < vec_len ((*hash_applied_mask_info_vec)); + order_index++) + { + minfo = vec_elt_at_index ((*hash_applied_mask_info_vec), order_index); + if (minfo->first_rule_index > curr_match_index) + { + /* Index in this and following (by construction) partitions are greater than our candidate, Avoid trying to match! */ + break; + } + + mask_type_index = minfo->mask_type_index; + ace_mask_type_entry_t *mte = + vec_elt_at_index (am->ace_mask_type_pool, mask_type_index); + pmatch = (u64 *) match; + pmask = (u64 *) & mte->mask; + pkey = (u64 *) kv.key; + /* + * unrolling the below loop results in a noticeable performance increase. + int i; + for(i=0; i<6; i++) { + kv.key[i] = pmatch[i] & pmask[i]; + } + */ + + *pkey++ = *pmatch++ & *pmask++; + *pkey++ = *pmatch++ & *pmask++; + *pkey++ = *pmatch++ & *pmask++; + *pkey++ = *pmatch++ & *pmask++; + *pkey++ = *pmatch++ & *pmask++; + *pkey++ = *pmatch++ & *pmask++; + + /* + * The use of temporary variable convinces the compiler + * to make a u64 write, avoiding the stall on crc32 operation + * just a bit later. + */ + fa_packet_info_t tmp_pkt = kv_key->pkt; + tmp_pkt.mask_type_index_lsb = mask_type_index; + kv_key->pkt.as_u64 = tmp_pkt.as_u64; + + int res = + clib_bihash_search_inline_2_48_8 (&am->acl_lookup_hash, &kv, &result); + + if (res == 0) + { + /* There is a hit in the hash, so check the collision vector */ + u32 curr_index = result_val->applied_entry_index; + applied_hash_ace_entry_t *pae = + vec_elt_at_index ((*applied_hash_aces), curr_index); + collision_match_rule_t *crs = pae->colliding_rules; + int i; + for (i = 0; i < vec_len (crs); i++) + { + if (crs[i].applied_entry_index >= curr_match_index) + { + continue; + } + if (single_rule_match_5tuple (&crs[i].rule, is_ip6, match)) + { + curr_match_index = crs[i].applied_entry_index; + } + } + } } - } - DBG("MATCH-RESULT: %d", curr_match_index); + DBG ("MATCH-RESULT: %d", curr_match_index); return curr_match_index; } @@ -584,7 +634,7 @@ hash_multi_acl_match_5tuple (void *p_acl_main, u32 lc_index, fa_5tuple_t * pkt_5 { acl_main_t *am = p_acl_main; applied_hash_ace_entry_t **applied_hash_aces = vec_elt_at_index(am->hash_entry_vec_by_lc_index, lc_index); - u32 match_index = multi_acl_match_get_applied_ace_index(am, pkt_5tuple); + u32 match_index = multi_acl_match_get_applied_ace_index(am, is_ip6, pkt_5tuple); if (match_index < vec_len((*applied_hash_aces))) { applied_hash_ace_entry_t *pae = vec_elt_at_index((*applied_hash_aces), match_index); pae->hitcount++; @@ -611,9 +661,20 @@ acl_plugin_match_5tuple_inline (void *p_acl_main, u32 lc_index, acl_main_t *am = p_acl_main; fa_5tuple_t * pkt_5tuple_internal = (fa_5tuple_t *)pkt_5tuple; pkt_5tuple_internal->pkt.lc_index = lc_index; - if (am->use_hash_acl_matching) { - return hash_multi_acl_match_5tuple(p_acl_main, lc_index, pkt_5tuple_internal, is_ip6, r_action, + if (PREDICT_TRUE(am->use_hash_acl_matching)) { + if (PREDICT_FALSE(pkt_5tuple_internal->pkt.is_nonfirst_fragment)) { + /* + * tuplemerge does not take fragments into account, + * and in general making fragments first class citizens has + * proved more overhead than it's worth - so just fall back to linear + * matching in that case. + */ + return linear_multi_acl_match_5tuple(p_acl_main, lc_index, pkt_5tuple_internal, is_ip6, r_action, r_acl_pos_p, r_acl_match_p, r_rule_match_p, trace_bitmap); + } else { + return hash_multi_acl_match_5tuple(p_acl_main, lc_index, pkt_5tuple_internal, is_ip6, r_action, + r_acl_pos_p, r_acl_match_p, r_rule_match_p, trace_bitmap); + } } else { return linear_multi_acl_match_5tuple(p_acl_main, lc_index, pkt_5tuple_internal, is_ip6, r_action, r_acl_pos_p, r_acl_match_p, r_rule_match_p, trace_bitmap);