X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fgtpu%2Fgtpu_encap.c;h=2c3c46a4be2aa00a314578424349599cdec5d1a1;hb=f9ab6985d;hp=ec33e1e4487493d8cdec9fd9edf066c0ab04a93a;hpb=fcdade1f8721da90017313bb9cb8a2326846d211;p=vpp.git diff --git a/src/plugins/gtpu/gtpu_encap.c b/src/plugins/gtpu/gtpu_encap.c index ec33e1e4487..2c3c46a4be2 100644 --- a/src/plugins/gtpu/gtpu_encap.c +++ b/src/plugins/gtpu/gtpu_encap.c @@ -199,7 +199,8 @@ gtpu_encap_inline (vlib_main_t * vm, copy_dst3 = (u64 *) ip4_3; copy_src3 = (u64 *) t3->rewrite; - /* Copy first 32 octets 8-bytes at a time */ + /* Copy first 32 octets 8-bytes at a time (minimum size) + * TODO: check if clib_memcpy_fast is better */ #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header4_offset; #undef _ @@ -212,19 +213,83 @@ gtpu_encap_inline (vlib_main_t * vm, #define _(offs) copy_dst3[offs] = copy_src3[offs]; foreach_fixed_header4_offset; #undef _ - /* Last 4 octets. Hopefully gcc will be our friend */ - copy_dst_last0 = (u32 *)(©_dst0[4]); - copy_src_last0 = (u32 *)(©_src0[4]); - copy_dst_last0[0] = copy_src_last0[0]; - copy_dst_last1 = (u32 *)(©_dst1[4]); - copy_src_last1 = (u32 *)(©_src1[4]); - copy_dst_last1[0] = copy_src_last1[0]; - copy_dst_last2 = (u32 *)(©_dst2[4]); - copy_src_last2 = (u32 *)(©_src2[4]); - copy_dst_last2[0] = copy_src_last2[0]; - copy_dst_last3 = (u32 *)(©_dst3[4]); - copy_src_last3 = (u32 *)(©_src3[4]); - copy_dst_last3[0] = copy_src_last3[0]; + + /* Copy last octets */ + if (_vec_len (t0->rewrite) == 36) + { + /* Last 4 octets. Hopefully gcc will be our friend */ + copy_dst_last0 = (u32 *) (©_dst0[4]); + copy_src_last0 = (u32 *) (©_src0[4]); + copy_dst_last0[0] = copy_src_last0[0]; + } + else + { + /* Near last 8 octets. */ +#define _(offs) copy_dst0[offs] = copy_src0[offs]; + _ (4); +#undef _ + /* Last 4 octets. Hopefully gcc will be our friend */ + copy_dst_last0 = (u32 *) (©_dst0[5]); + copy_src_last0 = (u32 *) (©_src0[5]); + copy_dst_last0[0] = copy_src_last0[0]; + } + + if (_vec_len (t1->rewrite) == 36) + { + /* Last 4 octets. Hopefully gcc will be our friend */ + copy_dst_last1 = (u32 *) (©_dst1[4]); + copy_src_last1 = (u32 *) (©_src1[4]); + copy_dst_last1[0] = copy_src_last1[0]; + } + else + { + /* Near last 8 octets. */ +#define _(offs) copy_dst1[offs] = copy_src1[offs]; + _ (4); +#undef _ + /* Last 4 octets. Hopefully gcc will be our friend */ + copy_dst_last1 = (u32 *) (©_dst1[5]); + copy_src_last1 = (u32 *) (©_src1[5]); + copy_dst_last1[0] = copy_src_last1[0]; + } + + if (_vec_len (t2->rewrite) == 36) + { + /* Last 4 octets. Hopefully gcc will be our friend */ + copy_dst_last2 = (u32 *) (©_dst2[4]); + copy_src_last2 = (u32 *) (©_src2[4]); + copy_dst_last2[0] = copy_src_last2[0]; + } + else + { + /* Near last 8 octets. */ +#define _(offs) copy_dst2[offs] = copy_src2[offs]; + _ (4); +#undef _ + /* Last 4 octets. Hopefully gcc will be our friend */ + copy_dst_last2 = (u32 *) (©_dst2[5]); + copy_src_last2 = (u32 *) (©_src2[5]); + copy_dst_last2[0] = copy_src_last2[0]; + } + + if (_vec_len (t3->rewrite) == 36) + { + /* Last 4 octets. Hopefully gcc will be our friend */ + copy_dst_last3 = (u32 *) (©_dst3[4]); + copy_src_last3 = (u32 *) (©_src3[4]); + copy_dst_last3[0] = copy_src_last3[0]; + } + else + { + /* Near last 8 octets. */ +#define _(offs) copy_dst3[offs] = copy_src3[offs]; + _ (4); +#undef _ + /* Last 4 octets. Hopefully gcc will be our friend */ + copy_dst_last3 = (u32 *) (©_dst3[5]); + copy_src_last3 = (u32 *) (©_src3[5]); + copy_dst_last3[0] = copy_src_last3[0]; + } /* Fix the IP4 checksum and length */ sum0 = ip4_0->checksum; @@ -318,7 +383,7 @@ gtpu_encap_inline (vlib_main_t * vm, copy_src2 = (u64 *) t2->rewrite; copy_dst3 = (u64 *) ip6_3; copy_src3 = (u64 *) t3->rewrite; - /* Copy first 56 (ip6) octets 8-bytes at a time */ + /* Copy first 56 (ip6) octets 8-bytes at a time (minimum size) */ #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header6_offset; #undef _ @@ -331,6 +396,40 @@ gtpu_encap_inline (vlib_main_t * vm, #define _(offs) copy_dst3[offs] = copy_src3[offs]; foreach_fixed_header6_offset; #undef _ + + /* Copy last octets */ + if (_vec_len (t0->rewrite) == 64) + { + /* Last 8 octets. */ +#define _(offs) copy_dst0[offs] = copy_src0[offs]; + _ (7); +#undef _ + } + + if (_vec_len (t1->rewrite) == 64) + { + /* Last 8 octets. */ +#define _(offs) copy_dst1[offs] = copy_src1[offs]; + _ (7); +#undef _ + } + + if (_vec_len (t2->rewrite) == 64) + { + /* Last 8 octets. */ +#define _(offs) copy_dst2[offs] = copy_src2[offs]; + _ (7); +#undef _ + } + + if (_vec_len (t3->rewrite) == 64) + { + /* Last 8 octets. */ +#define _(offs) copy_dst3[offs] = copy_src3[offs]; + _ (7); +#undef _ + } + /* Fix IP6 payload length */ new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) @@ -413,6 +512,12 @@ gtpu_encap_inline (vlib_main_t * vm, stats_n_packets += 4; stats_n_bytes += len0 + len1 + len2 + len3; + /* save inner packet flow_hash for load-balance node */ + vnet_buffer (b0)->ip.flow_hash = flow_hash0; + vnet_buffer (b1)->ip.flow_hash = flow_hash1; + vnet_buffer (b2)->ip.flow_hash = flow_hash2; + vnet_buffer (b3)->ip.flow_hash = flow_hash3; + /* Batch stats increment on the same gtpu tunnel so counter is not incremented per packet. Note stats are still incremented for deleted and admin-down tunnel where packets are dropped. It is not worthwhile @@ -459,32 +564,40 @@ gtpu_encap_inline (vlib_main_t * vm, gtpu_encap_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->tunnel_index = t0 - gtm->tunnels; - tr->teid = t0->teid; - } + tr->tteid = t0->tteid; + tr->pdu_extension = t0->pdu_extension; + tr->qfi = t0->qfi; + } - if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED)) - { - gtpu_encap_trace_t *tr = - vlib_add_trace (vm, node, b1, sizeof (*tr)); - tr->tunnel_index = t1 - gtm->tunnels; - tr->teid = t1->teid; - } + if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) + { + gtpu_encap_trace_t *tr = + vlib_add_trace (vm, node, b1, sizeof (*tr)); + tr->tunnel_index = t1 - gtm->tunnels; + tr->tteid = t1->tteid; + tr->pdu_extension = t1->pdu_extension; + tr->qfi = t1->qfi; + } if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED)) { gtpu_encap_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); tr->tunnel_index = t2 - gtm->tunnels; - tr->teid = t2->teid; - } + tr->tteid = t2->tteid; + tr->pdu_extension = t2->pdu_extension; + tr->qfi = t2->qfi; + } - if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED)) - { - gtpu_encap_trace_t *tr = - vlib_add_trace (vm, node, b3, sizeof (*tr)); - tr->tunnel_index = t3 - gtm->tunnels; - tr->teid = t3->teid; - } + if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) + { + gtpu_encap_trace_t *tr = + vlib_add_trace (vm, node, b3, sizeof (*tr)); + tr->tunnel_index = t3 - gtm->tunnels; + tr->tteid = t3->tteid; + tr->pdu_extension = t3->pdu_extension; + tr->qfi = t3->qfi; + } vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, n_left_to_next, @@ -526,8 +639,9 @@ gtpu_encap_inline (vlib_main_t * vm, next0 = t0->next_dpo.dpoi_next_node; vnet_buffer(b0)->ip.adj_index[VLIB_TX] = t0->next_dpo.dpoi_index; - /* Apply the rewrite string. $$$$ vnet_rewrite? */ - vlib_buffer_advance (b0, -(word)_vec_len(t0->rewrite)); + /* Apply the rewrite string. $$$$ vnet_rewrite. + * The correct total size is set in ip_udp_gtpu_rewrite() */ + vlib_buffer_advance (b0, -(word) _vec_len (t0->rewrite)); if (is_ip4) { @@ -540,10 +654,26 @@ gtpu_encap_inline (vlib_main_t * vm, #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header4_offset; #undef _ - /* Last 4 octets. Hopefully gcc will be our friend */ - copy_dst_last0 = (u32 *)(©_dst0[4]); - copy_src_last0 = (u32 *)(©_src0[4]); - copy_dst_last0[0] = copy_src_last0[0]; + + /* Copy last octets */ + if (_vec_len (t0->rewrite) == 36) + { + /* Last 4 octets. Hopefully gcc will be our friend */ + copy_dst_last0 = (u32 *) (©_dst0[4]); + copy_src_last0 = (u32 *) (©_src0[4]); + copy_dst_last0[0] = copy_src_last0[0]; + } + else + { + /* Near last 8 octets. */ +#define _(offs) copy_dst0[offs] = copy_src0[offs]; + _ (4); +#undef _ + /* Last 4 octets. Hopefully gcc will be our friend */ + copy_dst_last0 = (u32 *) (©_dst0[5]); + copy_src_last0 = (u32 *) (©_src0[5]); + copy_dst_last0[0] = copy_src_last0[0]; + } /* Fix the IP4 checksum and length */ sum0 = ip4_0->checksum; @@ -581,6 +711,16 @@ gtpu_encap_inline (vlib_main_t * vm, #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header6_offset; #undef _ + + /* Copy last octets */ + if (_vec_len (t0->rewrite) == 64) + { + /* Last 8 octets. */ +#define _(offs) copy_dst0[offs] = copy_src0[offs]; + _ (7); +#undef _ + } + /* Fix IP6 payload length */ new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) @@ -594,9 +734,9 @@ gtpu_encap_inline (vlib_main_t * vm, /* Fix GTPU length */ gtpu0 = (gtpu_header_t *)(udp0+1); - new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0) - - sizeof (*ip4_0) - sizeof(*udp0) - - GTPU_V1_HDR_LEN); + new_l0 = clib_host_to_net_u16 ( + vlib_buffer_length_in_chain (vm, b0) - sizeof (*ip6_0) - + sizeof (*udp0) - GTPU_V1_HDR_LEN); gtpu0->length = new_l0; /* IPv6 UDP checksum is mandatory */ @@ -611,6 +751,9 @@ gtpu_encap_inline (vlib_main_t * vm, stats_n_packets += 1; stats_n_bytes += len0; + /* save inner packet flow_hash for load-balance node */ + vnet_buffer (b0)->ip.flow_hash = flow_hash0; + /* Batch stats increment on the same gtpu tunnel so counter is not incremented per packet. Note stats are still incremented for deleted and admin-down tunnel where packets are dropped. It is not worthwhile @@ -634,8 +777,10 @@ gtpu_encap_inline (vlib_main_t * vm, gtpu_encap_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->tunnel_index = t0 - gtm->tunnels; - tr->teid = t0->teid; - } + tr->tteid = t0->tteid; + tr->pdu_extension = t0->pdu_extension; + tr->qfi = t0->qfi; + } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0);