{
u32 sw_if_index;
u32 flags;
- u16 gso_size;
- u8 gso_l4_hdr_sz;
- u8 data[128 - 3 * sizeof (u32)];
+ u8 data[128 - 2 * sizeof (u32)];
}
interface_output_trace_t;
format (s, "%U ", format_vnet_sw_interface_name, vnm, si,
t->flags);
}
-#define _(bit, name, v, x) \
- if (v && (t->flags & VNET_BUFFER_F_##name)) \
- s = format (s, "%s ", v);
- foreach_vnet_buffer_flag
-#undef _
- if (t->flags & VNET_BUFFER_F_GSO)
- {
- s = format (s, "\n%Ugso_sz %d gso_l4_hdr_sz %d",
- format_white_space, indent + 2, t->gso_size,
- t->gso_l4_hdr_sz);
- }
s =
format (s, "\n%U%U", format_white_space, indent,
node->format_buffer ? node->format_buffer : format_hex_bytes,
t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
t0->flags = b0->flags;
- t0->gso_size = vnet_buffer2 (b0)->gso_size;
- t0->gso_l4_hdr_sz = vnet_buffer2 (b0)->gso_l4_hdr_sz;
clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0),
sizeof (t0->data));
}
t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_TX];
t1->flags = b1->flags;
- t1->gso_size = vnet_buffer2 (b1)->gso_size;
- t1->gso_l4_hdr_sz = vnet_buffer2 (b1)->gso_l4_hdr_sz;
clib_memcpy_fast (t1->data, vlib_buffer_get_current (b1),
sizeof (t1->data));
}
t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
t0->flags = b0->flags;
- t0->gso_size = vnet_buffer2 (b0)->gso_size;
- t0->gso_l4_hdr_sz = vnet_buffer2 (b0)->gso_l4_hdr_sz;
clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0),
sizeof (t0->data));
}
b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
}
-static_always_inline u16
-tso_alloc_tx_bufs (vlib_main_t * vm,
- vnet_interface_per_thread_data_t * ptd,
- vlib_buffer_t * b0, u32 n_bytes_b0, u16 l234_sz,
- u16 gso_size)
-{
- u16 size =
- clib_min (gso_size, vlib_buffer_get_default_data_size (vm) - l234_sz);
-
- /* rounded-up division */
- u16 n_bufs = (n_bytes_b0 - l234_sz + (size - 1)) / size;
- u16 n_alloc;
-
- ASSERT (n_bufs > 0);
- vec_validate (ptd->split_buffers, n_bufs - 1);
-
- n_alloc = vlib_buffer_alloc (vm, ptd->split_buffers, n_bufs);
- if (n_alloc < n_bufs)
- {
- vlib_buffer_free (vm, ptd->split_buffers, n_alloc);
- return 0;
- }
- return n_alloc;
-}
-
-static_always_inline void
-tso_init_buf_from_template_base (vlib_buffer_t * nb0, vlib_buffer_t * b0,
- u32 flags, u16 length)
-{
- nb0->current_data = b0->current_data;
- nb0->total_length_not_including_first_buffer = 0;
- nb0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags;
- clib_memcpy_fast (&nb0->opaque, &b0->opaque, sizeof (nb0->opaque));
- clib_memcpy_fast (vlib_buffer_get_current (nb0),
- vlib_buffer_get_current (b0), length);
- nb0->current_length = length;
-}
-
-static_always_inline void
-tso_init_buf_from_template (vlib_main_t * vm, vlib_buffer_t * nb0,
- vlib_buffer_t * b0, u16 template_data_sz,
- u16 gso_size, u8 ** p_dst_ptr, u16 * p_dst_left,
- u32 next_tcp_seq, u32 flags)
-{
- tso_init_buf_from_template_base (nb0, b0, flags, template_data_sz);
-
- *p_dst_left =
- clib_min (gso_size,
- vlib_buffer_get_default_data_size (vm) - (template_data_sz +
- nb0->current_data));
- *p_dst_ptr = vlib_buffer_get_current (nb0) + template_data_sz;
-
- tcp_header_t *tcp =
- (tcp_header_t *) (nb0->data + vnet_buffer (nb0)->l4_hdr_offset);
- tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq);
-}
-
-static_always_inline void
-tso_fixup_segmented_buf (vlib_buffer_t * b0, u8 tcp_flags, int is_ip6)
-{
- u16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset;
- u16 l4_hdr_offset = vnet_buffer (b0)->l4_hdr_offset;
- ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l3_hdr_offset);
- ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l3_hdr_offset);
- tcp_header_t *tcp = (tcp_header_t *) (b0->data + l4_hdr_offset);
-
- tcp->flags = tcp_flags;
-
- if (is_ip6)
- ip6->payload_length =
- clib_host_to_net_u16 (b0->current_length -
- (l4_hdr_offset - b0->current_data));
- else
- ip4->length =
- clib_host_to_net_u16 (b0->current_length -
- (l3_hdr_offset - b0->current_data));
-}
-
-/**
- * Allocate the necessary number of ptd->split_buffers,
- * and segment the possibly chained buffer(s) from b0 into
- * there.
- *
- * Return the cumulative number of bytes sent or zero
- * if allocation failed.
- */
-
-static_always_inline u32
-tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd,
- int do_tx_offloads, u32 sbi0, vlib_buffer_t * sb0,
- u32 n_bytes_b0)
-{
- u32 n_tx_bytes = 0;
- int is_ip4 = sb0->flags & VNET_BUFFER_F_IS_IP4;
- int is_ip6 = sb0->flags & VNET_BUFFER_F_IS_IP6;
- ASSERT (is_ip4 || is_ip6);
- ASSERT (sb0->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID);
- ASSERT (sb0->flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID);
- ASSERT (sb0->flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
- u16 gso_size = vnet_buffer2 (sb0)->gso_size;
-
- int l4_hdr_sz = vnet_buffer2 (sb0)->gso_l4_hdr_sz;
- u8 save_tcp_flags = 0;
- u8 tcp_flags_no_fin_psh = 0;
- u32 next_tcp_seq = 0;
-
- tcp_header_t *tcp =
- (tcp_header_t *) (sb0->data + vnet_buffer (sb0)->l4_hdr_offset);
- next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number);
- /* store original flags for last packet and reset FIN and PSH */
- save_tcp_flags = tcp->flags;
- tcp_flags_no_fin_psh = tcp->flags & ~(TCP_FLAG_FIN | TCP_FLAG_PSH);
- tcp->checksum = 0;
-
- u32 default_bflags =
- sb0->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT);
- u16 l234_sz = vnet_buffer (sb0)->l4_hdr_offset + l4_hdr_sz
- - sb0->current_data;
- int first_data_size = clib_min (gso_size, sb0->current_length - l234_sz);
- next_tcp_seq += first_data_size;
-
- if (PREDICT_FALSE
- (!tso_alloc_tx_bufs (vm, ptd, sb0, n_bytes_b0, l234_sz, gso_size)))
- return 0;
-
- vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[0]);
- tso_init_buf_from_template_base (b0, sb0, default_bflags,
- l234_sz + first_data_size);
-
- u32 total_src_left = n_bytes_b0 - l234_sz - first_data_size;
- if (total_src_left)
- {
- /* Need to copy more segments */
- u8 *src_ptr, *dst_ptr;
- u16 src_left, dst_left;
- /* current source buffer */
- vlib_buffer_t *csb0 = sb0;
- u32 csbi0 = sbi0;
- /* current dest buffer */
- vlib_buffer_t *cdb0;
- u16 dbi = 1; /* the buffer [0] is b0 */
-
- src_ptr = vlib_buffer_get_current (sb0) + l234_sz + first_data_size;
- src_left = sb0->current_length - l234_sz - first_data_size;
-
- tso_fixup_segmented_buf (b0, tcp_flags_no_fin_psh, is_ip6);
- if (do_tx_offloads)
- calc_checksums (vm, b0);
-
- /* grab a second buffer and prepare the loop */
- ASSERT (dbi < vec_len (ptd->split_buffers));
- cdb0 = vlib_get_buffer (vm, ptd->split_buffers[dbi++]);
- tso_init_buf_from_template (vm, cdb0, b0, l234_sz, gso_size, &dst_ptr,
- &dst_left, next_tcp_seq, default_bflags);
-
- /* an arbitrary large number to catch the runaway loops */
- int nloops = 2000;
- while (total_src_left)
- {
- if (nloops-- <= 0)
- clib_panic ("infinite loop detected");
- u16 bytes_to_copy = clib_min (src_left, dst_left);
-
- clib_memcpy_fast (dst_ptr, src_ptr, bytes_to_copy);
-
- src_left -= bytes_to_copy;
- src_ptr += bytes_to_copy;
- total_src_left -= bytes_to_copy;
- dst_left -= bytes_to_copy;
- dst_ptr += bytes_to_copy;
- next_tcp_seq += bytes_to_copy;
- cdb0->current_length += bytes_to_copy;
-
- if (0 == src_left)
- {
- int has_next = (csb0->flags & VLIB_BUFFER_NEXT_PRESENT);
- u32 next_bi = csb0->next_buffer;
-
- /* init src to the next buffer in chain */
- if (has_next)
- {
- csbi0 = next_bi;
- csb0 = vlib_get_buffer (vm, csbi0);
- src_left = csb0->current_length;
- src_ptr = vlib_buffer_get_current (csb0);
- }
- else
- {
- ASSERT (total_src_left == 0);
- break;
- }
- }
- if (0 == dst_left && total_src_left)
- {
- if (do_tx_offloads)
- calc_checksums (vm, cdb0);
- n_tx_bytes += cdb0->current_length;
- ASSERT (dbi < vec_len (ptd->split_buffers));
- cdb0 = vlib_get_buffer (vm, ptd->split_buffers[dbi++]);
- tso_init_buf_from_template (vm, cdb0, b0, l234_sz,
- gso_size, &dst_ptr, &dst_left,
- next_tcp_seq, default_bflags);
- }
- }
-
- tso_fixup_segmented_buf (cdb0, save_tcp_flags, is_ip6);
- if (do_tx_offloads)
- calc_checksums (vm, cdb0);
-
- n_tx_bytes += cdb0->current_length;
- }
- n_tx_bytes += b0->current_length;
- return n_tx_bytes;
-}
-
-static_always_inline void
-drop_one_buffer_and_count (vlib_main_t * vm, vnet_main_t * vnm,
- vlib_node_runtime_t * node, u32 * pbi0,
- u32 drop_error_code)
-{
- u32 thread_index = vm->thread_index;
- vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
-
- vlib_simple_counter_main_t *cm;
- cm =
- vec_elt_at_index (vnm->interface_main.sw_if_counters,
- VNET_INTERFACE_COUNTER_TX_ERROR);
- vlib_increment_simple_counter (cm, thread_index, rt->sw_if_index, 1);
-
- vlib_error_drop_buffers (vm, node, pbi0,
- /* buffer stride */ 1,
- /* n_buffers */ 1,
- VNET_INTERFACE_OUTPUT_NEXT_DROP,
- node->node_index, drop_error_code);
-}
-
static_always_inline uword
-vnet_interface_output_node_inline_gso (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame,
- vnet_main_t * vnm,
- vnet_hw_interface_t * hi,
- int do_tx_offloads,
- int do_segmentation)
+vnet_interface_output_node_inline (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame,
+ vnet_main_t * vnm,
+ vnet_hw_interface_t * hi,
+ int do_tx_offloads)
{
vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
vnet_sw_interface_t *si;
u32 next_index = VNET_INTERFACE_OUTPUT_NEXT_TX;
u32 current_config_index = ~0;
u8 arc = im->output_feature_arc_index;
- vnet_interface_per_thread_data_t *ptd =
- vec_elt_at_index (im->per_thread_data, thread_index);
vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
n_buffers = frame->n_vectors;
or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
- if (do_segmentation)
- {
- /* go to single loop if we need TSO segmentation */
- if (PREDICT_FALSE (or_flags & VNET_BUFFER_F_GSO))
- break;
- }
from += 4;
to_tx += 4;
n_left_to_tx -= 4;
b[0]->current_config_index = current_config_index;
}
- if (do_segmentation)
- {
- if (PREDICT_FALSE (b[0]->flags & VNET_BUFFER_F_GSO))
- {
- /*
- * Undo the enqueue of the b0 - it is not going anywhere,
- * and will be freed either after it's segmented or
- * when dropped, if there is no buffers to segment into.
- */
- to_tx -= 1;
- n_left_to_tx += 1;
- /* undo the counting. */
- n_bytes -= n_bytes_b0;
- n_packets -= 1;
-
- u32 n_tx_bytes = 0;
-
- n_tx_bytes =
- tso_segment_buffer (vm, ptd, do_tx_offloads, bi0, b[0],
- n_bytes_b0);
-
- if (PREDICT_FALSE (n_tx_bytes == 0))
- {
- drop_one_buffer_and_count (vm, vnm, node, from - 1,
- VNET_INTERFACE_OUTPUT_ERROR_NO_BUFFERS_FOR_GSO);
- b += 1;
- continue;
- }
-
- u16 n_tx_bufs = vec_len (ptd->split_buffers);
- u32 *from_tx_seg = ptd->split_buffers;
-
- while (n_tx_bufs > 0)
- {
- if (n_tx_bufs >= n_left_to_tx)
- {
- while (n_left_to_tx > 0)
- {
- to_tx[0] = from_tx_seg[0];
- to_tx += 1;
- from_tx_seg += 1;
- n_left_to_tx -= 1;
- n_tx_bufs -= 1;
- n_packets += 1;
- }
- vlib_put_next_frame (vm, node, next_index,
- n_left_to_tx);
- vlib_get_new_next_frame (vm, node, next_index,
- to_tx, n_left_to_tx);
- }
- while (n_tx_bufs > 0)
- {
- to_tx[0] = from_tx_seg[0];
- to_tx += 1;
- from_tx_seg += 1;
- n_left_to_tx -= 1;
- n_tx_bufs -= 1;
- n_packets += 1;
- }
- }
- n_bytes += n_tx_bytes;
- if (PREDICT_FALSE (tx_swif0 != rt->sw_if_index))
- {
-
- vlib_increment_combined_counter
- (im->combined_sw_if_counters +
- VNET_INTERFACE_COUNTER_TX, thread_index, tx_swif0,
- _vec_len (ptd->split_buffers), n_tx_bytes);
- }
- /* The buffers were enqueued. Reset the length */
- _vec_len (ptd->split_buffers) = 0;
- /* Free the now segmented buffer */
- vlib_buffer_free_one (vm, bi0);
- b += 1;
- continue;
- }
- }
-
if (PREDICT_FALSE (tx_swif0 != rt->sw_if_index))
{
}
if (do_tx_offloads)
- calc_checksums (vm, b[0]);
-
+ {
+ if (b[0]->flags &
+ (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
+ VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
+ VNET_BUFFER_F_OFFLOAD_IP_CKSUM))
+ calc_checksums (vm, b[0]);
+ }
b += 1;
}
}
#ifndef CLIB_MARCH_VARIANT
-static_always_inline uword
-vnet_interface_output_node_inline (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame, vnet_main_t * vnm,
- vnet_hw_interface_t * hi,
- int do_tx_offloads)
-{
- /*
- * The 3-headed "if" is here because we want to err on the side
- * of not impacting the non-GSO performance - so for the more
- * common case of no GSO interfaces we want to prevent the
- * segmentation codepath from being there altogether.
- */
- if (PREDICT_TRUE (vnm->interface_main.gso_interface_count == 0))
- return vnet_interface_output_node_inline_gso (vm, node, frame, vnm, hi,
- do_tx_offloads,
- /* do_segmentation */ 0);
- else if (hi->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO)
- return vnet_interface_output_node_inline_gso (vm, node, frame, vnm, hi,
- do_tx_offloads,
- /* do_segmentation */ 0);
- else
- return vnet_interface_output_node_inline_gso (vm, node, frame, vnm, hi,
- do_tx_offloads,
- /* do_segmentation */ 1);
-}
uword
vnet_interface_output_node (vlib_main_t * vm, vlib_node_runtime_t * node,
Code Review / vpp.git / blobdiff