-}
-
-static_always_inline u16
-tso_alloc_tx_bufs (vlib_main_t * vm,
- vnet_interface_per_thread_data_t * ptd,
- vlib_buffer_t * b0, u16 l4_hdr_sz)
-{
- u32 n_bytes_b0 = vlib_buffer_length_in_chain (vm, b0);
- u16 gso_size = vnet_buffer2 (b0)->gso_size;
- u16 l234_sz = vnet_buffer (b0)->l4_hdr_offset + l4_hdr_sz;
- /* rounded-up division */
- u16 n_bufs = (n_bytes_b0 - l234_sz + (gso_size - 1)) / gso_size;
- u16 n_alloc;
-
- ASSERT (n_bufs > 0);
- vec_validate (ptd->split_buffers, n_bufs - 1);
-
- n_alloc = vlib_buffer_alloc (vm, ptd->split_buffers, n_bufs);
- if (n_alloc < n_bufs)
- {
- vlib_buffer_free (vm, ptd->split_buffers, n_alloc);
- return 0;
- }
- return 1;
-}
-
-static_always_inline void
-tso_init_buf_from_template_base (vlib_buffer_t * nb0, vlib_buffer_t * b0,
- u32 flags, u16 length)
-{
- nb0->current_data = 0;
- nb0->total_length_not_including_first_buffer = 0;
- nb0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags;
- clib_memcpy_fast (&nb0->opaque, &b0->opaque, sizeof (nb0->opaque));
- clib_memcpy_fast (nb0->data, b0->data, length);
- nb0->current_length = length;
-}
-
-static_always_inline void
-tso_init_buf_from_template (vlib_main_t * vm, vlib_buffer_t * nb0,
- vlib_buffer_t * b0, u16 template_data_sz,
- u16 gso_size, u8 ** p_dst_ptr, u16 * p_dst_left,
- u32 next_tcp_seq, u32 flags)
-{
- tso_init_buf_from_template_base (nb0, b0, flags, template_data_sz);
-
- *p_dst_left =
- clib_min (gso_size,
- vlib_buffer_get_default_data_size (vm) - template_data_sz);
- *p_dst_ptr = nb0->data + template_data_sz;
-
- tcp_header_t *tcp =
- (tcp_header_t *) (nb0->data + vnet_buffer (nb0)->l4_hdr_offset);
- tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq);
-}
-
-static_always_inline void
-tso_fixup_segmented_buf (vlib_buffer_t * b0, u8 tcp_flags, int is_ip6)
-{
- u16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset;
- u16 l4_hdr_offset = vnet_buffer (b0)->l4_hdr_offset;
- ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l3_hdr_offset);
- ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l3_hdr_offset);
- tcp_header_t *tcp = (tcp_header_t *) (b0->data + l4_hdr_offset);
-
- tcp->flags = tcp_flags;
-
- if (is_ip6)
- ip6->payload_length =
- clib_host_to_net_u16 (b0->current_length -
- vnet_buffer (b0)->l4_hdr_offset);
- else
- ip4->length =
- clib_host_to_net_u16 (b0->current_length -
- vnet_buffer (b0)->l3_hdr_offset);
-}
-
-/**
- * Allocate the necessary number of ptd->split_buffers,
- * and segment the possibly chained buffer(s) from b0 into
- * there.
- *
- * Return the cumulative number of bytes sent or zero
- * if allocation failed.
- */
-
-static_always_inline u32
-tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd,
- int do_tx_offloads, u32 sbi0, vlib_buffer_t * sb0,
- u32 n_bytes_b0)
-{
- u32 n_tx_bytes = 0;
- int is_ip4 = sb0->flags & VNET_BUFFER_F_IS_IP4;
- int is_ip6 = sb0->flags & VNET_BUFFER_F_IS_IP6;
- ASSERT (is_ip4 || is_ip6);
- ASSERT (sb0->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID);
- ASSERT (sb0->flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID);
- ASSERT (sb0->flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
- u16 gso_size = vnet_buffer2 (sb0)->gso_size;
-
- int l4_hdr_sz = vnet_buffer2 (sb0)->gso_l4_hdr_sz;
- u8 save_tcp_flags = 0;
- u8 tcp_flags_no_fin_psh = 0;
- u32 next_tcp_seq = 0;
-
- tcp_header_t *tcp =
- (tcp_header_t *) (sb0->data + vnet_buffer (sb0)->l4_hdr_offset);
- next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number);
- /* store original flags for last packet and reset FIN and PSH */
- save_tcp_flags = tcp->flags;
- tcp_flags_no_fin_psh = tcp->flags & ~(TCP_FLAG_FIN | TCP_FLAG_PSH);
- tcp->checksum = 0;
-
- u32 default_bflags =
- sb0->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT);
- u16 l234_sz = vnet_buffer (sb0)->l4_hdr_offset + l4_hdr_sz;
- int first_data_size = clib_min (gso_size, sb0->current_length - l234_sz);
- next_tcp_seq += first_data_size;
-
- if (PREDICT_FALSE (!tso_alloc_tx_bufs (vm, ptd, sb0, l4_hdr_sz)))
- return 0;
-
- vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[0]);
- tso_init_buf_from_template_base (b0, sb0, default_bflags,
- l234_sz + first_data_size);
-
- u32 total_src_left = n_bytes_b0 - l234_sz - first_data_size;
- if (total_src_left)
- {
- /* Need to copy more segments */
- u8 *src_ptr, *dst_ptr;
- u16 src_left, dst_left;
- /* current source buffer */
- vlib_buffer_t *csb0 = sb0;
- u32 csbi0 = sbi0;
- /* current dest buffer */
- vlib_buffer_t *cdb0;
- u16 dbi = 1; /* the buffer [0] is b0 */
-
- src_ptr = sb0->data + l234_sz + first_data_size;
- src_left = sb0->current_length - l234_sz - first_data_size;
- b0->current_length = l234_sz + first_data_size;
-
- tso_fixup_segmented_buf (b0, tcp_flags_no_fin_psh, is_ip6);
- if (do_tx_offloads)
- calc_checksums (vm, b0);
-
- /* grab a second buffer and prepare the loop */
- ASSERT (dbi < vec_len (ptd->split_buffers));
- cdb0 = vlib_get_buffer (vm, ptd->split_buffers[dbi++]);
- tso_init_buf_from_template (vm, cdb0, b0, l234_sz, gso_size, &dst_ptr,
- &dst_left, next_tcp_seq, default_bflags);
-
- /* an arbitrary large number to catch the runaway loops */
- int nloops = 2000;
- while (total_src_left)
- {
- if (nloops-- <= 0)
- clib_panic ("infinite loop detected");
- u16 bytes_to_copy = clib_min (src_left, dst_left);
-
- clib_memcpy_fast (dst_ptr, src_ptr, bytes_to_copy);
-
- src_left -= bytes_to_copy;
- src_ptr += bytes_to_copy;
- total_src_left -= bytes_to_copy;
- dst_left -= bytes_to_copy;
- dst_ptr += bytes_to_copy;
- next_tcp_seq += bytes_to_copy;
- cdb0->current_length += bytes_to_copy;
-
- if (0 == src_left)
- {
- int has_next = (csb0->flags & VLIB_BUFFER_NEXT_PRESENT);
- u32 next_bi = csb0->next_buffer;
-
- /* init src to the next buffer in chain */
- if (has_next)
- {
- csbi0 = next_bi;
- csb0 = vlib_get_buffer (vm, csbi0);
- src_left = csb0->current_length;
- src_ptr = vlib_buffer_get_current (csb0);
- }
- else
- {
- ASSERT (total_src_left == 0);
- break;
- }
- }
- if (0 == dst_left && total_src_left)
- {
- if (do_tx_offloads)
- calc_checksums (vm, cdb0);
- n_tx_bytes += cdb0->current_length;
- ASSERT (dbi < vec_len (ptd->split_buffers));
- cdb0 = vlib_get_buffer (vm, ptd->split_buffers[dbi++]);
- tso_init_buf_from_template (vm, cdb0, b0, l234_sz,
- gso_size, &dst_ptr, &dst_left,
- next_tcp_seq, default_bflags);
- }
- }
-
- tso_fixup_segmented_buf (cdb0, save_tcp_flags, is_ip6);
- if (do_tx_offloads)
- calc_checksums (vm, cdb0);
-
- n_tx_bytes += cdb0->current_length;
- }
- n_tx_bytes += b0->current_length;
- return n_tx_bytes;
-}
-
-static_always_inline void
-drop_one_buffer_and_count (vlib_main_t * vm, vnet_main_t * vnm,
- vlib_node_runtime_t * node, u32 * pbi0,
- u32 drop_error_code)
-{
- u32 thread_index = vm->thread_index;
- vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
-
- vlib_simple_counter_main_t *cm;
- cm =
- vec_elt_at_index (vnm->interface_main.sw_if_counters,
- VNET_INTERFACE_COUNTER_TX_ERROR);
- vlib_increment_simple_counter (cm, thread_index, rt->sw_if_index, 1);
-
- vlib_error_drop_buffers (vm, node, pbi0,
- /* buffer stride */ 1,
- /* n_buffers */ 1,
- VNET_INTERFACE_OUTPUT_NEXT_DROP,
- node->node_index, drop_error_code);