+static_always_inline u16
+tso_alloc_tx_bufs (vlib_main_t * vm,
+ vnet_interface_per_thread_data_t * ptd,
+ vlib_buffer_t * b0, u16 l4_hdr_sz)
+{
+ u32 n_bytes_b0 = vlib_buffer_length_in_chain (vm, b0);
+ u16 gso_size = vnet_buffer2 (b0)->gso_size;
+ u16 l234_sz = vnet_buffer (b0)->l4_hdr_offset + l4_hdr_sz;
+ /* rounded-up division */
+ u16 n_bufs = (n_bytes_b0 - l234_sz + (gso_size - 1)) / gso_size;
+ u16 n_alloc;
+
+ ASSERT (n_bufs > 0);
+ vec_validate (ptd->split_buffers, n_bufs - 1);
+
+ n_alloc = vlib_buffer_alloc (vm, ptd->split_buffers, n_bufs);
+ if (n_alloc < n_bufs)
+ {
+ vlib_buffer_free (vm, ptd->split_buffers, n_alloc);
+ return 0;
+ }
+ return 1;
+}
+
+static_always_inline void
+tso_init_buf_from_template_base (vlib_buffer_t * nb0, vlib_buffer_t * b0,
+ u32 flags, u16 length)
+{
+ nb0->current_data = 0;
+ nb0->total_length_not_including_first_buffer = 0;
+ nb0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags;
+ clib_memcpy_fast (&nb0->opaque, &b0->opaque, sizeof (nb0->opaque));
+ clib_memcpy_fast (nb0->data, b0->data, length);
+ nb0->current_length = length;
+}
+
+static_always_inline void
+tso_init_buf_from_template (vlib_main_t * vm, vlib_buffer_t * nb0,
+ vlib_buffer_t * b0, u16 template_data_sz,
+ u16 gso_size, u8 ** p_dst_ptr, u16 * p_dst_left,
+ u32 next_tcp_seq, u32 flags)
+{
+ tso_init_buf_from_template_base (nb0, b0, flags, template_data_sz);
+
+ *p_dst_left =
+ clib_min (gso_size,
+ vlib_buffer_get_default_data_size (vm) - template_data_sz);
+ *p_dst_ptr = nb0->data + template_data_sz;
+
+ tcp_header_t *tcp =
+ (tcp_header_t *) (nb0->data + vnet_buffer (nb0)->l4_hdr_offset);
+ tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq);
+}
+
+static_always_inline void
+tso_fixup_segmented_buf (vlib_buffer_t * b0, u8 tcp_flags, int is_ip6)
+{
+ u16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset;
+ u16 l4_hdr_offset = vnet_buffer (b0)->l4_hdr_offset;
+ ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l3_hdr_offset);
+ ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l3_hdr_offset);
+ tcp_header_t *tcp = (tcp_header_t *) (b0->data + l4_hdr_offset);
+
+ tcp->flags = tcp_flags;
+
+ if (is_ip6)
+ ip6->payload_length =
+ clib_host_to_net_u16 (b0->current_length -
+ vnet_buffer (b0)->l4_hdr_offset);
+ else
+ ip4->length =
+ clib_host_to_net_u16 (b0->current_length -
+ vnet_buffer (b0)->l3_hdr_offset);
+}
+
+/**
+ * Allocate the necessary number of ptd->split_buffers,
+ * and segment the possibly chained buffer(s) from b0 into
+ * there.
+ *
+ * Return the cumulative number of bytes sent or zero
+ * if allocation failed.
+ */
+
+static_always_inline u32
+tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd,
+ int do_tx_offloads, u32 sbi0, vlib_buffer_t * sb0,
+ u32 n_bytes_b0)
+{
+ u32 n_tx_bytes = 0;
+ int is_ip4 = sb0->flags & VNET_BUFFER_F_IS_IP4;
+ int is_ip6 = sb0->flags & VNET_BUFFER_F_IS_IP6;
+ ASSERT (is_ip4 || is_ip6);
+ ASSERT (sb0->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID);
+ ASSERT (sb0->flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID);
+ ASSERT (sb0->flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
+ u16 gso_size = vnet_buffer2 (sb0)->gso_size;
+
+ int l4_hdr_sz = vnet_buffer2 (sb0)->gso_l4_hdr_sz;
+ u8 save_tcp_flags = 0;
+ u8 tcp_flags_no_fin_psh = 0;
+ u32 next_tcp_seq = 0;
+
+ tcp_header_t *tcp =
+ (tcp_header_t *) (sb0->data + vnet_buffer (sb0)->l4_hdr_offset);
+ next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number);
+ /* store original flags for last packet and reset FIN and PSH */
+ save_tcp_flags = tcp->flags;
+ tcp_flags_no_fin_psh = tcp->flags & ~(TCP_FLAG_FIN | TCP_FLAG_PSH);
+ tcp->checksum = 0;
+
+ u32 default_bflags =
+ sb0->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT);
+ u16 l234_sz = vnet_buffer (sb0)->l4_hdr_offset + l4_hdr_sz;
+ int first_data_size = clib_min (gso_size, sb0->current_length - l234_sz);
+ next_tcp_seq += first_data_size;
+
+ if (PREDICT_FALSE (!tso_alloc_tx_bufs (vm, ptd, sb0, l4_hdr_sz)))
+ return 0;
+
+ vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[0]);
+ tso_init_buf_from_template_base (b0, sb0, default_bflags,
+ l234_sz + first_data_size);
+
+ u32 total_src_left = n_bytes_b0 - l234_sz - first_data_size;
+ if (total_src_left)
+ {
+ /* Need to copy more segments */
+ u8 *src_ptr, *dst_ptr;
+ u16 src_left, dst_left;
+ /* current source buffer */
+ vlib_buffer_t *csb0 = sb0;
+ u32 csbi0 = sbi0;
+ /* current dest buffer */
+ vlib_buffer_t *cdb0;
+ u16 dbi = 1; /* the buffer [0] is b0 */
+
+ src_ptr = sb0->data + l234_sz + first_data_size;
+ src_left = sb0->current_length - l234_sz - first_data_size;
+ b0->current_length = l234_sz + first_data_size;
+
+ tso_fixup_segmented_buf (b0, tcp_flags_no_fin_psh, is_ip6);
+ if (do_tx_offloads)
+ calc_checksums (vm, b0);
+
+ /* grab a second buffer and prepare the loop */
+ ASSERT (dbi < vec_len (ptd->split_buffers));
+ cdb0 = vlib_get_buffer (vm, ptd->split_buffers[dbi++]);
+ tso_init_buf_from_template (vm, cdb0, b0, l234_sz, gso_size, &dst_ptr,
+ &dst_left, next_tcp_seq, default_bflags);
+
+ /* an arbitrary large number to catch the runaway loops */
+ int nloops = 2000;
+ while (total_src_left)
+ {
+ if (nloops-- <= 0)
+ clib_panic ("infinite loop detected");
+ u16 bytes_to_copy = clib_min (src_left, dst_left);
+
+ clib_memcpy_fast (dst_ptr, src_ptr, bytes_to_copy);
+
+ src_left -= bytes_to_copy;
+ src_ptr += bytes_to_copy;
+ total_src_left -= bytes_to_copy;
+ dst_left -= bytes_to_copy;
+ dst_ptr += bytes_to_copy;
+ next_tcp_seq += bytes_to_copy;
+ cdb0->current_length += bytes_to_copy;
+
+ if (0 == src_left)
+ {
+ int has_next = (csb0->flags & VLIB_BUFFER_NEXT_PRESENT);
+ u32 next_bi = csb0->next_buffer;
+
+ /* init src to the next buffer in chain */
+ if (has_next)
+ {
+ csbi0 = next_bi;
+ csb0 = vlib_get_buffer (vm, csbi0);
+ src_left = csb0->current_length;
+ src_ptr = vlib_buffer_get_current (csb0);
+ }
+ else
+ {
+ ASSERT (total_src_left == 0);
+ break;
+ }
+ }
+ if (0 == dst_left && total_src_left)
+ {
+ if (do_tx_offloads)
+ calc_checksums (vm, cdb0);
+ n_tx_bytes += cdb0->current_length;
+ ASSERT (dbi < vec_len (ptd->split_buffers));
+ cdb0 = vlib_get_buffer (vm, ptd->split_buffers[dbi++]);
+ tso_init_buf_from_template (vm, cdb0, b0, l234_sz,
+ gso_size, &dst_ptr, &dst_left,
+ next_tcp_seq, default_bflags);
+ }
+ }
+
+ tso_fixup_segmented_buf (cdb0, save_tcp_flags, is_ip6);
+ if (do_tx_offloads)
+ calc_checksums (vm, cdb0);
+
+ n_tx_bytes += cdb0->current_length;
+ }
+ n_tx_bytes += b0->current_length;
+ return n_tx_bytes;
+}
+
+static_always_inline void
+drop_one_buffer_and_count (vlib_main_t * vm, vnet_main_t * vnm,
+ vlib_node_runtime_t * node, u32 * pbi0,
+ u32 drop_error_code)
+{
+ u32 thread_index = vm->thread_index;
+ vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
+
+ vlib_simple_counter_main_t *cm;
+ cm =
+ vec_elt_at_index (vnm->interface_main.sw_if_counters,
+ VNET_INTERFACE_COUNTER_TX_ERROR);
+ vlib_increment_simple_counter (cm, thread_index, rt->sw_if_index, 1);
+
+ vlib_error_drop_buffers (vm, node, pbi0,
+ /* buffer stride */ 1,
+ /* n_buffers */ 1,
+ VNET_INTERFACE_OUTPUT_NEXT_DROP,
+ node->node_index, drop_error_code);
+}
+