if (is_ip4)
ip4->checksum = 0;
+ if (is_tso)
+ {
+ if (is_ip4)
+ ip4->length = 0;
+ else
+ ip6->payload_length = 0;
+ }
+
if (is_tcp || is_udp)
{
if (is_ip4)
return flags;
}
+static_always_inline int
+avf_tx_fill_ctx_desc (vlib_main_t * vm, avf_txq_t * txq, avf_tx_desc_t * d,
+ vlib_buffer_t * b)
+{
+ vlib_buffer_t *ctx_ph = vlib_get_buffer (vm, txq->ctx_desc_placeholder_bi);
+ if (PREDICT_FALSE (ctx_ph->ref_count == 255))
+ {
+ /* We need a new placeholder buffer */
+ u32 new_bi;
+ u8 bpi = vlib_buffer_pool_get_default_for_numa (vm, vm->numa_node);
+ if (PREDICT_TRUE
+ (vlib_buffer_alloc_from_pool (vm, &new_bi, 1, bpi) == 1))
+ {
+ /* Remove our own reference on the current placeholder buffer */
+ ctx_ph->ref_count--;
+ /* Replace with the new placeholder buffer */
+ txq->ctx_desc_placeholder_bi = new_bi;
+ ctx_ph = vlib_get_buffer (vm, new_bi);
+ }
+ else
+ /* Impossible to enqueue a ctx descriptor, fail */
+ return 1;
+ }
+
+ /* Acquire a reference on the placeholder buffer */
+ ctx_ph->ref_count++;
+
+ u16 l234hdr_sz =
+ vnet_buffer (b)->l4_hdr_offset -
+ vnet_buffer (b)->l2_hdr_offset + vnet_buffer2 (b)->gso_l4_hdr_sz;
+ u16 tlen = vlib_buffer_length_in_chain (vm, b) - l234hdr_sz;
+ d[0].qword[0] = 0;
+ d[0].qword[1] = AVF_TXD_DTYP_CTX | AVF_TXD_CTX_CMD_TSO
+ | AVF_TXD_CTX_SEG_MSS (vnet_buffer2 (b)->gso_size) |
+ AVF_TXD_CTX_SEG_TLEN (tlen);
+ return 0;
+}
+
+
static_always_inline u16
avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
u32 * buffers, u32 n_packets, int use_va_dma)
{
u16 next = txq->next;
u64 bits = AVF_TXD_CMD_EOP | AVF_TXD_CMD_RSV;
+ const u32 offload_mask = VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
+ VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
+ VNET_BUFFER_F_GSO;
u64 one_by_one_offload_flags = 0;
+ int is_tso;
u16 n_desc = 0;
u16 *slot, n_desc_left, n_packets_left = n_packets;
u16 mask = txq->size - 1;
or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
- if (or_flags &
- (VLIB_BUFFER_NEXT_PRESENT | VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ if (or_flags & (VLIB_BUFFER_NEXT_PRESENT | offload_mask))
goto one_by_one;
vlib_buffer_copy_indices (txq->bufs + next, buffers, 4);
one_by_one_offload_flags = 0;
txq->bufs[next] = buffers[0];
b[0] = vlib_get_buffer (vm, buffers[0]);
-
- if (PREDICT_FALSE (b[0]->flags & (VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)))
- one_by_one_offload_flags |=
- avf_tx_prepare_cksum (b[0], 0 /* no TSO */ );
+ is_tso = ! !(b[0]->flags & VNET_BUFFER_F_GSO);
+ if (PREDICT_FALSE (is_tso || b[0]->flags & offload_mask))
+ one_by_one_offload_flags |= avf_tx_prepare_cksum (b[0], is_tso);
/* Deal with chain buffer if present */
- if (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
+ if (is_tso || b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
{
- n_desc_needed = 1;
+ n_desc_needed = 1 + is_tso;
b0 = b[0];
/* Wish there were a buffer count for chain buffer */
}
/* spec says data descriptor is limited to 8 segments */
- if (PREDICT_FALSE (n_desc_needed > 8))
+ if (PREDICT_FALSE (!is_tso && n_desc_needed > 8))
{
vlib_buffer_free_one (vm, buffers[0]);
vlib_error_count (vm, node->node_index,
*/
break;
+ /* Enqueue a context descriptor if needed */
+ if (PREDICT_FALSE (is_tso))
+ {
+ if (avf_tx_fill_ctx_desc (vm, txq, d, b[0]))
+ /* Failure to acquire ref on ctx placeholder */
+ break;
+ txq->bufs[next + 1] = txq->bufs[next];
+ txq->bufs[next] = txq->ctx_desc_placeholder_bi;
+ next += 1;
+ n_desc += 1;
+ n_desc_left -= 1;
+ d += 1;
+ }
while (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
{
if (use_va_dma)
b[0] = vlib_get_buffer (vm, buffers[0]);
one_by_one_offload_flags = 0;
- if (PREDICT_FALSE (b[0]->flags & (VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)))
- one_by_one_offload_flags |=
- avf_tx_prepare_cksum (b[0], 0 /* no TSO */ );
+ is_tso = ! !(b[0]->flags & VNET_BUFFER_F_GSO);
+ if (PREDICT_FALSE (is_tso || b[0]->flags & offload_mask))
+ one_by_one_offload_flags |= avf_tx_prepare_cksum (b[0], is_tso);
/* Deal with chain buffer if present */
- if (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
+ if (is_tso || b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
{
- n_desc_needed = 1;
+ n_desc_needed = 1 + is_tso;
b0 = b[0];
while (b0->flags & VLIB_BUFFER_NEXT_PRESENT)
}
/* Spec says data descriptor is limited to 8 segments */
- if (PREDICT_FALSE (n_desc_needed > 8))
+ if (PREDICT_FALSE (!is_tso && n_desc_needed > 8))
{
vlib_buffer_free_one (vm, buffers[0]);
vlib_error_count (vm, node->node_index,
if (PREDICT_FALSE (n_desc_left < n_desc_needed))
break;
+ /* Enqueue a context descriptor if needed */
+ if (PREDICT_FALSE (is_tso))
+ {
+ if (avf_tx_fill_ctx_desc (vm, txq, d, b[0]))
+ /* Failure to acquire ref on ctx placeholder */
+ break;
+
+ txq->bufs[(next + 1) & mask] = txq->bufs[next & mask];
+ txq->bufs[next & mask] = txq->ctx_desc_placeholder_bi;
+ next += 1;
+ n_desc += 1;
+ n_desc_left -= 1;
+ d = txq->descs + (next & mask);
+ }
while (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
{
if (use_va_dma)
Code Review / vpp.git / blobdiff