1 /* SPDX-License-Identifier: Apache-2.0
2 * Copyright (c) 2023 Cisco Systems, Inc.
6 #include <vppinfra/ring.h>
7 #include <vppinfra/vector/ip_csum.h>
9 #include <vnet/dev/dev.h>
10 #include <vnet/ethernet/ethernet.h>
11 #include <vnet/ip/ip4_packet.h>
12 #include <vnet/ip/ip6_packet.h>
13 #include <vnet/udp/udp_packet.h>
14 #include <vnet/tcp/tcp_packet.h>
16 #include <dev_octeon/octeon.h>
20 union nix_send_hdr_w0_u hdr_w0_teplate;
21 vlib_node_runtime_t *node;
24 vlib_buffer_t *drop[VLIB_FRAME_SIZE];
25 u32 batch_alloc_not_ready;
26 u32 batch_alloc_issue_fail;
29 lmt_line_t *lmt_lines;
32 static_always_inline u32
33 oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq)
35 oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq);
39 oct_npa_batch_alloc_cl128_t *cl;
41 num_cl = ctq->ba_num_cl;
44 u16 off = ctq->hdr_off;
45 u32 *bi = (u32 *) ctq->ba_buffer;
47 for (cl = ctq->ba_buffer + ctq->ba_first_cl; num_cl > 0; num_cl--, cl++)
49 oct_npa_batch_alloc_status_t st;
51 if ((st.as_u64 = __atomic_load_n (cl->iova, __ATOMIC_RELAXED)) ==
52 OCT_BATCH_ALLOC_IOVA0_MASK + ALLOC_CCODE_INVAL)
55 ctx->batch_alloc_not_ready++;
56 n_freed = bi - (u32 *) ctq->ba_buffer;
59 vlib_buffer_free_no_next (vm, (u32 *) ctq->ba_buffer,
61 ctq->ba_num_cl = num_cl;
62 ctq->ba_first_cl = cl - ctq->ba_buffer;
69 if (st.status.count > 8 &&
70 __atomic_load_n (cl->iova + 8, __ATOMIC_RELAXED) ==
71 OCT_BATCH_ALLOC_IOVA0_MASK)
75 cl->iova[0] &= OCT_BATCH_ALLOC_IOVA0_MASK;
77 if (PREDICT_TRUE (st.status.count == 16))
79 /* optimize for likely case where cacheline is full */
80 vlib_get_buffer_indices_with_offset (vm, (void **) cl, bi, 16,
86 vlib_get_buffer_indices_with_offset (vm, (void **) cl, bi,
87 st.status.count, off);
88 bi += st.status.count;
92 n_freed = bi - (u32 *) ctq->ba_buffer;
94 vlib_buffer_free_no_next (vm, (u32 *) ctq->ba_buffer, n_freed);
96 /* clear status bits in each cacheline */
97 n = cl - ctq->ba_buffer;
98 for (u32 i = 0; i < n; i++)
99 ctq->ba_buffer[i].iova[0] = ctq->ba_buffer[i].iova[8] =
100 OCT_BATCH_ALLOC_IOVA0_MASK;
102 ctq->ba_num_cl = ctq->ba_first_cl = 0;
105 ah = ctq->aura_handle;
107 if ((n = roc_npa_aura_op_available (ah)) >= 32)
111 n = clib_min (n, ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS);
113 oct_npa_batch_alloc_compare_t cmp = {
114 .compare_s = { .aura = roc_npa_aura_handle_to_aura (ah),
115 .stype = ALLOC_STYPE_STF,
119 addr = roc_npa_aura_handle_to_base (ah) + NPA_LF_AURA_BATCH_ALLOC;
120 res = roc_atomic64_casl (cmp.as_u64, (uint64_t) ctq->ba_buffer,
122 if (res == ALLOC_RESULT_ACCEPTED || res == ALLOC_RESULT_NOCORE)
124 ctq->ba_num_cl = (n + 15) / 16;
125 ctq->ba_first_cl = 0;
128 ctx->batch_alloc_issue_fail++;
134 static_always_inline u8
135 oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b,
136 lmt_line_t *line, u32 flags, int simple, int trace)
141 .hdr_w0 = ctx->hdr_w0_teplate,
144 .subdc = NIX_SUBDC_SG,
147 .subdc = NIX_SUBDC_SG,
151 if (!simple && flags & VLIB_BUFFER_NEXT_PRESENT)
154 vlib_buffer_t *tail_segs[5], *t = b;
156 while (t->flags & VLIB_BUFFER_NEXT_PRESENT)
158 t = vlib_get_buffer (vm, t->next_buffer);
159 tail_segs[n_tail_segs++] = t;
162 ctx->drop[ctx->n_drop++] = t;
170 d.sg[7].u = (u64) vlib_buffer_get_current (tail_segs[4]);
171 total_len += d.sg[4].seg3_size = tail_segs[4]->current_length;
174 d.sg[6].u = (u64) vlib_buffer_get_current (tail_segs[3]);
175 total_len += d.sg[4].seg2_size = tail_segs[3]->current_length;
179 d.sg[5].u = (u64) vlib_buffer_get_current (tail_segs[2]);
180 total_len += d.sg[4].seg1_size = tail_segs[2]->current_length;
184 d.sg[3].u = (u64) vlib_buffer_get_current (tail_segs[1]);
185 total_len += d.sg[0].seg3_size = tail_segs[1]->current_length;
188 d.sg[2].u = (u64) vlib_buffer_get_current (tail_segs[0]);
189 total_len += d.sg[0].seg2_size = tail_segs[0]->current_length;
195 d.hdr_w0.sizem1 = n_dwords - 1;
198 if (!simple && flags & VNET_BUFFER_F_OFFLOAD)
200 vnet_buffer_oflags_t oflags = vnet_buffer (b)->oflags;
201 if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
203 d.hdr_w1.ol3type = NIX_SENDL3TYPE_IP4_CKSUM;
204 d.hdr_w1.ol3ptr = vnet_buffer (b)->l3_hdr_offset;
206 vnet_buffer (b)->l3_hdr_offset + sizeof (ip4_header_t);
208 if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
210 d.hdr_w1.ol4type = NIX_SENDL4TYPE_UDP_CKSUM;
211 d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset;
213 else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
215 d.hdr_w1.ol4type = NIX_SENDL4TYPE_TCP_CKSUM;
216 d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset;
220 total_len += d.sg[0].seg1_size = b->current_length;
221 d.hdr_w0.total = total_len;
222 d.sg[1].u = (u64) vlib_buffer_get_current (b);
224 if (trace && flags & VLIB_BUFFER_IS_TRACED)
226 oct_tx_trace_t *t = vlib_add_trace (vm, ctx->node, b, sizeof (*t));
228 t->sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_TX];
231 for (u32 i = 0; i < n_dwords; i++)
232 line->dwords[i] = d.as_u128[i];
237 static_always_inline u32
238 oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq,
239 vlib_buffer_t **b, u32 n_pkts, int trace)
241 u8 dwords_per_line[16], *dpl = dwords_per_line;
242 u64 lmt_arg, ioaddr, n_lines;
243 u32 n_left, or_flags_16 = 0;
244 const u32 not_simple_flags =
245 VLIB_BUFFER_NEXT_PRESENT | VNET_BUFFER_F_OFFLOAD;
246 lmt_line_t *l = ctx->lmt_lines;
248 /* Data Store Memory Barrier - outer shareable domain */
249 asm volatile("dmb oshst" ::: "memory");
251 for (n_left = n_pkts; n_left >= 8; n_left -= 8, b += 8, l += 8)
253 u32 f0, f1, f2, f3, f4, f5, f6, f7, or_f = 0;
254 vlib_prefetch_buffer_header (b[8], LOAD);
255 or_f |= f0 = b[0]->flags;
256 or_f |= f1 = b[1]->flags;
257 vlib_prefetch_buffer_header (b[9], LOAD);
258 or_f |= f2 = b[2]->flags;
259 or_f |= f3 = b[3]->flags;
260 vlib_prefetch_buffer_header (b[10], LOAD);
261 or_f |= f4 = b[4]->flags;
262 or_f |= f5 = b[5]->flags;
263 vlib_prefetch_buffer_header (b[11], LOAD);
264 or_f |= f6 = b[6]->flags;
265 or_f |= f7 = b[7]->flags;
266 vlib_prefetch_buffer_header (b[12], LOAD);
269 if ((or_f & not_simple_flags) == 0)
272 oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace);
273 oct_tx_enq1 (vm, ctx, b[1], l + 1, f1, simple, trace);
274 vlib_prefetch_buffer_header (b[13], LOAD);
275 oct_tx_enq1 (vm, ctx, b[2], l + 2, f2, simple, trace);
276 oct_tx_enq1 (vm, ctx, b[3], l + 3, f3, simple, trace);
277 vlib_prefetch_buffer_header (b[14], LOAD);
278 oct_tx_enq1 (vm, ctx, b[4], l + 4, f4, simple, trace);
279 oct_tx_enq1 (vm, ctx, b[5], l + 5, f5, simple, trace);
280 vlib_prefetch_buffer_header (b[15], LOAD);
281 oct_tx_enq1 (vm, ctx, b[6], l + 6, f6, simple, trace);
282 oct_tx_enq1 (vm, ctx, b[7], l + 7, f7, simple, trace);
283 dpl[0] = dpl[1] = dpl[2] = dpl[3] = 2;
284 dpl[4] = dpl[5] = dpl[6] = dpl[7] = 2;
289 dpl[0] = oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace);
290 dpl[1] = oct_tx_enq1 (vm, ctx, b[1], l + 1, f1, simple, trace);
291 vlib_prefetch_buffer_header (b[13], LOAD);
292 dpl[2] = oct_tx_enq1 (vm, ctx, b[2], l + 2, f2, simple, trace);
293 dpl[3] = oct_tx_enq1 (vm, ctx, b[3], l + 3, f3, simple, trace);
294 vlib_prefetch_buffer_header (b[14], LOAD);
295 dpl[4] = oct_tx_enq1 (vm, ctx, b[4], l + 4, f4, simple, trace);
296 dpl[5] = oct_tx_enq1 (vm, ctx, b[5], l + 5, f5, simple, trace);
297 vlib_prefetch_buffer_header (b[15], LOAD);
298 dpl[6] = oct_tx_enq1 (vm, ctx, b[6], l + 6, f6, simple, trace);
299 dpl[7] = oct_tx_enq1 (vm, ctx, b[7], l + 7, f7, simple, trace);
304 for (; n_left > 0; n_left -= 1, b += 1, l += 1)
306 u32 f0 = b[0]->flags;
307 dpl++[0] = oct_tx_enq1 (vm, ctx, b[0], l, f0, 0, trace);
311 lmt_arg = ctx->lmt_id;
312 ioaddr = ctx->lmt_ioaddr;
315 if (PREDICT_FALSE (or_flags_16 & VLIB_BUFFER_NEXT_PRESENT))
317 dpl = dwords_per_line;
318 ioaddr |= (dpl[0] - 1) << 4;
322 lmt_arg |= (--n_lines) << 12;
324 for (u8 bit_off = 19; n_lines; n_lines--, bit_off += 3, dpl++)
325 lmt_arg |= ((u64) dpl[1] - 1) << bit_off;
330 const u64 n_dwords = 2;
331 ioaddr |= (n_dwords - 1) << 4;
335 lmt_arg |= (--n_lines) << 12;
337 for (u8 bit_off = 19; n_lines; n_lines--, bit_off += 3)
338 lmt_arg |= (n_dwords - 1) << bit_off;
342 roc_lmt_submit_steorl (lmt_arg, ioaddr);
347 VNET_DEV_NODE_FN (oct_tx_node)
348 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
350 vnet_dev_tx_node_runtime_t *rt = vnet_dev_get_tx_node_runtime (node);
351 vnet_dev_tx_queue_t *txq = rt->tx_queue;
352 oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq);
353 u32 node_index = node->node_index;
354 u32 *from = vlib_frame_vector_args (frame);
355 u32 n, n_enq, n_left, n_pkts = frame->n_vectors;
356 vlib_buffer_t *buffers[VLIB_FRAME_SIZE + 8], **b = buffers;
357 u64 lmt_id = vm->thread_index << ROC_LMT_LINES_PER_CORE_LOG2;
362 .aura = roc_npa_aura_handle_to_aura (ctq->aura_handle),
367 .lmt_ioaddr = ctq->io_addr,
368 .lmt_lines = ctq->lmt_addr + (lmt_id << ROC_LMT_LINE_SIZE_LOG2),
371 vlib_get_buffers (vm, vlib_frame_vector_args (frame), b, n_pkts);
372 for (int i = 0; i < 8; i++)
373 b[n_pkts + i] = b[n_pkts - 1];
375 vnet_dev_tx_queue_lock_if_needed (txq);
378 n_enq -= oct_batch_free (vm, &ctx, txq);
380 if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
382 for (n_left = clib_min (n_pkts, txq->size - n_enq), n = 0; n_left >= 16;
383 n_left -= 16, b += 16)
384 n += oct_tx_enq16 (vm, &ctx, txq, b, 16, /* trace */ 1);
387 n += oct_tx_enq16 (vm, &ctx, txq, b, n_left, /* trace */ 1);
391 for (n_left = clib_min (n_pkts, txq->size - n_enq), n = 0; n_left >= 16;
392 n_left -= 16, b += 16)
393 n += oct_tx_enq16 (vm, &ctx, txq, b, 16, /* trace */ 0);
396 n += oct_tx_enq16 (vm, &ctx, txq, b, n_left, /* trace */ 0);
399 ctq->n_enq = n_enq + n;
403 u32 n_free = n_pkts - n;
404 vlib_buffer_free (vm, from + n, n_free);
405 vlib_error_count (vm, node->node_index, OCT_TX_NODE_CTR_NO_FREE_SLOTS,
411 vlib_error_count (vm, node->node_index, OCT_TX_NODE_CTR_CHAIN_TOO_LONG,
414 if (ctx.batch_alloc_not_ready)
415 vlib_error_count (vm, node_index,
416 OCT_TX_NODE_CTR_AURA_BATCH_ALLOC_NOT_READY,
417 ctx.batch_alloc_not_ready);
419 if (ctx.batch_alloc_issue_fail)
420 vlib_error_count (vm, node_index,
421 OCT_TX_NODE_CTR_AURA_BATCH_ALLOC_ISSUE_FAIL,
422 ctx.batch_alloc_issue_fail);
424 vnet_dev_tx_queue_unlock_if_needed (txq);
428 u32 bi[VLIB_FRAME_SIZE];
429 vlib_get_buffer_indices (vm, ctx.drop, bi, ctx.n_drop);
430 vlib_buffer_free (vm, bi, ctx.n_drop);
431 n_pkts -= ctx.n_drop;