2 *------------------------------------------------------------------
3 * Copyright (c) 2018 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vnet/ethernet/ethernet.h>
22 #include <vnet/devices/devices.h>
26 #define foreach_avf_input_error \
27 _(BUFFER_ALLOC, "buffer alloc error") \
28 _(RX_PACKET_ERROR, "Rx packet errors")
32 #define _(f,s) AVF_INPUT_ERROR_##f,
33 foreach_avf_input_error
38 static __clib_unused char *avf_input_error_strings[] = {
40 foreach_avf_input_error
44 #define AVF_RX_DESC_STATUS(x) (1 << x)
45 #define AVF_RX_DESC_STATUS_DD AVF_RX_DESC_STATUS(0)
46 #define AVF_RX_DESC_STATUS_EOP AVF_RX_DESC_STATUS(1)
48 static_always_inline void
49 avf_input_trace (vlib_main_t * vm, vlib_node_runtime_t * node, u32 next0,
50 vlib_buffer_t * b0, uword * n_trace, avf_device_t * ad,
51 avf_rx_vector_entry_t * rxve)
53 avf_input_trace_t *tr;
54 vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
55 vlib_set_trace_count (vm, node, --(*n_trace));
56 tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
57 tr->next_index = next0;
58 tr->hw_if_index = ad->hw_if_index;
59 clib_memcpy (&tr->rxve, rxve, sizeof (avf_rx_vector_entry_t));
62 #define AVF_INPUT_REFILL_TRESHOLD 32
63 static_always_inline void
64 avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq,
67 u16 n_refill, mask, n_alloc, slot;
70 n_refill = rxq->size - 1 - rxq->n_bufs;
71 if (PREDICT_TRUE (n_refill <= AVF_INPUT_REFILL_TRESHOLD))
75 slot = (rxq->next - n_refill - 1) & mask;
77 n_refill &= ~7; /* round to 8 */
78 n_alloc = vlib_buffer_alloc_to_ring (vm, rxq->bufs, slot, rxq->size,
81 if (PREDICT_FALSE (n_alloc != n_refill))
83 vlib_error_count (vm, node->node_index,
84 AVF_INPUT_ERROR_BUFFER_ALLOC, 1);
86 vlib_buffer_free (vm, rxq->bufs + slot, n_alloc);
90 rxq->n_bufs += n_alloc;
95 d = ((avf_rx_desc_t *) rxq->descs) + slot;
98 vlib_buffer_t *b = vlib_get_buffer (vm, rxq->bufs[slot]);
99 addr = pointer_to_uword (b->data);
102 addr = vlib_get_buffer_data_physical_address (vm, rxq->bufs[slot]);
105 slot = (slot + 1) & mask;
108 CLIB_MEMORY_BARRIER ();
109 *(rxq->qrx_tail) = slot;
112 static_always_inline void
113 avf_check_for_error (vlib_node_runtime_t * node, avf_rx_vector_entry_t * rxve,
114 vlib_buffer_t * b, u32 * next)
116 avf_main_t *am = &avf_main;
118 if (PREDICT_FALSE (rxve->error))
120 b->error = node->errors[AVF_INPUT_ERROR_RX_PACKET_ERROR];
121 ptype = am->ptypes + rxve->ptype;
123 vlib_buffer_advance (b, --ptype->buffer_advance);
124 *next = VNET_DEVICE_INPUT_NEXT_DROP;
128 static_always_inline u32
129 avf_find_next (avf_rx_vector_entry_t * rxve, vlib_buffer_t * b,
132 avf_main_t *am = &avf_main;
133 ethernet_header_t *e = (ethernet_header_t *) b->data;
135 if (maybe_tagged && ethernet_frame_is_tagged (e->type))
136 return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
137 ptype = am->ptypes + rxve->ptype;
138 vlib_buffer_advance (b, ptype->buffer_advance);
139 b->flags |= ptype->flags;
140 return ptype->next_node;
144 static_always_inline uword
145 avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
146 vlib_frame_t * frame, avf_device_t * ad, u16 qid,
147 int with_features_or_tracing)
149 avf_main_t *am = &avf_main;
150 vnet_main_t *vnm = vnet_get_main ();
151 u32 thr_idx = vlib_get_thread_index ();
152 avf_per_thread_data_t *ptd =
153 vec_elt_at_index (am->per_thread_data, thr_idx);
154 avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid);
155 avf_rx_vector_entry_t *rxve;
156 uword n_trace = vlib_get_trace_count (vm, node);
157 u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
160 u32 n_rx_packets = 0;
162 u32 sw_if_idx[VLIB_N_RX_TX] = {[VLIB_RX] = ad->sw_if_index,[VLIB_TX] = ~0 };
163 u16 mask = rxq->size - 1;
167 /* fetch up to AVF_RX_VECTOR_SZ from the rx ring, unflatten them and
168 copy needed data from descriptor to rx vector */
169 d = rxq->descs + rxq->next;
170 while ((d->qword[1] & AVF_RX_DESC_STATUS_DD) && n_rxv < AVF_RX_VECTOR_SZ)
172 u16 next_pf = (rxq->next + 8) & mask;
173 CLIB_PREFETCH (rxq->descs + next_pf, CLIB_CACHE_LINE_BYTES, LOAD);
174 rxve = ptd->rx_vector + n_rxv;
175 rxve->bi = rxq->bufs[rxq->next];
176 rxve->status = avf_get_u64_bits (d, 8, 18, 0);
177 rxve->error = avf_get_u64_bits (d, 8, 26, 19);
178 rxve->ptype = avf_get_u64_bits (d, 8, 37, 30);
179 rxve->length = avf_get_u64_bits (d, 8, 63, 38);
180 maybe_error |= rxve->error;
182 /* deal with chained buffers */
183 while (PREDICT_FALSE ((d->qword[1] & AVF_RX_DESC_STATUS_EOP) == 0))
185 clib_error ("fixme");
189 rxq->next = (rxq->next + 1) & mask;
190 d = rxq->descs + rxq->next;
199 if (ad->flags & AVF_DEVICE_F_IOVA)
200 avf_rxq_refill (vm, node, rxq, 1 /* use_iova */ );
202 avf_rxq_refill (vm, node, rxq, 0 /* use_iova */ );
204 n_rx_packets = n_rxv;
205 rxve = ptd->rx_vector;
209 u32 bi0, bi1, bi2, bi3;
210 vlib_buffer_t *b0, *b1, *b2, *b3;
211 u32 next0, next1, next2, next3;
213 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
215 while (n_rxv >= 12 && n_left_to_next >= 4)
218 p = vlib_get_buffer (vm, rxve[8].bi);
219 vlib_prefetch_buffer_header (p, LOAD);
220 CLIB_PREFETCH (p->data, CLIB_CACHE_LINE_BYTES, LOAD);
222 p = vlib_get_buffer (vm, rxve[9].bi);
223 vlib_prefetch_buffer_header (p, LOAD);
224 CLIB_PREFETCH (p->data, CLIB_CACHE_LINE_BYTES, LOAD);
226 p = vlib_get_buffer (vm, rxve[10].bi);
227 vlib_prefetch_buffer_header (p, LOAD);
228 CLIB_PREFETCH (p->data, CLIB_CACHE_LINE_BYTES, LOAD);
230 p = vlib_get_buffer (vm, rxve[11].bi);
231 vlib_prefetch_buffer_header (p, LOAD);
232 CLIB_PREFETCH (p->data, CLIB_CACHE_LINE_BYTES, LOAD);
234 to_next[0] = bi0 = rxve[0].bi;
235 to_next[1] = bi1 = rxve[1].bi;
236 to_next[2] = bi2 = rxve[2].bi;
237 to_next[3] = bi3 = rxve[3].bi;
239 b0 = vlib_get_buffer (vm, bi0);
240 b1 = vlib_get_buffer (vm, bi1);
241 b2 = vlib_get_buffer (vm, bi2);
242 b3 = vlib_get_buffer (vm, bi3);
244 b0->current_length = rxve[0].length;
245 b1->current_length = rxve[1].length;
246 b2->current_length = rxve[2].length;
247 b3->current_length = rxve[3].length;
249 n_rx_bytes += b0->current_length;
250 n_rx_bytes += b1->current_length;
251 n_rx_bytes += b2->current_length;
252 n_rx_bytes += b3->current_length;
254 if (PREDICT_TRUE (ad->per_interface_next_index == ~0))
256 ethernet_header_t *e0, *e1, *e2, *e3;
258 e0 = (ethernet_header_t *) b0->data;
259 e1 = (ethernet_header_t *) b1->data;
260 e2 = (ethernet_header_t *) b2->data;
261 e3 = (ethernet_header_t *) b3->data;
263 if (ethernet_frame_is_any_tagged_x4 (e0->type, e1->type,
266 next0 = avf_find_next (rxve, b0, 1);
267 next1 = avf_find_next (rxve + 1, b1, 1);
268 next2 = avf_find_next (rxve + 2, b2, 1);
269 next3 = avf_find_next (rxve + 3, b3, 1);
273 next0 = avf_find_next (rxve, b0, 0);
274 next1 = avf_find_next (rxve + 1, b1, 0);
275 next2 = avf_find_next (rxve + 2, b2, 0);
276 next3 = avf_find_next (rxve + 3, b3, 0);
279 if (with_features_or_tracing)
280 vnet_feature_start_device_input_x4 (ad->sw_if_index, &next0,
281 &next1, &next2, &next3,
284 if (PREDICT_FALSE (maybe_error))
286 avf_check_for_error (node, rxve + 0, b0, &next0);
287 avf_check_for_error (node, rxve + 1, b1, &next1);
288 avf_check_for_error (node, rxve + 2, b2, &next2);
289 avf_check_for_error (node, rxve + 3, b3, &next3);
293 next0 = next1 = next2 = next3 = ad->per_interface_next_index;
295 clib_memcpy (vnet_buffer (b0)->sw_if_index, sw_if_idx,
297 clib_memcpy (vnet_buffer (b1)->sw_if_index, sw_if_idx,
299 clib_memcpy (vnet_buffer (b2)->sw_if_index, sw_if_idx,
301 clib_memcpy (vnet_buffer (b3)->sw_if_index, sw_if_idx,
304 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
305 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
306 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2);
307 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3);
309 if (with_features_or_tracing && PREDICT_FALSE (n_trace))
311 avf_input_trace (vm, node, next0, b0, &n_trace, ad, rxve);
313 avf_input_trace (vm, node, next1, b1, &n_trace, ad, rxve + 1);
315 avf_input_trace (vm, node, next2, b2, &n_trace, ad, rxve + 2);
317 avf_input_trace (vm, node, next3, b3, &n_trace, ad, rxve + 3);
327 vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
328 n_left_to_next, bi0, bi1, bi2, bi3,
329 next0, next1, next2, next3);
331 while (n_rxv && n_left_to_next)
335 b0 = vlib_get_buffer (vm, bi0);
337 b0->current_length = rxve->length;
338 n_rx_bytes += b0->current_length;
340 if (PREDICT_TRUE (ad->per_interface_next_index == ~0))
342 next0 = avf_find_next (rxve, b0, 1);
343 if (with_features_or_tracing)
344 vnet_feature_start_device_input_x1 (ad->sw_if_index, &next0,
346 avf_check_for_error (node, rxve + 0, b0, &next0);
349 next0 = ad->per_interface_next_index;
351 clib_memcpy (vnet_buffer (b0)->sw_if_index, sw_if_idx,
354 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
355 if (with_features_or_tracing && PREDICT_FALSE (n_trace > 0))
356 avf_input_trace (vm, node, next0, b0, &n_trace, ad, rxve);
365 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
366 n_left_to_next, bi0, next0);
368 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
371 vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters
372 + VNET_INTERFACE_COUNTER_RX, thr_idx,
373 ad->hw_if_index, n_rx_packets, n_rx_bytes);
379 CLIB_MULTIARCH_FN (avf_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
380 vlib_frame_t * frame)
383 avf_main_t *am = &avf_main;
384 vnet_device_input_runtime_t *rt = (void *) node->runtime_data;
385 vnet_device_and_queue_t *dq;
387 foreach_device_and_queue (dq, rt->devices_and_queues)
390 ad = vec_elt_at_index (am->devices, dq->dev_instance);
391 if ((ad->flags & AVF_DEVICE_F_ADMIN_UP) == 0)
393 if (vnet_device_input_have_features (ad->sw_if_index) ||
394 vlib_get_trace_count (vm, node))
395 n_rx += avf_device_input_inline (vm, node, frame, ad, dq->queue_id, 1);
397 n_rx += avf_device_input_inline (vm, node, frame, ad, dq->queue_id, 0);
402 #ifndef CLIB_MULTIARCH_VARIANT
404 VLIB_REGISTER_NODE (avf_input_node) = {
405 .function = avf_input,
407 .sibling_of = "device-input",
408 .format_trace = format_avf_input_trace,
409 .type = VLIB_NODE_TYPE_INPUT,
410 .state = VLIB_NODE_STATE_DISABLED,
411 .n_errors = AVF_INPUT_N_ERROR,
412 .error_strings = avf_input_error_strings,
416 vlib_node_function_t __clib_weak avf_input_avx512;
417 vlib_node_function_t __clib_weak avf_input_avx2;
418 static void __clib_constructor
419 avf_input_multiarch_select (void)
421 if (avf_input_avx512 && clib_cpu_supports_avx512f ())
422 avf_input_node.function = avf_input_avx512;
423 else if (avf_input_avx2 && clib_cpu_supports_avx2 ())
424 avf_input_node.function = avf_input_avx2;
434 * fd.io coding-style-patch-verification: ON
437 * eval: (c-set-style "gnu")