2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * ip/ip4_forward.h: IP v4 forwarding
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
40 #ifndef __included_ip4_forward_h__
41 #define __included_ip4_forward_h__
43 #include <vppinfra/cache.h>
44 #include <vnet/fib/ip4_fib.h>
45 #include <vnet/dpo/load_balance_map.h>
49 * @brief IPv4 Forwarding.
51 * This file contains the source code for IPv4 forwarding.
55 ip4_lookup_inline (vlib_main_t * vm,
56 vlib_node_runtime_t * node,
58 int lookup_for_responses_to_locally_received_packets)
60 ip4_main_t *im = &ip4_main;
61 vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters;
62 u32 n_left_from, n_left_to_next, *from, *to_next;
63 ip_lookup_next_t next;
64 u32 thread_index = vm->thread_index;
66 from = vlib_frame_vector_args (frame);
67 n_left_from = frame->n_vectors;
68 next = node->cached_next_index;
70 while (n_left_from > 0)
72 vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
74 #if (CLIB_N_PREFETCHES >= 8)
75 while (n_left_from >= 8 && n_left_to_next >= 4)
77 vlib_buffer_t *p0, *p1, *p2, *p3;
78 ip4_header_t *ip0, *ip1, *ip2, *ip3;
79 ip_lookup_next_t next0, next1, next2, next3;
80 const load_balance_t *lb0, *lb1, *lb2, *lb3;
81 ip4_fib_mtrie_t *mtrie0, *mtrie1, *mtrie2, *mtrie3;
82 ip4_fib_mtrie_leaf_t leaf0, leaf1, leaf2, leaf3;
83 ip4_address_t *dst_addr0, *dst_addr1, *dst_addr2, *dst_addr3;
84 u32 pi0, pi1, pi2, pi3, lb_index0, lb_index1, lb_index2, lb_index3;
85 flow_hash_config_t flow_hash_config0, flow_hash_config1;
86 flow_hash_config_t flow_hash_config2, flow_hash_config3;
87 u32 hash_c0, hash_c1, hash_c2, hash_c3;
88 const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
90 /* Prefetch next iteration. */
92 vlib_buffer_t *p4, *p5, *p6, *p7;
94 p4 = vlib_get_buffer (vm, from[4]);
95 p5 = vlib_get_buffer (vm, from[5]);
96 p6 = vlib_get_buffer (vm, from[6]);
97 p7 = vlib_get_buffer (vm, from[7]);
99 vlib_prefetch_buffer_header (p4, LOAD);
100 vlib_prefetch_buffer_header (p5, LOAD);
101 vlib_prefetch_buffer_header (p6, LOAD);
102 vlib_prefetch_buffer_header (p7, LOAD);
104 CLIB_PREFETCH (p4->data, sizeof (ip0[0]), LOAD);
105 CLIB_PREFETCH (p5->data, sizeof (ip0[0]), LOAD);
106 CLIB_PREFETCH (p6->data, sizeof (ip0[0]), LOAD);
107 CLIB_PREFETCH (p7->data, sizeof (ip0[0]), LOAD);
110 pi0 = to_next[0] = from[0];
111 pi1 = to_next[1] = from[1];
112 pi2 = to_next[2] = from[2];
113 pi3 = to_next[3] = from[3];
120 p0 = vlib_get_buffer (vm, pi0);
121 p1 = vlib_get_buffer (vm, pi1);
122 p2 = vlib_get_buffer (vm, pi2);
123 p3 = vlib_get_buffer (vm, pi3);
125 ip0 = vlib_buffer_get_current (p0);
126 ip1 = vlib_buffer_get_current (p1);
127 ip2 = vlib_buffer_get_current (p2);
128 ip3 = vlib_buffer_get_current (p3);
130 dst_addr0 = &ip0->dst_address;
131 dst_addr1 = &ip1->dst_address;
132 dst_addr2 = &ip2->dst_address;
133 dst_addr3 = &ip3->dst_address;
135 ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p0);
136 ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p1);
137 ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p2);
138 ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p3);
140 if (!lookup_for_responses_to_locally_received_packets)
142 mtrie0 = &ip4_fib_get (vnet_buffer (p0)->ip.fib_index)->mtrie;
143 mtrie1 = &ip4_fib_get (vnet_buffer (p1)->ip.fib_index)->mtrie;
144 mtrie2 = &ip4_fib_get (vnet_buffer (p2)->ip.fib_index)->mtrie;
145 mtrie3 = &ip4_fib_get (vnet_buffer (p3)->ip.fib_index)->mtrie;
147 leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
148 leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, dst_addr1);
149 leaf2 = ip4_fib_mtrie_lookup_step_one (mtrie2, dst_addr2);
150 leaf3 = ip4_fib_mtrie_lookup_step_one (mtrie3, dst_addr3);
153 if (!lookup_for_responses_to_locally_received_packets)
155 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
156 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
157 leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 2);
158 leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 2);
161 if (!lookup_for_responses_to_locally_received_packets)
163 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
164 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
165 leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 3);
166 leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 3);
169 if (lookup_for_responses_to_locally_received_packets)
171 lb_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
172 lb_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
173 lb_index2 = vnet_buffer (p2)->ip.adj_index[VLIB_RX];
174 lb_index3 = vnet_buffer (p3)->ip.adj_index[VLIB_RX];
178 lb_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
179 lb_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
180 lb_index2 = ip4_fib_mtrie_leaf_get_adj_index (leaf2);
181 lb_index3 = ip4_fib_mtrie_leaf_get_adj_index (leaf3);
184 ASSERT (lb_index0 && lb_index1 && lb_index2 && lb_index3);
185 lb0 = load_balance_get (lb_index0);
186 lb1 = load_balance_get (lb_index1);
187 lb2 = load_balance_get (lb_index2);
188 lb3 = load_balance_get (lb_index3);
190 ASSERT (lb0->lb_n_buckets > 0);
191 ASSERT (is_pow2 (lb0->lb_n_buckets));
192 ASSERT (lb1->lb_n_buckets > 0);
193 ASSERT (is_pow2 (lb1->lb_n_buckets));
194 ASSERT (lb2->lb_n_buckets > 0);
195 ASSERT (is_pow2 (lb2->lb_n_buckets));
196 ASSERT (lb3->lb_n_buckets > 0);
197 ASSERT (is_pow2 (lb3->lb_n_buckets));
199 /* Use flow hash to compute multipath adjacency. */
200 hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
201 hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
202 hash_c2 = vnet_buffer (p2)->ip.flow_hash = 0;
203 hash_c3 = vnet_buffer (p3)->ip.flow_hash = 0;
204 if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
206 flow_hash_config0 = lb0->lb_hash_config;
207 hash_c0 = vnet_buffer (p0)->ip.flow_hash =
208 ip4_compute_flow_hash (ip0, flow_hash_config0);
210 load_balance_get_fwd_bucket (lb0,
212 (lb0->lb_n_buckets_minus_1)));
216 dpo0 = load_balance_get_bucket_i (lb0, 0);
218 if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
220 flow_hash_config1 = lb1->lb_hash_config;
221 hash_c1 = vnet_buffer (p1)->ip.flow_hash =
222 ip4_compute_flow_hash (ip1, flow_hash_config1);
224 load_balance_get_fwd_bucket (lb1,
226 (lb1->lb_n_buckets_minus_1)));
230 dpo1 = load_balance_get_bucket_i (lb1, 0);
232 if (PREDICT_FALSE (lb2->lb_n_buckets > 1))
234 flow_hash_config2 = lb2->lb_hash_config;
235 hash_c2 = vnet_buffer (p2)->ip.flow_hash =
236 ip4_compute_flow_hash (ip2, flow_hash_config2);
238 load_balance_get_fwd_bucket (lb2,
240 (lb2->lb_n_buckets_minus_1)));
244 dpo2 = load_balance_get_bucket_i (lb2, 0);
246 if (PREDICT_FALSE (lb3->lb_n_buckets > 1))
248 flow_hash_config3 = lb3->lb_hash_config;
249 hash_c3 = vnet_buffer (p3)->ip.flow_hash =
250 ip4_compute_flow_hash (ip3, flow_hash_config3);
252 load_balance_get_fwd_bucket (lb3,
254 (lb3->lb_n_buckets_minus_1)));
258 dpo3 = load_balance_get_bucket_i (lb3, 0);
261 next0 = dpo0->dpoi_next_node;
262 vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
263 next1 = dpo1->dpoi_next_node;
264 vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
265 next2 = dpo2->dpoi_next_node;
266 vnet_buffer (p2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
267 next3 = dpo3->dpoi_next_node;
268 vnet_buffer (p3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
270 vlib_increment_combined_counter
271 (cm, thread_index, lb_index0, 1,
272 vlib_buffer_length_in_chain (vm, p0));
273 vlib_increment_combined_counter
274 (cm, thread_index, lb_index1, 1,
275 vlib_buffer_length_in_chain (vm, p1));
276 vlib_increment_combined_counter
277 (cm, thread_index, lb_index2, 1,
278 vlib_buffer_length_in_chain (vm, p2));
279 vlib_increment_combined_counter
280 (cm, thread_index, lb_index3, 1,
281 vlib_buffer_length_in_chain (vm, p3));
283 vlib_validate_buffer_enqueue_x4 (vm, node, next,
284 to_next, n_left_to_next,
286 next0, next1, next2, next3);
288 #elif (CLIB_N_PREFETCHES >= 4)
289 while (n_left_from >= 4 && n_left_to_next >= 2)
291 vlib_buffer_t *p0, *p1;
292 ip4_header_t *ip0, *ip1;
293 ip_lookup_next_t next0, next1;
294 const load_balance_t *lb0, *lb1;
295 ip4_fib_mtrie_t *mtrie0, *mtrie1;
296 ip4_fib_mtrie_leaf_t leaf0, leaf1;
297 ip4_address_t *dst_addr0, *dst_addr1;
298 u32 pi0, pi1, lb_index0, lb_index1;
299 flow_hash_config_t flow_hash_config0, flow_hash_config1;
300 u32 hash_c0, hash_c1;
301 const dpo_id_t *dpo0, *dpo1;
303 /* Prefetch next iteration. */
305 vlib_buffer_t *p2, *p3;
307 p2 = vlib_get_buffer (vm, from[2]);
308 p3 = vlib_get_buffer (vm, from[3]);
310 vlib_prefetch_buffer_header (p2, LOAD);
311 vlib_prefetch_buffer_header (p3, LOAD);
313 CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
314 CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
317 pi0 = to_next[0] = from[0];
318 pi1 = to_next[1] = from[1];
325 p0 = vlib_get_buffer (vm, pi0);
326 p1 = vlib_get_buffer (vm, pi1);
328 ip0 = vlib_buffer_get_current (p0);
329 ip1 = vlib_buffer_get_current (p1);
331 dst_addr0 = &ip0->dst_address;
332 dst_addr1 = &ip1->dst_address;
334 ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p0);
335 ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p1);
337 if (!lookup_for_responses_to_locally_received_packets)
339 mtrie0 = &ip4_fib_get (vnet_buffer (p0)->ip.fib_index)->mtrie;
340 mtrie1 = &ip4_fib_get (vnet_buffer (p1)->ip.fib_index)->mtrie;
342 leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
343 leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, dst_addr1);
346 if (!lookup_for_responses_to_locally_received_packets)
348 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
349 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
352 if (!lookup_for_responses_to_locally_received_packets)
354 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
355 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
358 if (lookup_for_responses_to_locally_received_packets)
360 lb_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
361 lb_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
365 lb_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
366 lb_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
369 ASSERT (lb_index0 && lb_index1);
370 lb0 = load_balance_get (lb_index0);
371 lb1 = load_balance_get (lb_index1);
373 ASSERT (lb0->lb_n_buckets > 0);
374 ASSERT (is_pow2 (lb0->lb_n_buckets));
375 ASSERT (lb1->lb_n_buckets > 0);
376 ASSERT (is_pow2 (lb1->lb_n_buckets));
378 /* Use flow hash to compute multipath adjacency. */
379 hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
380 hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
381 if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
383 flow_hash_config0 = lb0->lb_hash_config;
384 hash_c0 = vnet_buffer (p0)->ip.flow_hash =
385 ip4_compute_flow_hash (ip0, flow_hash_config0);
387 load_balance_get_fwd_bucket (lb0,
389 (lb0->lb_n_buckets_minus_1)));
393 dpo0 = load_balance_get_bucket_i (lb0, 0);
395 if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
397 flow_hash_config1 = lb1->lb_hash_config;
398 hash_c1 = vnet_buffer (p1)->ip.flow_hash =
399 ip4_compute_flow_hash (ip1, flow_hash_config1);
401 load_balance_get_fwd_bucket (lb1,
403 (lb1->lb_n_buckets_minus_1)));
407 dpo1 = load_balance_get_bucket_i (lb1, 0);
410 next0 = dpo0->dpoi_next_node;
411 vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
412 next1 = dpo1->dpoi_next_node;
413 vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
415 vlib_increment_combined_counter
416 (cm, thread_index, lb_index0, 1,
417 vlib_buffer_length_in_chain (vm, p0));
418 vlib_increment_combined_counter
419 (cm, thread_index, lb_index1, 1,
420 vlib_buffer_length_in_chain (vm, p1));
422 vlib_validate_buffer_enqueue_x2 (vm, node, next,
423 to_next, n_left_to_next,
424 pi0, pi1, next0, next1);
427 while (n_left_from > 0 && n_left_to_next > 0)
431 ip_lookup_next_t next0;
432 const load_balance_t *lb0;
433 ip4_fib_mtrie_t *mtrie0;
434 ip4_fib_mtrie_leaf_t leaf0;
435 ip4_address_t *dst_addr0;
437 flow_hash_config_t flow_hash_config0;
438 const dpo_id_t *dpo0;
444 p0 = vlib_get_buffer (vm, pi0);
445 ip0 = vlib_buffer_get_current (p0);
446 dst_addr0 = &ip0->dst_address;
447 ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p0);
449 if (!lookup_for_responses_to_locally_received_packets)
451 mtrie0 = &ip4_fib_get (vnet_buffer (p0)->ip.fib_index)->mtrie;
452 leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
455 if (!lookup_for_responses_to_locally_received_packets)
456 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
458 if (!lookup_for_responses_to_locally_received_packets)
459 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
461 if (lookup_for_responses_to_locally_received_packets)
462 lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
465 /* Handle default route. */
466 lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
470 lb0 = load_balance_get (lbi0);
472 ASSERT (lb0->lb_n_buckets > 0);
473 ASSERT (is_pow2 (lb0->lb_n_buckets));
475 /* Use flow hash to compute multipath adjacency. */
476 hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
477 if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
479 flow_hash_config0 = lb0->lb_hash_config;
481 hash_c0 = vnet_buffer (p0)->ip.flow_hash =
482 ip4_compute_flow_hash (ip0, flow_hash_config0);
484 load_balance_get_fwd_bucket (lb0,
486 (lb0->lb_n_buckets_minus_1)));
490 dpo0 = load_balance_get_bucket_i (lb0, 0);
493 next0 = dpo0->dpoi_next_node;
494 vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
496 vlib_increment_combined_counter (cm, thread_index, lbi0, 1,
497 vlib_buffer_length_in_chain (vm,
505 if (PREDICT_FALSE (next0 != next))
508 vlib_put_next_frame (vm, node, next, n_left_to_next);
510 vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
517 vlib_put_next_frame (vm, node, next, n_left_to_next);
520 if (node->flags & VLIB_NODE_FLAG_TRACE)
521 ip4_forward_next_trace (vm, node, frame, VLIB_TX);
523 return frame->n_vectors;
526 #endif /* __included_ip4_forward_h__ */
529 * fd.io coding-style-patch-verification: ON
532 * eval: (c-set-style "gnu")