2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * ip/ip4_forward.c: IP v4 forwarding
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
40 #include <vnet/vnet.h>
41 #include <vnet/ip/ip.h>
42 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
43 #include <vnet/ethernet/arp_packet.h> /* for ethernet_arp_header_t */
44 #include <vnet/ppp/ppp.h>
45 #include <vnet/srp/srp.h> /* for srp_hw_interface_class */
46 #include <vnet/api_errno.h> /* for API error numbers */
48 /* This is really, really simple but stupid fib. */
50 ip4_fib_lookup_with_table (ip4_main_t * im, u32 fib_index,
52 u32 disable_default_route)
54 ip_lookup_main_t * lm = &im->lookup_main;
55 ip4_fib_t * fib = vec_elt_at_index (im->fibs, fib_index);
56 uword * p, * hash, key;
57 i32 i, i_min, dst_address, ai;
59 i_min = disable_default_route ? 1 : 0;
60 dst_address = clib_mem_unaligned (&dst->data_u32, u32);
61 for (i = ARRAY_LEN (fib->adj_index_by_dst_address) - 1; i >= i_min; i--)
63 hash = fib->adj_index_by_dst_address[i];
67 key = dst_address & im->fib_masks[i];
68 if ((p = hash_get (hash, key)) != 0)
75 /* Nothing matches in table. */
76 ai = lm->miss_adj_index;
83 create_fib_with_table_id (ip4_main_t * im, u32 table_id)
86 hash_set (im->fib_index_by_table_id, table_id, vec_len (im->fibs));
87 vec_add2 (im->fibs, fib, 1);
88 fib->table_id = table_id;
89 fib->index = fib - im->fibs;
90 fib->flow_hash_config = IP_FLOW_HASH_DEFAULT;
91 fib->fwd_classify_table_index = ~0;
92 fib->rev_classify_table_index = ~0;
93 ip4_mtrie_init (&fib->mtrie);
98 find_ip4_fib_by_table_index_or_id (ip4_main_t * im,
99 u32 table_index_or_id, u32 flags)
101 uword * p, fib_index;
103 fib_index = table_index_or_id;
104 if (! (flags & IP4_ROUTE_FLAG_FIB_INDEX))
106 if (table_index_or_id == ~0) {
107 table_index_or_id = 0;
108 while ((p = hash_get (im->fib_index_by_table_id, table_index_or_id))) {
111 return create_fib_with_table_id (im, table_index_or_id);
114 p = hash_get (im->fib_index_by_table_id, table_index_or_id);
116 return create_fib_with_table_id (im, table_index_or_id);
119 return vec_elt_at_index (im->fibs, fib_index);
123 ip4_fib_init_adj_index_by_dst_address (ip_lookup_main_t * lm,
130 ASSERT (lm->fib_result_n_bytes >= sizeof (uword));
131 lm->fib_result_n_words = round_pow2 (lm->fib_result_n_bytes, sizeof (uword)) / sizeof (uword);
133 fib->adj_index_by_dst_address[address_length] =
134 hash_create (32 /* elts */, lm->fib_result_n_words * sizeof (uword));
136 hash_set_flags (fib->adj_index_by_dst_address[address_length],
137 HASH_FLAG_NO_AUTO_SHRINK);
139 h = hash_header (fib->adj_index_by_dst_address[address_length]);
140 max_index = (hash_value_bytes (h) / sizeof (fib->new_hash_values[0])) - 1;
142 /* Initialize new/old hash value vectors. */
143 vec_validate_init_empty (fib->new_hash_values, max_index, ~0);
144 vec_validate_init_empty (fib->old_hash_values, max_index, ~0);
148 ip4_fib_set_adj_index (ip4_main_t * im,
152 u32 dst_address_length,
155 ip_lookup_main_t * lm = &im->lookup_main;
158 if (vec_bytes(fib->old_hash_values))
159 memset (fib->old_hash_values, ~0, vec_bytes (fib->old_hash_values));
160 if (vec_bytes(fib->new_hash_values))
161 memset (fib->new_hash_values, ~0, vec_bytes (fib->new_hash_values));
162 fib->new_hash_values[0] = adj_index;
164 /* Make sure adj index is valid. */
166 (void) ip_get_adjacency (lm, adj_index);
168 hash = fib->adj_index_by_dst_address[dst_address_length];
170 hash = _hash_set3 (hash, dst_address_u32,
171 fib->new_hash_values,
172 fib->old_hash_values);
174 fib->adj_index_by_dst_address[dst_address_length] = hash;
176 if (vec_len (im->add_del_route_callbacks) > 0)
178 ip4_add_del_route_callback_t * cb;
182 d.data_u32 = dst_address_u32;
183 vec_foreach (cb, im->add_del_route_callbacks)
184 if ((flags & cb->required_flags) == cb->required_flags)
185 cb->function (im, cb->function_opaque,
187 &d, dst_address_length,
188 fib->old_hash_values,
189 fib->new_hash_values);
191 p = hash_get (hash, dst_address_u32);
192 clib_memcpy (p, fib->new_hash_values, vec_bytes (fib->new_hash_values));
196 void ip4_add_del_route (ip4_main_t * im, ip4_add_del_route_args_t * a)
198 ip_lookup_main_t * lm = &im->lookup_main;
200 u32 dst_address, dst_address_length, adj_index, old_adj_index;
201 uword * hash, is_del;
202 ip4_add_del_route_callback_t * cb;
204 /* Either create new adjacency or use given one depending on arguments. */
205 if (a->n_add_adj > 0)
207 ip_add_adjacency (lm, a->add_adj, a->n_add_adj, &adj_index);
208 ip_call_add_del_adjacency_callbacks (lm, adj_index, /* is_del */ 0);
211 adj_index = a->adj_index;
213 dst_address = a->dst_address.data_u32;
214 dst_address_length = a->dst_address_length;
215 fib = find_ip4_fib_by_table_index_or_id (im, a->table_index_or_table_id, a->flags);
217 ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
218 dst_address &= im->fib_masks[dst_address_length];
220 if (! fib->adj_index_by_dst_address[dst_address_length])
221 ip4_fib_init_adj_index_by_dst_address (lm, fib, dst_address_length);
223 hash = fib->adj_index_by_dst_address[dst_address_length];
225 is_del = (a->flags & IP4_ROUTE_FLAG_DEL) != 0;
229 fib->old_hash_values[0] = ~0;
230 hash = _hash_unset (hash, dst_address, fib->old_hash_values);
231 fib->adj_index_by_dst_address[dst_address_length] = hash;
233 if (vec_len (im->add_del_route_callbacks) > 0
234 && fib->old_hash_values[0] != ~0) /* make sure destination was found in hash */
236 fib->new_hash_values[0] = ~0;
237 vec_foreach (cb, im->add_del_route_callbacks)
238 if ((a->flags & cb->required_flags) == cb->required_flags)
239 cb->function (im, cb->function_opaque,
241 &a->dst_address, dst_address_length,
242 fib->old_hash_values,
243 fib->new_hash_values);
247 ip4_fib_set_adj_index (im, fib, a->flags, dst_address, dst_address_length,
250 old_adj_index = fib->old_hash_values[0];
252 /* Avoid spurious reference count increments */
253 if (old_adj_index == adj_index
255 && !(a->flags & IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
257 ip_adjacency_t * adj = ip_get_adjacency (lm, adj_index);
258 if (adj->share_count > 0)
262 ip4_fib_mtrie_add_del_route (fib, a->dst_address, dst_address_length,
263 is_del ? old_adj_index : adj_index,
266 /* Delete old adjacency index if present and changed. */
267 if (! (a->flags & IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY)
268 && old_adj_index != ~0
269 && old_adj_index != adj_index)
270 ip_del_adjacency (lm, old_adj_index);
274 ip4_add_del_route_next_hop (ip4_main_t * im,
276 ip4_address_t * dst_address,
277 u32 dst_address_length,
278 ip4_address_t * next_hop,
279 u32 next_hop_sw_if_index,
280 u32 next_hop_weight, u32 adj_index,
281 u32 explicit_fib_index)
283 vnet_main_t * vnm = vnet_get_main();
284 ip_lookup_main_t * lm = &im->lookup_main;
287 u32 dst_address_u32, old_mp_adj_index, new_mp_adj_index;
288 u32 dst_adj_index, nh_adj_index;
289 uword * dst_hash, * dst_result;
290 uword * nh_hash, * nh_result;
291 ip_adjacency_t * dst_adj;
292 ip_multipath_adjacency_t * old_mp, * new_mp;
293 int is_del = (flags & IP4_ROUTE_FLAG_DEL) != 0;
294 int is_interface_next_hop;
295 clib_error_t * error = 0;
297 if (explicit_fib_index == (u32)~0)
298 fib_index = vec_elt (im->fib_index_by_sw_if_index, next_hop_sw_if_index);
300 fib_index = explicit_fib_index;
302 fib = vec_elt_at_index (im->fibs, fib_index);
304 /* Lookup next hop to be added or deleted. */
305 is_interface_next_hop = next_hop->data_u32 == 0;
306 if (adj_index == (u32)~0)
308 if (is_interface_next_hop)
310 nh_result = hash_get (im->interface_route_adj_index_by_sw_if_index, next_hop_sw_if_index);
312 nh_adj_index = *nh_result;
315 ip_adjacency_t * adj;
316 adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
318 ip4_adjacency_set_interface_route (vnm, adj, next_hop_sw_if_index, /* if_address_index */ ~0);
319 ip_call_add_del_adjacency_callbacks (lm, nh_adj_index, /* is_del */ 0);
320 hash_set (im->interface_route_adj_index_by_sw_if_index, next_hop_sw_if_index, nh_adj_index);
325 nh_hash = fib->adj_index_by_dst_address[32];
326 nh_result = hash_get (nh_hash, next_hop->data_u32);
328 /* Next hop must be known. */
331 ip_adjacency_t * adj;
333 nh_adj_index = ip4_fib_lookup_with_table (im, fib_index,
335 adj = ip_get_adjacency (lm, nh_adj_index);
336 /* if ARP interface adjacencty is present, we need to
337 install ARP adjaceny for specific next hop */
338 if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP &&
339 adj->arp.next_hop.ip4.as_u32 == 0)
341 nh_adj_index = vnet_arp_glean_add(fib_index, next_hop);
345 /* Next hop is not known, so create indirect adj */
346 ip_adjacency_t add_adj;
347 memset (&add_adj, 0, sizeof(add_adj));
349 add_adj.lookup_next_index = IP_LOOKUP_NEXT_INDIRECT;
350 add_adj.indirect.next_hop.ip4.as_u32 = next_hop->as_u32;
351 add_adj.explicit_fib_index = explicit_fib_index;
352 ip_add_adjacency (lm, &add_adj, 1, &nh_adj_index);
356 nh_adj_index = *nh_result;
361 nh_adj_index = adj_index;
363 ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
364 dst_address_u32 = dst_address->data_u32 & im->fib_masks[dst_address_length];
366 dst_hash = fib->adj_index_by_dst_address[dst_address_length];
367 dst_result = hash_get (dst_hash, dst_address_u32);
370 dst_adj_index = dst_result[0];
371 dst_adj = ip_get_adjacency (lm, dst_adj_index);
375 /* For deletes destination must be known. */
378 vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
379 error = clib_error_return (0, "unknown destination %U/%d",
380 format_ip4_address, dst_address,
389 /* Ignore adds of X/32 with next hop of X. */
391 && dst_address_length == 32
392 && dst_address->data_u32 == next_hop->data_u32
393 && adj_index != (u32)~0)
395 vnm->api_errno = VNET_API_ERROR_PREFIX_MATCHES_NEXT_HOP;
396 error = clib_error_return (0, "prefix matches next hop %U/%d",
397 format_ip4_address, dst_address,
402 /* Destination is not known and default weight is set so add route
403 to existing non-multipath adjacency */
404 if (dst_adj_index == ~0 && next_hop_weight == 1 && next_hop_sw_if_index == ~0)
406 /* create new adjacency */
407 ip4_add_del_route_args_t a;
408 a.table_index_or_table_id = fib_index;
409 a.flags = ((is_del ? IP4_ROUTE_FLAG_DEL : IP4_ROUTE_FLAG_ADD)
410 | IP4_ROUTE_FLAG_FIB_INDEX
411 | IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY
412 | (flags & (IP4_ROUTE_FLAG_NO_REDISTRIBUTE
413 | IP4_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
414 a.dst_address = dst_address[0];
415 a.dst_address_length = dst_address_length;
416 a.adj_index = nh_adj_index;
420 ip4_add_del_route (im, &a);
425 old_mp_adj_index = dst_adj ? dst_adj->heap_handle : ~0;
427 if (! ip_multipath_adjacency_add_del_next_hop
434 vnm->api_errno = VNET_API_ERROR_NEXT_HOP_NOT_FOUND_MP;
435 error = clib_error_return (0, "requested deleting next-hop %U not found in multi-path",
436 format_ip4_address, next_hop);
441 if (old_mp_adj_index != ~0)
442 old_mp = vec_elt_at_index (lm->multipath_adjacencies, old_mp_adj_index);
443 if (new_mp_adj_index != ~0)
444 new_mp = vec_elt_at_index (lm->multipath_adjacencies, new_mp_adj_index);
446 if (old_mp != new_mp)
448 ip4_add_del_route_args_t a;
449 a.table_index_or_table_id = fib_index;
450 a.flags = ((is_del && ! new_mp ? IP4_ROUTE_FLAG_DEL : IP4_ROUTE_FLAG_ADD)
451 | IP4_ROUTE_FLAG_FIB_INDEX
452 | IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY
453 | (flags & (IP4_ROUTE_FLAG_NO_REDISTRIBUTE | IP4_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
454 a.dst_address = dst_address[0];
455 a.dst_address_length = dst_address_length;
456 a.adj_index = new_mp ? new_mp->adj_index : dst_adj_index;
460 ip4_add_del_route (im, &a);
465 clib_error_report (error);
469 ip4_get_route (ip4_main_t * im,
470 u32 table_index_or_table_id,
475 ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
476 u32 dst_address = * (u32 *) address;
479 ASSERT (address_length < ARRAY_LEN (im->fib_masks));
480 dst_address &= im->fib_masks[address_length];
482 hash = fib->adj_index_by_dst_address[address_length];
483 p = hash_get (hash, dst_address);
488 ip4_foreach_matching_route (ip4_main_t * im,
489 u32 table_index_or_table_id,
491 ip4_address_t * address,
493 ip4_address_t ** results,
494 u8 ** result_lengths)
496 ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
497 u32 dst_address = address->data_u32;
498 u32 this_length = address_length;
501 _vec_len (*results) = 0;
503 _vec_len (*result_lengths) = 0;
505 while (this_length <= 32 && vec_len (results) == 0)
508 hash_foreach (k, v, fib->adj_index_by_dst_address[this_length], ({
509 if (0 == ((k ^ dst_address) & im->fib_masks[address_length]))
513 vec_add1 (*results, a);
514 vec_add1 (*result_lengths, this_length);
522 void ip4_maybe_remap_adjacencies (ip4_main_t * im,
523 u32 table_index_or_table_id,
526 ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
527 ip_lookup_main_t * lm = &im->lookup_main;
530 ip4_add_del_route_callback_t * cb;
531 static ip4_address_t * to_delete;
533 if (lm->n_adjacency_remaps == 0)
536 for (l = 0; l <= 32; l++)
539 uword * hash = fib->adj_index_by_dst_address[l];
541 if (hash_elts (hash) == 0)
545 _vec_len (to_delete) = 0;
547 hash_foreach_pair (p, hash, ({
548 u32 adj_index = p->value[0];
549 u32 m = vec_elt (lm->adjacency_remap_table, adj_index);
553 /* Record destination address from hash key. */
556 /* New adjacency points to nothing: so delete prefix. */
558 vec_add1 (to_delete, a);
561 /* Remap to new adjacency. */
562 clib_memcpy (fib->old_hash_values, p->value, vec_bytes (fib->old_hash_values));
564 /* Set new adjacency value. */
565 fib->new_hash_values[0] = p->value[0] = m - 1;
567 vec_foreach (cb, im->add_del_route_callbacks)
568 if ((flags & cb->required_flags) == cb->required_flags)
569 cb->function (im, cb->function_opaque,
570 fib, flags | IP4_ROUTE_FLAG_ADD,
572 fib->old_hash_values,
573 fib->new_hash_values);
578 fib->new_hash_values[0] = ~0;
579 for (i = 0; i < vec_len (to_delete); i++)
581 hash = _hash_unset (hash, to_delete[i].data_u32, fib->old_hash_values);
582 vec_foreach (cb, im->add_del_route_callbacks)
583 if ((flags & cb->required_flags) == cb->required_flags)
584 cb->function (im, cb->function_opaque,
585 fib, flags | IP4_ROUTE_FLAG_DEL,
587 fib->old_hash_values,
588 fib->new_hash_values);
592 /* Also remap adjacencies in mtrie. */
593 ip4_mtrie_maybe_remap_adjacencies (lm, &fib->mtrie);
595 /* Reset mapping table. */
596 vec_zero (lm->adjacency_remap_table);
598 /* All remaps have been performed. */
599 lm->n_adjacency_remaps = 0;
602 void ip4_delete_matching_routes (ip4_main_t * im,
603 u32 table_index_or_table_id,
605 ip4_address_t * address,
608 static ip4_address_t * matching_addresses;
609 static u8 * matching_address_lengths;
611 ip4_add_del_route_args_t a;
613 a.flags = IP4_ROUTE_FLAG_DEL | IP4_ROUTE_FLAG_NO_REDISTRIBUTE | flags;
614 a.table_index_or_table_id = table_index_or_table_id;
619 for (l = address_length + 1; l <= 32; l++)
621 ip4_foreach_matching_route (im, table_index_or_table_id, flags,
625 &matching_address_lengths);
626 for (i = 0; i < vec_len (matching_addresses); i++)
628 a.dst_address = matching_addresses[i];
629 a.dst_address_length = matching_address_lengths[i];
630 ip4_add_del_route (im, &a);
634 ip4_maybe_remap_adjacencies (im, table_index_or_table_id, flags);
638 ip4_forward_next_trace (vlib_main_t * vm,
639 vlib_node_runtime_t * node,
640 vlib_frame_t * frame,
641 vlib_rx_or_tx_t which_adj_index);
644 ip4_lookup_inline (vlib_main_t * vm,
645 vlib_node_runtime_t * node,
646 vlib_frame_t * frame,
647 int lookup_for_responses_to_locally_received_packets,
650 ip4_main_t * im = &ip4_main;
651 ip_lookup_main_t * lm = &im->lookup_main;
652 vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
653 u32 n_left_from, n_left_to_next, * from, * to_next;
654 ip_lookup_next_t next;
655 u32 cpu_index = os_get_cpu_number();
657 from = vlib_frame_vector_args (frame);
658 n_left_from = frame->n_vectors;
659 next = node->cached_next_index;
661 while (n_left_from > 0)
663 vlib_get_next_frame (vm, node, next,
664 to_next, n_left_to_next);
666 while (n_left_from >= 4 && n_left_to_next >= 2)
668 vlib_buffer_t * p0, * p1;
669 ip4_header_t * ip0, * ip1;
670 __attribute__((unused)) tcp_header_t * tcp0, * tcp1;
671 ip_lookup_next_t next0, next1;
672 ip_adjacency_t * adj0, * adj1;
673 ip4_fib_mtrie_t * mtrie0, * mtrie1;
674 ip4_fib_mtrie_leaf_t leaf0, leaf1;
675 ip4_address_t * dst_addr0, *dst_addr1;
676 __attribute__((unused)) u32 pi0, fib_index0, adj_index0, is_tcp_udp0;
677 __attribute__((unused)) u32 pi1, fib_index1, adj_index1, is_tcp_udp1;
678 u32 flow_hash_config0, flow_hash_config1;
679 u32 hash_c0, hash_c1;
682 /* Prefetch next iteration. */
684 vlib_buffer_t * p2, * p3;
686 p2 = vlib_get_buffer (vm, from[2]);
687 p3 = vlib_get_buffer (vm, from[3]);
689 vlib_prefetch_buffer_header (p2, LOAD);
690 vlib_prefetch_buffer_header (p3, LOAD);
692 CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
693 CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
696 pi0 = to_next[0] = from[0];
697 pi1 = to_next[1] = from[1];
699 p0 = vlib_get_buffer (vm, pi0);
700 p1 = vlib_get_buffer (vm, pi1);
702 ip0 = vlib_buffer_get_current (p0);
703 ip1 = vlib_buffer_get_current (p1);
707 ip_adjacency_t * iadj0, * iadj1;
708 iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
709 iadj1 = ip_get_adjacency (lm, vnet_buffer(p1)->ip.adj_index[VLIB_TX]);
710 dst_addr0 = &iadj0->indirect.next_hop.ip4;
711 dst_addr1 = &iadj1->indirect.next_hop.ip4;
715 dst_addr0 = &ip0->dst_address;
716 dst_addr1 = &ip1->dst_address;
719 fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
720 fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
721 fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
722 fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
723 fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
724 fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
727 if (! lookup_for_responses_to_locally_received_packets)
729 mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
730 mtrie1 = &vec_elt_at_index (im->fibs, fib_index1)->mtrie;
732 leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
734 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
735 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 0);
738 tcp0 = (void *) (ip0 + 1);
739 tcp1 = (void *) (ip1 + 1);
741 is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
742 || ip0->protocol == IP_PROTOCOL_UDP);
743 is_tcp_udp1 = (ip1->protocol == IP_PROTOCOL_TCP
744 || ip1->protocol == IP_PROTOCOL_UDP);
746 if (! lookup_for_responses_to_locally_received_packets)
748 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
749 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 1);
752 if (! lookup_for_responses_to_locally_received_packets)
754 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
755 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
758 if (! lookup_for_responses_to_locally_received_packets)
760 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
761 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
764 if (lookup_for_responses_to_locally_received_packets)
766 adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
767 adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
771 /* Handle default route. */
772 leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
773 leaf1 = (leaf1 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
775 adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
776 adj_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
779 ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
781 /* no_default_route */ 0));
782 ASSERT (adj_index1 == ip4_fib_lookup_with_table (im, fib_index1,
784 /* no_default_route */ 0));
785 adj0 = ip_get_adjacency (lm, adj_index0);
786 adj1 = ip_get_adjacency (lm, adj_index1);
788 next0 = adj0->lookup_next_index;
789 next1 = adj1->lookup_next_index;
791 /* Use flow hash to compute multipath adjacency. */
792 hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
793 hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
794 if (PREDICT_FALSE (adj0->n_adj > 1))
797 vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
798 hash_c0 = vnet_buffer (p0)->ip.flow_hash =
799 ip4_compute_flow_hash (ip0, flow_hash_config0);
801 if (PREDICT_FALSE(adj1->n_adj > 1))
804 vec_elt_at_index (im->fibs, fib_index1)->flow_hash_config;
805 hash_c1 = vnet_buffer (p1)->ip.flow_hash =
806 ip4_compute_flow_hash (ip1, flow_hash_config1);
809 ASSERT (adj0->n_adj > 0);
810 ASSERT (adj1->n_adj > 0);
811 ASSERT (is_pow2 (adj0->n_adj));
812 ASSERT (is_pow2 (adj1->n_adj));
813 adj_index0 += (hash_c0 & (adj0->n_adj - 1));
814 adj_index1 += (hash_c1 & (adj1->n_adj - 1));
816 vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
817 vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
819 vlib_increment_combined_counter
820 (cm, cpu_index, adj_index0, 1,
821 vlib_buffer_length_in_chain (vm, p0)
822 + sizeof(ethernet_header_t));
823 vlib_increment_combined_counter
824 (cm, cpu_index, adj_index1, 1,
825 vlib_buffer_length_in_chain (vm, p1)
826 + sizeof(ethernet_header_t));
833 wrong_next = (next0 != next) + 2*(next1 != next);
834 if (PREDICT_FALSE (wrong_next != 0))
843 vlib_set_next_frame_buffer (vm, node, next0, pi0);
850 vlib_set_next_frame_buffer (vm, node, next1, pi1);
857 vlib_set_next_frame_buffer (vm, node, next0, pi0);
858 vlib_set_next_frame_buffer (vm, node, next1, pi1);
862 vlib_put_next_frame (vm, node, next, n_left_to_next);
864 vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
870 while (n_left_from > 0 && n_left_to_next > 0)
874 __attribute__((unused)) tcp_header_t * tcp0;
875 ip_lookup_next_t next0;
876 ip_adjacency_t * adj0;
877 ip4_fib_mtrie_t * mtrie0;
878 ip4_fib_mtrie_leaf_t leaf0;
879 ip4_address_t * dst_addr0;
880 __attribute__((unused)) u32 pi0, fib_index0, adj_index0, is_tcp_udp0;
881 u32 flow_hash_config0, hash_c0;
886 p0 = vlib_get_buffer (vm, pi0);
888 ip0 = vlib_buffer_get_current (p0);
892 ip_adjacency_t * iadj0;
893 iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
894 dst_addr0 = &iadj0->indirect.next_hop.ip4;
898 dst_addr0 = &ip0->dst_address;
901 fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
902 fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
903 fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
905 if (! lookup_for_responses_to_locally_received_packets)
907 mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
909 leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
911 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
914 tcp0 = (void *) (ip0 + 1);
916 is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
917 || ip0->protocol == IP_PROTOCOL_UDP);
919 if (! lookup_for_responses_to_locally_received_packets)
920 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
922 if (! lookup_for_responses_to_locally_received_packets)
923 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
925 if (! lookup_for_responses_to_locally_received_packets)
926 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
928 if (lookup_for_responses_to_locally_received_packets)
929 adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
932 /* Handle default route. */
933 leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
934 adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
937 ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
939 /* no_default_route */ 0));
941 adj0 = ip_get_adjacency (lm, adj_index0);
943 next0 = adj0->lookup_next_index;
945 /* Use flow hash to compute multipath adjacency. */
946 hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
947 if (PREDICT_FALSE(adj0->n_adj > 1))
950 vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
952 hash_c0 = vnet_buffer (p0)->ip.flow_hash =
953 ip4_compute_flow_hash (ip0, flow_hash_config0);
956 ASSERT (adj0->n_adj > 0);
957 ASSERT (is_pow2 (adj0->n_adj));
958 adj_index0 += (hash_c0 & (adj0->n_adj - 1));
960 vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
962 vlib_increment_combined_counter
963 (cm, cpu_index, adj_index0, 1,
964 vlib_buffer_length_in_chain (vm, p0)
965 + sizeof(ethernet_header_t));
972 if (PREDICT_FALSE (next0 != next))
975 vlib_put_next_frame (vm, node, next, n_left_to_next);
977 vlib_get_next_frame (vm, node, next,
978 to_next, n_left_to_next);
985 vlib_put_next_frame (vm, node, next, n_left_to_next);
988 if (node->flags & VLIB_NODE_FLAG_TRACE)
989 ip4_forward_next_trace(vm, node, frame, VLIB_TX);
991 return frame->n_vectors;
995 ip4_lookup (vlib_main_t * vm,
996 vlib_node_runtime_t * node,
997 vlib_frame_t * frame)
999 return ip4_lookup_inline (vm, node, frame,
1000 /* lookup_for_responses_to_locally_received_packets */ 0,
1001 /* is_indirect */ 0);
1005 void ip4_adjacency_set_interface_route (vnet_main_t * vnm,
1006 ip_adjacency_t * adj,
1008 u32 if_address_index)
1010 vnet_hw_interface_t * hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
1012 vnet_l3_packet_type_t packet_type;
1015 if (hw->hw_class_index == ethernet_hw_interface_class.index
1016 || hw->hw_class_index == srp_hw_interface_class.index)
1019 * We have a bit of a problem in this case. ip4-arp uses
1020 * the rewrite_header.next_index to hand pkts to the
1021 * indicated inteface output node. We can end up in
1022 * ip4_rewrite_local, too, which also pays attention to
1023 * rewrite_header.next index. Net result: a hack in
1024 * ip4_rewrite_local...
1026 n = IP_LOOKUP_NEXT_ARP;
1027 node_index = ip4_arp_node.index;
1028 adj->if_address_index = if_address_index;
1029 adj->arp.next_hop.ip4.as_u32 = 0;
1030 ip46_address_reset(&adj->arp.next_hop);
1031 packet_type = VNET_L3_PACKET_TYPE_ARP;
1035 n = IP_LOOKUP_NEXT_REWRITE;
1036 node_index = ip4_rewrite_node.index;
1037 packet_type = VNET_L3_PACKET_TYPE_IP4;
1040 adj->lookup_next_index = n;
1041 vnet_rewrite_for_sw_interface
1046 VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST,
1047 &adj->rewrite_header,
1048 sizeof (adj->rewrite_data));
1052 ip4_add_interface_routes (u32 sw_if_index,
1053 ip4_main_t * im, u32 fib_index,
1054 ip_interface_address_t * a)
1056 vnet_main_t * vnm = vnet_get_main();
1057 ip_lookup_main_t * lm = &im->lookup_main;
1058 ip_adjacency_t * adj;
1059 ip4_address_t * address = ip_interface_address_get_address (lm, a);
1060 ip4_add_del_route_args_t x;
1061 vnet_hw_interface_t * hw_if = vnet_get_sup_hw_interface (vnm, sw_if_index);
1062 u32 classify_table_index;
1064 /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1065 x.table_index_or_table_id = fib_index;
1066 x.flags = (IP4_ROUTE_FLAG_ADD
1067 | IP4_ROUTE_FLAG_FIB_INDEX
1068 | IP4_ROUTE_FLAG_NO_REDISTRIBUTE);
1069 x.dst_address = address[0];
1070 x.dst_address_length = a->address_length;
1074 a->neighbor_probe_adj_index = ~0;
1075 if (a->address_length < 32)
1077 adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1079 ip4_adjacency_set_interface_route (vnm, adj, sw_if_index, a - lm->if_address_pool);
1080 ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1081 ip4_add_del_route (im, &x);
1082 a->neighbor_probe_adj_index = x.adj_index;
1085 /* Add e.g. 1.1.1.1/32 as local to this host. */
1086 adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1089 classify_table_index = ~0;
1090 if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
1091 classify_table_index = lm->classify_table_index_by_sw_if_index [sw_if_index];
1092 if (classify_table_index != (u32) ~0)
1094 adj->lookup_next_index = IP_LOOKUP_NEXT_CLASSIFY;
1095 adj->classify.table_index = classify_table_index;
1098 adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
1100 adj->if_address_index = a - lm->if_address_pool;
1101 adj->rewrite_header.sw_if_index = sw_if_index;
1102 adj->rewrite_header.max_l3_packet_bytes = hw_if->max_l3_packet_bytes[VLIB_RX];
1104 * Local adjs are never to be rewritten. Spoofed pkts w/ src = dst = local
1105 * fail an RPF-ish check, but still go thru the rewrite code...
1107 adj->rewrite_header.data_bytes = 0;
1109 ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1110 x.dst_address_length = 32;
1111 ip4_add_del_route (im, &x);
1115 ip4_del_interface_routes (ip4_main_t * im, u32 fib_index, ip4_address_t * address, u32 address_length)
1117 ip4_add_del_route_args_t x;
1119 /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1120 x.table_index_or_table_id = fib_index;
1121 x.flags = (IP4_ROUTE_FLAG_DEL
1122 | IP4_ROUTE_FLAG_FIB_INDEX
1123 | IP4_ROUTE_FLAG_NO_REDISTRIBUTE);
1124 x.dst_address = address[0];
1125 x.dst_address_length = address_length;
1130 if (address_length < 32)
1131 ip4_add_del_route (im, &x);
1133 x.dst_address_length = 32;
1134 ip4_add_del_route (im, &x);
1136 ip4_delete_matching_routes (im,
1138 IP4_ROUTE_FLAG_FIB_INDEX,
1145 ip4_address_t address;
1147 } ip4_interface_address_t;
1149 static clib_error_t *
1150 ip4_add_del_interface_address_internal (vlib_main_t * vm,
1152 ip4_address_t * new_address,
1158 static clib_error_t *
1159 ip4_add_del_interface_address_internal (vlib_main_t * vm,
1161 ip4_address_t * address,
1167 vnet_main_t * vnm = vnet_get_main();
1168 ip4_main_t * im = &ip4_main;
1169 ip_lookup_main_t * lm = &im->lookup_main;
1170 clib_error_t * error = 0;
1171 u32 if_address_index, elts_before;
1172 ip4_address_fib_t ip4_af, * addr_fib = 0;
1174 vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1175 ip4_addr_fib_init (&ip4_af, address,
1176 vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
1177 vec_add1 (addr_fib, ip4_af);
1179 /* When adding an address check that it does not conflict with an existing address. */
1182 ip_interface_address_t * ia;
1183 foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1184 0 /* honor unnumbered */,
1186 ip4_address_t * x = ip_interface_address_get_address (&im->lookup_main, ia);
1188 if (ip4_destination_matches_route (im, address, x, ia->address_length)
1189 || ip4_destination_matches_route (im, x, address, address_length))
1190 return clib_error_create ("failed to add %U which conflicts with %U for interface %U",
1191 format_ip4_address_and_length, address, address_length,
1192 format_ip4_address_and_length, x, ia->address_length,
1193 format_vnet_sw_if_index_name, vnm, sw_if_index);
1197 elts_before = pool_elts (lm->if_address_pool);
1199 error = ip_interface_address_add_del
1209 if (vnet_sw_interface_is_admin_up (vnm, sw_if_index) && insert_routes)
1212 ip4_del_interface_routes (im, ip4_af.fib_index, address,
1216 ip4_add_interface_routes (sw_if_index,
1217 im, ip4_af.fib_index,
1219 (lm->if_address_pool, if_address_index));
1222 /* If pool did not grow/shrink: add duplicate address. */
1223 if (elts_before != pool_elts (lm->if_address_pool))
1225 ip4_add_del_interface_address_callback_t * cb;
1226 vec_foreach (cb, im->add_del_interface_address_callbacks)
1227 cb->function (im, cb->function_opaque, sw_if_index,
1228 address, address_length,
1234 vec_free (addr_fib);
1239 ip4_add_del_interface_address (vlib_main_t * vm, u32 sw_if_index,
1240 ip4_address_t * address, u32 address_length,
1243 return ip4_add_del_interface_address_internal
1244 (vm, sw_if_index, address, address_length,
1245 /* redistribute */ 1,
1246 /* insert_routes */ 1,
1250 static clib_error_t *
1251 ip4_sw_interface_admin_up_down (vnet_main_t * vnm,
1255 ip4_main_t * im = &ip4_main;
1256 ip_interface_address_t * ia;
1258 u32 is_admin_up, fib_index;
1260 /* Fill in lookup tables with default table (0). */
1261 vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1263 vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
1265 is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1267 fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
1269 foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1270 0 /* honor unnumbered */,
1272 a = ip_interface_address_get_address (&im->lookup_main, ia);
1274 ip4_add_interface_routes (sw_if_index,
1278 ip4_del_interface_routes (im, fib_index,
1279 a, ia->address_length);
1285 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip4_sw_interface_admin_up_down);
1287 /* Built-in ip4 unicast rx feature path definition */
1288 VNET_IP4_UNICAST_FEATURE_INIT (ip4_inacl, static) = {
1289 .node_name = "ip4-inacl",
1290 .runs_before = {"ip4-source-check-via-rx", 0},
1291 .feature_index = &ip4_main.ip4_unicast_rx_feature_check_access,
1294 VNET_IP4_UNICAST_FEATURE_INIT (ip4_source_check_1, static) = {
1295 .node_name = "ip4-source-check-via-rx",
1296 .runs_before = {"ip4-source-check-via-any", 0},
1298 &ip4_main.ip4_unicast_rx_feature_source_reachable_via_rx,
1301 VNET_IP4_UNICAST_FEATURE_INIT (ip4_source_check_2, static) = {
1302 .node_name = "ip4-source-check-via-any",
1303 .runs_before = {"ipsec-input-ip4", 0},
1305 &ip4_main.ip4_unicast_rx_feature_source_reachable_via_any,
1308 VNET_IP4_UNICAST_FEATURE_INIT (ip4_ipsec, static) = {
1309 .node_name = "ipsec-input-ip4",
1310 .runs_before = {"vpath-input-ip4", 0},
1311 .feature_index = &ip4_main.ip4_unicast_rx_feature_ipsec,
1314 VNET_IP4_UNICAST_FEATURE_INIT (ip4_vpath, static) = {
1315 .node_name = "vpath-input-ip4",
1316 .runs_before = {"ip4-lookup", 0},
1317 .feature_index = &ip4_main.ip4_unicast_rx_feature_vpath,
1320 VNET_IP4_UNICAST_FEATURE_INIT (ip4_lookup, static) = {
1321 .node_name = "ip4-lookup",
1322 .runs_before = {0}, /* not before any other features */
1323 .feature_index = &ip4_main.ip4_unicast_rx_feature_lookup,
1326 /* Built-in ip4 multicast rx feature path definition */
1327 VNET_IP4_MULTICAST_FEATURE_INIT (ip4_vpath_mc, static) = {
1328 .node_name = "vpath-input-ip4",
1329 .runs_before = {"ip4-lookup-multicast", 0},
1330 .feature_index = &ip4_main.ip4_multicast_rx_feature_vpath,
1333 VNET_IP4_MULTICAST_FEATURE_INIT (ip4_lookup_mc, static) = {
1334 .node_name = "ip4-lookup-multicast",
1335 .runs_before = {0}, /* not before any other features */
1336 .feature_index = &ip4_main.ip4_multicast_rx_feature_lookup,
1339 static char * feature_start_nodes[] =
1340 { "ip4-input", "ip4-input-no-checksum"};
1342 static clib_error_t *
1343 ip4_feature_init (vlib_main_t * vm, ip4_main_t * im)
1345 ip_lookup_main_t * lm = &im->lookup_main;
1346 clib_error_t * error;
1349 for (cast = 0; cast < VNET_N_CAST; cast++)
1351 ip_config_main_t * cm = &lm->rx_config_mains[cast];
1352 vnet_config_main_t * vcm = &cm->config_main;
1354 if ((error = ip_feature_init_cast (vm, cm, vcm,
1355 feature_start_nodes,
1356 ARRAY_LEN(feature_start_nodes),
1364 static clib_error_t *
1365 ip4_sw_interface_add_del (vnet_main_t * vnm,
1369 vlib_main_t * vm = vnm->vlib_main;
1370 ip4_main_t * im = &ip4_main;
1371 ip_lookup_main_t * lm = &im->lookup_main;
1375 for (cast = 0; cast < VNET_N_CAST; cast++)
1377 ip_config_main_t * cm = &lm->rx_config_mains[cast];
1378 vnet_config_main_t * vcm = &cm->config_main;
1380 vec_validate_init_empty (cm->config_index_by_sw_if_index, sw_if_index, ~0);
1381 ci = cm->config_index_by_sw_if_index[sw_if_index];
1383 if (cast == VNET_UNICAST)
1384 feature_index = im->ip4_unicast_rx_feature_lookup;
1386 feature_index = im->ip4_multicast_rx_feature_lookup;
1389 ci = vnet_config_add_feature (vm, vcm,
1392 /* config data */ 0,
1393 /* # bytes of config data */ 0);
1395 ci = vnet_config_del_feature (vm, vcm,
1398 /* config data */ 0,
1399 /* # bytes of config data */ 0);
1401 cm->config_index_by_sw_if_index[sw_if_index] = ci;
1404 return /* no error */ 0;
1407 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
1409 static u8 * format_ip4_lookup_trace (u8 * s, va_list * args);
1411 VLIB_REGISTER_NODE (ip4_lookup_node) = {
1412 .function = ip4_lookup,
1413 .name = "ip4-lookup",
1414 .vector_size = sizeof (u32),
1416 .format_trace = format_ip4_lookup_trace,
1418 .n_next_nodes = IP4_LOOKUP_N_NEXT,
1419 .next_nodes = IP4_LOOKUP_NEXT_NODES,
1422 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup)
1425 ip4_indirect (vlib_main_t * vm,
1426 vlib_node_runtime_t * node,
1427 vlib_frame_t * frame)
1429 return ip4_lookup_inline (vm, node, frame,
1430 /* lookup_for_responses_to_locally_received_packets */ 0,
1431 /* is_indirect */ 1);
1434 VLIB_REGISTER_NODE (ip4_indirect_node) = {
1435 .function = ip4_indirect,
1436 .name = "ip4-indirect",
1437 .vector_size = sizeof (u32),
1438 .sibling_of = "ip4-lookup",
1439 .format_trace = format_ip4_lookup_trace,
1444 VLIB_NODE_FUNCTION_MULTIARCH (ip4_indirect_node, ip4_indirect)
1447 /* Global IP4 main. */
1448 ip4_main_t ip4_main;
1451 ip4_lookup_init (vlib_main_t * vm)
1453 ip4_main_t * im = &ip4_main;
1454 clib_error_t * error;
1457 for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
1462 m = pow2_mask (i) << (32 - i);
1465 im->fib_masks[i] = clib_host_to_net_u32 (m);
1468 /* Create FIB with index 0 and table id of 0. */
1469 find_ip4_fib_by_table_index_or_id (im, /* table id */ 0, IP4_ROUTE_FLAG_TABLE_ID);
1471 ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
1475 pn = pg_get_node (ip4_lookup_node.index);
1476 pn->unformat_edit = unformat_pg_ip4_header;
1480 ethernet_arp_header_t h;
1482 memset (&h, 0, sizeof (h));
1484 /* Set target ethernet address to all zeros. */
1485 memset (h.ip4_over_ethernet[1].ethernet, 0, sizeof (h.ip4_over_ethernet[1].ethernet));
1487 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
1488 #define _8(f,v) h.f = v;
1489 _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
1490 _16 (l3_type, ETHERNET_TYPE_IP4);
1491 _8 (n_l2_address_bytes, 6);
1492 _8 (n_l3_address_bytes, 4);
1493 _16 (opcode, ETHERNET_ARP_OPCODE_request);
1497 vlib_packet_template_init (vm,
1498 &im->ip4_arp_request_packet_template,
1501 /* alloc chunk size */ 8,
1505 error = ip4_feature_init (vm, im);
1510 VLIB_INIT_FUNCTION (ip4_lookup_init);
1513 /* Adjacency taken. */
1518 /* Packet data, possibly *after* rewrite. */
1519 u8 packet_data[64 - 1*sizeof(u32)];
1520 } ip4_forward_next_trace_t;
1522 static u8 * format_ip4_forward_next_trace (u8 * s, va_list * args)
1524 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1525 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1526 ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1527 uword indent = format_get_indent (s);
1528 s = format (s, "%U%U",
1529 format_white_space, indent,
1530 format_ip4_header, t->packet_data);
1534 static u8 * format_ip4_lookup_trace (u8 * s, va_list * args)
1536 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1537 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1538 ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1539 vnet_main_t * vnm = vnet_get_main();
1540 ip4_main_t * im = &ip4_main;
1541 uword indent = format_get_indent (s);
1543 s = format (s, "fib %d adj-idx %d : %U flow hash: 0x%08x",
1544 t->fib_index, t->adj_index, format_ip_adjacency,
1545 vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1546 s = format (s, "\n%U%U",
1547 format_white_space, indent,
1548 format_ip4_header, t->packet_data);
1552 static u8 * format_ip4_rewrite_trace (u8 * s, va_list * args)
1554 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1555 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1556 ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1557 vnet_main_t * vnm = vnet_get_main();
1558 ip4_main_t * im = &ip4_main;
1559 uword indent = format_get_indent (s);
1561 s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
1562 t->fib_index, t->adj_index, format_ip_adjacency,
1563 vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1564 s = format (s, "\n%U%U",
1565 format_white_space, indent,
1566 format_ip_adjacency_packet_data,
1567 vnm, &im->lookup_main, t->adj_index,
1568 t->packet_data, sizeof (t->packet_data));
1572 /* Common trace function for all ip4-forward next nodes. */
1574 ip4_forward_next_trace (vlib_main_t * vm,
1575 vlib_node_runtime_t * node,
1576 vlib_frame_t * frame,
1577 vlib_rx_or_tx_t which_adj_index)
1580 ip4_main_t * im = &ip4_main;
1582 n_left = frame->n_vectors;
1583 from = vlib_frame_vector_args (frame);
1588 vlib_buffer_t * b0, * b1;
1589 ip4_forward_next_trace_t * t0, * t1;
1591 /* Prefetch next iteration. */
1592 vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1593 vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1598 b0 = vlib_get_buffer (vm, bi0);
1599 b1 = vlib_get_buffer (vm, bi1);
1601 if (b0->flags & VLIB_BUFFER_IS_TRACED)
1603 t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1604 t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1605 t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1606 t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1607 vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1608 vec_elt (im->fib_index_by_sw_if_index,
1609 vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1611 clib_memcpy (t0->packet_data,
1612 vlib_buffer_get_current (b0),
1613 sizeof (t0->packet_data));
1615 if (b1->flags & VLIB_BUFFER_IS_TRACED)
1617 t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1618 t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1619 t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1620 t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
1621 vnet_buffer(b1)->sw_if_index[VLIB_TX] :
1622 vec_elt (im->fib_index_by_sw_if_index,
1623 vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1624 clib_memcpy (t1->packet_data,
1625 vlib_buffer_get_current (b1),
1626 sizeof (t1->packet_data));
1636 ip4_forward_next_trace_t * t0;
1640 b0 = vlib_get_buffer (vm, bi0);
1642 if (b0->flags & VLIB_BUFFER_IS_TRACED)
1644 t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1645 t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1646 t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1647 t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1648 vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1649 vec_elt (im->fib_index_by_sw_if_index,
1650 vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1651 clib_memcpy (t0->packet_data,
1652 vlib_buffer_get_current (b0),
1653 sizeof (t0->packet_data));
1661 ip4_drop_or_punt (vlib_main_t * vm,
1662 vlib_node_runtime_t * node,
1663 vlib_frame_t * frame,
1664 ip4_error_t error_code)
1666 u32 * buffers = vlib_frame_vector_args (frame);
1667 uword n_packets = frame->n_vectors;
1669 vlib_error_drop_buffers (vm, node,
1674 ip4_input_node.index,
1677 if (node->flags & VLIB_NODE_FLAG_TRACE)
1678 ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1684 ip4_drop (vlib_main_t * vm,
1685 vlib_node_runtime_t * node,
1686 vlib_frame_t * frame)
1687 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_DROP); }
1690 ip4_punt (vlib_main_t * vm,
1691 vlib_node_runtime_t * node,
1692 vlib_frame_t * frame)
1693 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_PUNT); }
1696 ip4_miss (vlib_main_t * vm,
1697 vlib_node_runtime_t * node,
1698 vlib_frame_t * frame)
1699 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_DST_LOOKUP_MISS); }
1701 VLIB_REGISTER_NODE (ip4_drop_node,static) = {
1702 .function = ip4_drop,
1704 .vector_size = sizeof (u32),
1706 .format_trace = format_ip4_forward_next_trace,
1714 VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop)
1716 VLIB_REGISTER_NODE (ip4_punt_node,static) = {
1717 .function = ip4_punt,
1719 .vector_size = sizeof (u32),
1721 .format_trace = format_ip4_forward_next_trace,
1729 VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt)
1731 VLIB_REGISTER_NODE (ip4_miss_node,static) = {
1732 .function = ip4_miss,
1734 .vector_size = sizeof (u32),
1736 .format_trace = format_ip4_forward_next_trace,
1744 VLIB_NODE_FUNCTION_MULTIARCH (ip4_miss_node, ip4_miss)
1746 /* Compute TCP/UDP/ICMP4 checksum in software. */
1748 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1752 u32 ip_header_length, payload_length_host_byte_order;
1753 u32 n_this_buffer, n_bytes_left;
1755 void * data_this_buffer;
1757 /* Initialize checksum with ip header. */
1758 ip_header_length = ip4_header_bytes (ip0);
1759 payload_length_host_byte_order = clib_net_to_host_u16 (ip0->length) - ip_header_length;
1760 sum0 = clib_host_to_net_u32 (payload_length_host_byte_order + (ip0->protocol << 16));
1762 if (BITS (uword) == 32)
1764 sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u32));
1765 sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->dst_address, u32));
1768 sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1770 n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1771 data_this_buffer = (void *) ip0 + ip_header_length;
1772 if (n_this_buffer + ip_header_length > p0->current_length)
1773 n_this_buffer = p0->current_length > ip_header_length ? p0->current_length - ip_header_length : 0;
1776 sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1777 n_bytes_left -= n_this_buffer;
1778 if (n_bytes_left == 0)
1781 ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1782 p0 = vlib_get_buffer (vm, p0->next_buffer);
1783 data_this_buffer = vlib_buffer_get_current (p0);
1784 n_this_buffer = p0->current_length;
1787 sum16 = ~ ip_csum_fold (sum0);
1793 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1795 ip4_header_t * ip0 = vlib_buffer_get_current (p0);
1796 udp_header_t * udp0;
1799 ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1800 || ip0->protocol == IP_PROTOCOL_UDP);
1802 udp0 = (void *) (ip0 + 1);
1803 if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1805 p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1806 | IP_BUFFER_L4_CHECKSUM_CORRECT);
1810 sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1812 p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1813 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1819 ip4_local (vlib_main_t * vm,
1820 vlib_node_runtime_t * node,
1821 vlib_frame_t * frame)
1823 ip4_main_t * im = &ip4_main;
1824 ip_lookup_main_t * lm = &im->lookup_main;
1825 ip_local_next_t next_index;
1826 u32 * from, * to_next, n_left_from, n_left_to_next;
1827 vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip4_input_node.index);
1829 from = vlib_frame_vector_args (frame);
1830 n_left_from = frame->n_vectors;
1831 next_index = node->cached_next_index;
1833 if (node->flags & VLIB_NODE_FLAG_TRACE)
1834 ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1836 while (n_left_from > 0)
1838 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1840 while (n_left_from >= 4 && n_left_to_next >= 2)
1842 vlib_buffer_t * p0, * p1;
1843 ip4_header_t * ip0, * ip1;
1844 udp_header_t * udp0, * udp1;
1845 ip4_fib_mtrie_t * mtrie0, * mtrie1;
1846 ip4_fib_mtrie_leaf_t leaf0, leaf1;
1847 ip_adjacency_t * adj0, * adj1;
1848 u32 pi0, ip_len0, udp_len0, flags0, next0, fib_index0, adj_index0;
1849 u32 pi1, ip_len1, udp_len1, flags1, next1, fib_index1, adj_index1;
1850 i32 len_diff0, len_diff1;
1851 u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1852 u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1855 pi0 = to_next[0] = from[0];
1856 pi1 = to_next[1] = from[1];
1860 n_left_to_next -= 2;
1862 p0 = vlib_get_buffer (vm, pi0);
1863 p1 = vlib_get_buffer (vm, pi1);
1865 ip0 = vlib_buffer_get_current (p0);
1866 ip1 = vlib_buffer_get_current (p1);
1868 fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
1869 vnet_buffer(p0)->sw_if_index[VLIB_RX]);
1870 fib_index1 = vec_elt (im->fib_index_by_sw_if_index,
1871 vnet_buffer(p1)->sw_if_index[VLIB_RX]);
1873 mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
1874 mtrie1 = &vec_elt_at_index (im->fibs, fib_index1)->mtrie;
1876 leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
1878 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
1879 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 0);
1881 /* Treat IP frag packets as "experimental" protocol for now
1882 until support of IP frag reassembly is implemented */
1883 proto0 = ip4_is_fragment(ip0) ? 0xfe : ip0->protocol;
1884 proto1 = ip4_is_fragment(ip1) ? 0xfe : ip1->protocol;
1885 is_udp0 = proto0 == IP_PROTOCOL_UDP;
1886 is_udp1 = proto1 == IP_PROTOCOL_UDP;
1887 is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1888 is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1893 good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1894 good_tcp_udp1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1896 udp0 = ip4_next_header (ip0);
1897 udp1 = ip4_next_header (ip1);
1899 /* Don't verify UDP checksum for packets with explicit zero checksum. */
1900 good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1901 good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1903 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
1904 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 1);
1906 /* Verify UDP length. */
1907 ip_len0 = clib_net_to_host_u16 (ip0->length);
1908 ip_len1 = clib_net_to_host_u16 (ip1->length);
1909 udp_len0 = clib_net_to_host_u16 (udp0->length);
1910 udp_len1 = clib_net_to_host_u16 (udp1->length);
1912 len_diff0 = ip_len0 - udp_len0;
1913 len_diff1 = ip_len1 - udp_len1;
1915 len_diff0 = is_udp0 ? len_diff0 : 0;
1916 len_diff1 = is_udp1 ? len_diff1 : 0;
1918 if (PREDICT_FALSE (! (is_tcp_udp0 & is_tcp_udp1
1919 & good_tcp_udp0 & good_tcp_udp1)))
1924 && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1925 flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1927 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1928 good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1933 && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1934 flags1 = ip4_tcp_udp_validate_checksum (vm, p1);
1936 (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1937 good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1941 good_tcp_udp0 &= len_diff0 >= 0;
1942 good_tcp_udp1 &= len_diff1 >= 0;
1944 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1945 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 2);
1947 error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1949 error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1950 error1 = len_diff1 < 0 ? IP4_ERROR_UDP_LENGTH : error1;
1952 ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1953 error0 = (is_tcp_udp0 && ! good_tcp_udp0
1954 ? IP4_ERROR_TCP_CHECKSUM + is_udp0
1956 error1 = (is_tcp_udp1 && ! good_tcp_udp1
1957 ? IP4_ERROR_TCP_CHECKSUM + is_udp1
1960 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1961 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 3);
1963 vnet_buffer (p0)->ip.adj_index[VLIB_RX] = adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1964 vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
1966 vnet_buffer (p1)->ip.adj_index[VLIB_RX] = adj_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1967 vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
1969 ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
1971 /* no_default_route */ 1));
1972 ASSERT (adj_index1 == ip4_fib_lookup_with_table (im, fib_index1,
1974 /* no_default_route */ 1));
1976 adj0 = ip_get_adjacency (lm, adj_index0);
1977 adj1 = ip_get_adjacency (lm, adj_index1);
1980 * Must have a route to source otherwise we drop the packet.
1981 * ip4 broadcasts are accepted, e.g. to make dhcp client work
1983 error0 = (error0 == IP4_ERROR_UNKNOWN_PROTOCOL
1984 && adj0->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
1985 && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP
1986 && adj0->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
1987 && ip0->dst_address.as_u32 != 0xFFFFFFFF
1988 ? IP4_ERROR_SRC_LOOKUP_MISS
1990 error1 = (error1 == IP4_ERROR_UNKNOWN_PROTOCOL
1991 && adj1->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
1992 && adj1->lookup_next_index != IP_LOOKUP_NEXT_ARP
1993 && adj1->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
1994 && ip0->dst_address.as_u32 != 0xFFFFFFFF
1995 ? IP4_ERROR_SRC_LOOKUP_MISS
1998 next0 = lm->local_next_by_ip_protocol[proto0];
1999 next1 = lm->local_next_by_ip_protocol[proto1];
2001 next0 = error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
2002 next1 = error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
2004 p0->error = error0 ? error_node->errors[error0] : 0;
2005 p1->error = error1 ? error_node->errors[error1] : 0;
2007 enqueue_code = (next0 != next_index) + 2*(next1 != next_index);
2009 if (PREDICT_FALSE (enqueue_code != 0))
2011 switch (enqueue_code)
2017 n_left_to_next += 1;
2018 vlib_set_next_frame_buffer (vm, node, next0, pi0);
2024 n_left_to_next += 1;
2025 vlib_set_next_frame_buffer (vm, node, next1, pi1);
2029 /* A B B or A B C */
2031 n_left_to_next += 2;
2032 vlib_set_next_frame_buffer (vm, node, next0, pi0);
2033 vlib_set_next_frame_buffer (vm, node, next1, pi1);
2036 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2038 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2045 while (n_left_from > 0 && n_left_to_next > 0)
2049 udp_header_t * udp0;
2050 ip4_fib_mtrie_t * mtrie0;
2051 ip4_fib_mtrie_leaf_t leaf0;
2052 ip_adjacency_t * adj0;
2053 u32 pi0, next0, ip_len0, udp_len0, flags0, fib_index0, adj_index0;
2055 u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
2057 pi0 = to_next[0] = from[0];
2061 n_left_to_next -= 1;
2063 p0 = vlib_get_buffer (vm, pi0);
2065 ip0 = vlib_buffer_get_current (p0);
2067 fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
2068 vnet_buffer(p0)->sw_if_index[VLIB_RX]);
2070 mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
2072 leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
2074 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
2076 /* Treat IP frag packets as "experimental" protocol for now
2077 until support of IP frag reassembly is implemented */
2078 proto0 = ip4_is_fragment(ip0) ? 0xfe : ip0->protocol;
2079 is_udp0 = proto0 == IP_PROTOCOL_UDP;
2080 is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
2084 good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
2086 udp0 = ip4_next_header (ip0);
2088 /* Don't verify UDP checksum for packets with explicit zero checksum. */
2089 good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
2091 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
2093 /* Verify UDP length. */
2094 ip_len0 = clib_net_to_host_u16 (ip0->length);
2095 udp_len0 = clib_net_to_host_u16 (udp0->length);
2097 len_diff0 = ip_len0 - udp_len0;
2099 len_diff0 = is_udp0 ? len_diff0 : 0;
2101 if (PREDICT_FALSE (! (is_tcp_udp0 & good_tcp_udp0)))
2106 && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
2107 flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
2109 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
2110 good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
2114 good_tcp_udp0 &= len_diff0 >= 0;
2116 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
2118 error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
2120 error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
2122 ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
2123 error0 = (is_tcp_udp0 && ! good_tcp_udp0
2124 ? IP4_ERROR_TCP_CHECKSUM + is_udp0
2127 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
2129 vnet_buffer (p0)->ip.adj_index[VLIB_RX] = adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2130 vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
2132 ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
2134 /* no_default_route */ 1));
2136 adj0 = ip_get_adjacency (lm, adj_index0);
2138 /* Must have a route to source otherwise we drop the packet. */
2139 error0 = (error0 == IP4_ERROR_UNKNOWN_PROTOCOL
2140 && adj0->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
2141 && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP
2142 && adj0->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
2143 && ip0->dst_address.as_u32 != 0xFFFFFFFF
2144 ? IP4_ERROR_SRC_LOOKUP_MISS
2147 next0 = lm->local_next_by_ip_protocol[proto0];
2149 next0 = error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
2151 p0->error = error0? error_node->errors[error0] : 0;
2153 if (PREDICT_FALSE (next0 != next_index))
2155 n_left_to_next += 1;
2156 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2159 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2162 n_left_to_next -= 1;
2166 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2169 return frame->n_vectors;
2172 VLIB_REGISTER_NODE (ip4_local_node,static) = {
2173 .function = ip4_local,
2174 .name = "ip4-local",
2175 .vector_size = sizeof (u32),
2177 .format_trace = format_ip4_forward_next_trace,
2179 .n_next_nodes = IP_LOCAL_N_NEXT,
2181 [IP_LOCAL_NEXT_DROP] = "error-drop",
2182 [IP_LOCAL_NEXT_PUNT] = "error-punt",
2183 [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
2184 [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",
2188 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local)
2190 void ip4_register_protocol (u32 protocol, u32 node_index)
2192 vlib_main_t * vm = vlib_get_main();
2193 ip4_main_t * im = &ip4_main;
2194 ip_lookup_main_t * lm = &im->lookup_main;
2196 ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
2197 lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip4_local_node.index, node_index);
2200 static clib_error_t *
2201 show_ip_local_command_fn (vlib_main_t * vm,
2202 unformat_input_t * input,
2203 vlib_cli_command_t * cmd)
2205 ip4_main_t * im = &ip4_main;
2206 ip_lookup_main_t * lm = &im->lookup_main;
2209 vlib_cli_output (vm, "Protocols handled by ip4_local");
2210 for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
2212 if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
2213 vlib_cli_output (vm, "%d", i);
2220 VLIB_CLI_COMMAND (show_ip_local, static) = {
2221 .path = "show ip local",
2222 .function = show_ip_local_command_fn,
2223 .short_help = "Show ip local protocol table",
2227 ip4_arp (vlib_main_t * vm,
2228 vlib_node_runtime_t * node,
2229 vlib_frame_t * frame)
2231 vnet_main_t * vnm = vnet_get_main();
2232 ip4_main_t * im = &ip4_main;
2233 ip_lookup_main_t * lm = &im->lookup_main;
2234 u32 * from, * to_next_drop;
2235 uword n_left_from, n_left_to_next_drop, next_index;
2236 static f64 time_last_seed_change = -1e100;
2237 static u32 hash_seeds[3];
2238 static uword hash_bitmap[256 / BITS (uword)];
2241 if (node->flags & VLIB_NODE_FLAG_TRACE)
2242 ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2244 time_now = vlib_time_now (vm);
2245 if (time_now - time_last_seed_change > 1e-3)
2248 u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
2249 sizeof (hash_seeds));
2250 for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
2251 hash_seeds[i] = r[i];
2253 /* Mark all hash keys as been no-seen before. */
2254 for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
2257 time_last_seed_change = time_now;
2260 from = vlib_frame_vector_args (frame);
2261 n_left_from = frame->n_vectors;
2262 next_index = node->cached_next_index;
2263 if (next_index == IP4_ARP_NEXT_DROP)
2264 next_index = IP4_ARP_N_NEXT; /* point to first interface */
2266 while (n_left_from > 0)
2268 vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
2269 to_next_drop, n_left_to_next_drop);
2271 while (n_left_from > 0 && n_left_to_next_drop > 0)
2275 ethernet_header_t * eh0;
2276 u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
2278 ip_adjacency_t * adj0;
2282 p0 = vlib_get_buffer (vm, pi0);
2284 adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2285 adj0 = ip_get_adjacency (lm, adj_index0);
2286 ip0 = vlib_buffer_get_current (p0);
2288 /* If packet destination is not local, send ARP to next hop */
2289 if (adj0->arp.next_hop.ip4.as_u32)
2290 ip0->dst_address.data_u32 = adj0->arp.next_hop.ip4.as_u32;
2293 * if ip4_rewrite_local applied the IP_LOOKUP_NEXT_ARP
2294 * rewrite to this packet, we need to skip it here.
2295 * Note, to distinguish from src IP addr *.8.6.*, we
2296 * check for a bcast eth dest instead of IPv4 version.
2298 eh0 = (ethernet_header_t*)ip0;
2299 if ((ip0->ip_version_and_header_length & 0xF0) != 0x40)
2302 u16 * etype = &eh0->type;
2303 while ((*etype == clib_host_to_net_u16 (0x8100)) //dot1q
2304 || (*etype == clib_host_to_net_u16 (0x88a8)))//dot1ad
2307 etype += 2; //vlan tag also 16 bits, same as etype
2309 if (*etype == clib_host_to_net_u16 (0x0806)) //arp
2311 vlib_buffer_advance (
2312 p0, sizeof(ethernet_header_t) + (4*vlan_num));
2313 ip0 = vlib_buffer_get_current (p0);
2321 sw_if_index0 = adj0->rewrite_header.sw_if_index;
2322 vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
2324 a0 ^= ip0->dst_address.data_u32;
2327 hash_v3_finalize32 (a0, b0, c0);
2329 c0 &= BITS (hash_bitmap) - 1;
2330 c0 = c0 / BITS (uword);
2331 m0 = (uword) 1 << (c0 % BITS (uword));
2333 bm0 = hash_bitmap[c0];
2334 drop0 = (bm0 & m0) != 0;
2336 /* Mark it as seen. */
2337 hash_bitmap[c0] = bm0 | m0;
2341 to_next_drop[0] = pi0;
2343 n_left_to_next_drop -= 1;
2345 p0->error = node->errors[drop0 ? IP4_ARP_ERROR_DROP : IP4_ARP_ERROR_REQUEST_SENT];
2351 * Can happen if the control-plane is programming tables
2352 * with traffic flowing; at least that's today's lame excuse.
2354 if (adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP)
2356 p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
2359 /* Send ARP request. */
2363 ethernet_arp_header_t * h0;
2364 vnet_hw_interface_t * hw_if0;
2366 h0 = vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template, &bi0);
2368 /* Add rewrite/encap string for ARP packet. */
2369 vnet_rewrite_one_header (adj0[0], h0, sizeof (ethernet_header_t));
2371 hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2373 /* Src ethernet address in ARP header. */
2374 clib_memcpy (h0->ip4_over_ethernet[0].ethernet, hw_if0->hw_address,
2375 sizeof (h0->ip4_over_ethernet[0].ethernet));
2377 if (ip4_src_address_for_packet (im, p0, &h0->ip4_over_ethernet[0].ip4, sw_if_index0)) {
2378 //No source address available
2379 p0->error = node->errors[IP4_ARP_ERROR_NO_SOURCE_ADDRESS];
2380 vlib_buffer_free(vm, &bi0, 1);
2384 /* Copy in destination address we are requesting. */
2385 h0->ip4_over_ethernet[1].ip4.data_u32 = ip0->dst_address.data_u32;
2387 vlib_buffer_copy_trace_flag (vm, p0, bi0);
2388 b0 = vlib_get_buffer (vm, bi0);
2389 vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
2391 vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2393 vlib_set_next_frame_buffer (vm, node, adj0->rewrite_header.next_index, bi0);
2397 vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
2400 return frame->n_vectors;
2403 static char * ip4_arp_error_strings[] = {
2404 [IP4_ARP_ERROR_DROP] = "address overflow drops",
2405 [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
2406 [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
2407 [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
2408 [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
2409 [IP4_ARP_ERROR_NO_SOURCE_ADDRESS] = "no source address for ARP request",
2412 VLIB_REGISTER_NODE (ip4_arp_node) = {
2413 .function = ip4_arp,
2415 .vector_size = sizeof (u32),
2417 .format_trace = format_ip4_forward_next_trace,
2419 .n_errors = ARRAY_LEN (ip4_arp_error_strings),
2420 .error_strings = ip4_arp_error_strings,
2422 .n_next_nodes = IP4_ARP_N_NEXT,
2424 [IP4_ARP_NEXT_DROP] = "error-drop",
2428 #define foreach_notrace_ip4_arp_error \
2434 clib_error_t * arp_notrace_init (vlib_main_t * vm)
2436 vlib_node_runtime_t *rt =
2437 vlib_node_get_runtime (vm, ip4_arp_node.index);
2439 /* don't trace ARP request packets */
2441 vnet_pcap_drop_trace_filter_add_del \
2442 (rt->errors[IP4_ARP_ERROR_##a], \
2444 foreach_notrace_ip4_arp_error;
2449 VLIB_INIT_FUNCTION(arp_notrace_init);
2452 /* Send an ARP request to see if given destination is reachable on given interface. */
2454 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
2456 vnet_main_t * vnm = vnet_get_main();
2457 ip4_main_t * im = &ip4_main;
2458 ethernet_arp_header_t * h;
2459 ip4_address_t * src;
2460 ip_interface_address_t * ia;
2461 ip_adjacency_t * adj;
2462 vnet_hw_interface_t * hi;
2463 vnet_sw_interface_t * si;
2467 si = vnet_get_sw_interface (vnm, sw_if_index);
2469 if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2471 return clib_error_return (0, "%U: interface %U down",
2472 format_ip4_address, dst,
2473 format_vnet_sw_if_index_name, vnm,
2477 src = ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2480 vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2481 return clib_error_return
2482 (0, "no matching interface address for destination %U (interface %U)",
2483 format_ip4_address, dst,
2484 format_vnet_sw_if_index_name, vnm, sw_if_index);
2487 adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
2489 h = vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template, &bi);
2491 hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2493 clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address, sizeof (h->ip4_over_ethernet[0].ethernet));
2495 h->ip4_over_ethernet[0].ip4 = src[0];
2496 h->ip4_over_ethernet[1].ip4 = dst[0];
2498 b = vlib_get_buffer (vm, bi);
2499 vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2501 /* Add encapsulation string for software interface (e.g. ethernet header). */
2502 vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2503 vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2506 vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
2507 u32 * to_next = vlib_frame_vector_args (f);
2510 vlib_put_frame_to_node (vm, hi->output_node_index, f);
2513 return /* no error */ 0;
2517 IP4_REWRITE_NEXT_DROP,
2518 IP4_REWRITE_NEXT_ARP,
2519 IP4_REWRITE_NEXT_ICMP_ERROR,
2520 } ip4_rewrite_next_t;
2523 ip4_rewrite_inline (vlib_main_t * vm,
2524 vlib_node_runtime_t * node,
2525 vlib_frame_t * frame,
2526 int rewrite_for_locally_received_packets)
2528 ip_lookup_main_t * lm = &ip4_main.lookup_main;
2529 u32 * from = vlib_frame_vector_args (frame);
2530 u32 n_left_from, n_left_to_next, * to_next, next_index;
2531 vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip4_input_node.index);
2532 vlib_rx_or_tx_t adj_rx_tx = rewrite_for_locally_received_packets ? VLIB_RX : VLIB_TX;
2534 n_left_from = frame->n_vectors;
2535 next_index = node->cached_next_index;
2536 u32 cpu_index = os_get_cpu_number();
2538 while (n_left_from > 0)
2540 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2542 while (n_left_from >= 4 && n_left_to_next >= 2)
2544 ip_adjacency_t * adj0, * adj1;
2545 vlib_buffer_t * p0, * p1;
2546 ip4_header_t * ip0, * ip1;
2547 u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
2548 u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
2549 u32 next0_override, next1_override;
2551 if (rewrite_for_locally_received_packets)
2552 next0_override = next1_override = 0;
2554 /* Prefetch next iteration. */
2556 vlib_buffer_t * p2, * p3;
2558 p2 = vlib_get_buffer (vm, from[2]);
2559 p3 = vlib_get_buffer (vm, from[3]);
2561 vlib_prefetch_buffer_header (p2, STORE);
2562 vlib_prefetch_buffer_header (p3, STORE);
2564 CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2565 CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2568 pi0 = to_next[0] = from[0];
2569 pi1 = to_next[1] = from[1];
2574 n_left_to_next -= 2;
2576 p0 = vlib_get_buffer (vm, pi0);
2577 p1 = vlib_get_buffer (vm, pi1);
2579 adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2580 adj_index1 = vnet_buffer (p1)->ip.adj_index[adj_rx_tx];
2582 /* We should never rewrite a pkt using the MISS adjacency */
2583 ASSERT(adj_index0 && adj_index1);
2585 ip0 = vlib_buffer_get_current (p0);
2586 ip1 = vlib_buffer_get_current (p1);
2588 error0 = error1 = IP4_ERROR_NONE;
2589 next0 = next1 = IP4_REWRITE_NEXT_DROP;
2591 /* Decrement TTL & update checksum.
2592 Works either endian, so no need for byte swap. */
2593 if (! rewrite_for_locally_received_packets)
2595 i32 ttl0 = ip0->ttl, ttl1 = ip1->ttl;
2597 /* Input node should have reject packets with ttl 0. */
2598 ASSERT (ip0->ttl > 0);
2599 ASSERT (ip1->ttl > 0);
2601 checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2602 checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2604 checksum0 += checksum0 >= 0xffff;
2605 checksum1 += checksum1 >= 0xffff;
2607 ip0->checksum = checksum0;
2608 ip1->checksum = checksum1;
2617 * If the ttl drops below 1 when forwarding, generate
2620 if (PREDICT_FALSE(ttl0 <= 0))
2622 error0 = IP4_ERROR_TIME_EXPIRED;
2623 vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2624 icmp4_error_set_vnet_buffer(p0, ICMP4_time_exceeded,
2625 ICMP4_time_exceeded_ttl_exceeded_in_transit, 0);
2626 next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2628 if (PREDICT_FALSE(ttl1 <= 0))
2630 error1 = IP4_ERROR_TIME_EXPIRED;
2631 vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32)~0;
2632 icmp4_error_set_vnet_buffer(p1, ICMP4_time_exceeded,
2633 ICMP4_time_exceeded_ttl_exceeded_in_transit, 0);
2634 next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
2637 /* Verify checksum. */
2638 ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2639 ASSERT (ip1->checksum == ip4_header_checksum (ip1));
2642 /* Rewrite packet header and updates lengths. */
2643 adj0 = ip_get_adjacency (lm, adj_index0);
2644 adj1 = ip_get_adjacency (lm, adj_index1);
2646 if (rewrite_for_locally_received_packets)
2649 * If someone sends e.g. an icmp4 w/ src = dst = interface addr,
2650 * we end up here with a local adjacency in hand
2651 * The local adj rewrite data is 0xfefe on purpose.
2652 * Bad engineer, no donut for you.
2654 if (PREDICT_FALSE(adj0->lookup_next_index
2655 == IP_LOOKUP_NEXT_LOCAL))
2656 error0 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2657 if (PREDICT_FALSE(adj0->lookup_next_index
2658 == IP_LOOKUP_NEXT_ARP))
2659 next0_override = IP4_REWRITE_NEXT_ARP;
2660 if (PREDICT_FALSE(adj1->lookup_next_index
2661 == IP_LOOKUP_NEXT_LOCAL))
2662 error1 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2663 if (PREDICT_FALSE(adj1->lookup_next_index
2664 == IP_LOOKUP_NEXT_ARP))
2665 next1_override = IP4_REWRITE_NEXT_ARP;
2668 /* Worth pipelining. No guarantee that adj0,1 are hot... */
2669 rw_len0 = adj0[0].rewrite_header.data_bytes;
2670 rw_len1 = adj1[0].rewrite_header.data_bytes;
2672 /* Check MTU of outgoing interface. */
2673 error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2674 ? IP4_ERROR_MTU_EXCEEDED
2676 error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
2677 ? IP4_ERROR_MTU_EXCEEDED
2680 next0 = (error0 == IP4_ERROR_NONE)
2681 ? adj0[0].rewrite_header.next_index : next0;
2683 if (rewrite_for_locally_received_packets)
2684 next0 = next0 && next0_override ? next0_override : next0;
2686 next1 = (error1 == IP4_ERROR_NONE)
2687 ? adj1[0].rewrite_header.next_index : next1;
2689 if (rewrite_for_locally_received_packets)
2690 next1 = next1 && next1_override ? next1_override : next1;
2693 * We've already accounted for an ethernet_header_t elsewhere
2695 if (PREDICT_FALSE (rw_len0 > sizeof(ethernet_header_t)))
2696 vlib_increment_combined_counter
2697 (&lm->adjacency_counters,
2698 cpu_index, adj_index0,
2699 /* packet increment */ 0,
2700 /* byte increment */ rw_len0-sizeof(ethernet_header_t));
2702 if (PREDICT_FALSE (rw_len1 > sizeof(ethernet_header_t)))
2703 vlib_increment_combined_counter
2704 (&lm->adjacency_counters,
2705 cpu_index, adj_index1,
2706 /* packet increment */ 0,
2707 /* byte increment */ rw_len1-sizeof(ethernet_header_t));
2709 /* Don't adjust the buffer for ttl issue; icmp-error node wants
2710 * to see the IP headerr */
2711 if (PREDICT_TRUE(error0 == IP4_ERROR_NONE))
2713 p0->current_data -= rw_len0;
2714 p0->current_length += rw_len0;
2715 p0->error = error_node->errors[error0];
2716 vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2717 adj0[0].rewrite_header.sw_if_index;
2719 if (PREDICT_TRUE(error1 == IP4_ERROR_NONE))
2721 p1->current_data -= rw_len1;
2722 p1->current_length += rw_len1;
2723 p1->error = error_node->errors[error1];
2724 vnet_buffer (p1)->sw_if_index[VLIB_TX] =
2725 adj1[0].rewrite_header.sw_if_index;
2728 /* Guess we are only writing on simple Ethernet header. */
2729 vnet_rewrite_two_headers (adj0[0], adj1[0],
2731 sizeof (ethernet_header_t));
2733 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2734 to_next, n_left_to_next,
2735 pi0, pi1, next0, next1);
2738 while (n_left_from > 0 && n_left_to_next > 0)
2740 ip_adjacency_t * adj0;
2743 u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2746 if (rewrite_for_locally_received_packets)
2749 pi0 = to_next[0] = from[0];
2751 p0 = vlib_get_buffer (vm, pi0);
2753 adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2755 /* We should never rewrite a pkt using the MISS adjacency */
2758 adj0 = ip_get_adjacency (lm, adj_index0);
2760 ip0 = vlib_buffer_get_current (p0);
2762 error0 = IP4_ERROR_NONE;
2763 next0 = IP4_REWRITE_NEXT_DROP; /* drop on error */
2765 /* Decrement TTL & update checksum. */
2766 if (! rewrite_for_locally_received_packets)
2768 i32 ttl0 = ip0->ttl;
2770 checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2772 checksum0 += checksum0 >= 0xffff;
2774 ip0->checksum = checksum0;
2776 ASSERT (ip0->ttl > 0);
2782 ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2784 if (PREDICT_FALSE(ttl0 <= 0))
2787 * If the ttl drops below 1 when forwarding, generate
2790 error0 = IP4_ERROR_TIME_EXPIRED;
2791 next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2792 vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2793 icmp4_error_set_vnet_buffer(p0, ICMP4_time_exceeded,
2794 ICMP4_time_exceeded_ttl_exceeded_in_transit, 0);
2798 if (rewrite_for_locally_received_packets)
2801 * If someone sends e.g. an icmp4 w/ src = dst = interface addr,
2802 * we end up here with a local adjacency in hand
2803 * The local adj rewrite data is 0xfefe on purpose.
2804 * Bad engineer, no donut for you.
2806 if (PREDICT_FALSE(adj0->lookup_next_index
2807 == IP_LOOKUP_NEXT_LOCAL))
2808 error0 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2810 * We have to override the next_index in ARP adjacencies,
2811 * because they're set up for ip4-arp, not this node...
2813 if (PREDICT_FALSE(adj0->lookup_next_index
2814 == IP_LOOKUP_NEXT_ARP))
2815 next0_override = IP4_REWRITE_NEXT_ARP;
2818 /* Guess we are only writing on simple Ethernet header. */
2819 vnet_rewrite_one_header (adj0[0], ip0,
2820 sizeof (ethernet_header_t));
2822 /* Update packet buffer attributes/set output interface. */
2823 rw_len0 = adj0[0].rewrite_header.data_bytes;
2825 if (PREDICT_FALSE (rw_len0 > sizeof(ethernet_header_t)))
2826 vlib_increment_combined_counter
2827 (&lm->adjacency_counters,
2828 cpu_index, adj_index0,
2829 /* packet increment */ 0,
2830 /* byte increment */ rw_len0-sizeof(ethernet_header_t));
2832 /* Check MTU of outgoing interface. */
2833 error0 = (vlib_buffer_length_in_chain (vm, p0)
2834 > adj0[0].rewrite_header.max_l3_packet_bytes
2835 ? IP4_ERROR_MTU_EXCEEDED
2838 p0->error = error_node->errors[error0];
2840 /* Don't adjust the buffer for ttl issue; icmp-error node wants
2841 * to see the IP headerr */
2842 if (PREDICT_TRUE(error0 == IP4_ERROR_NONE))
2844 p0->current_data -= rw_len0;
2845 p0->current_length += rw_len0;
2847 vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2848 adj0[0].rewrite_header.sw_if_index;
2849 next0 = adj0[0].rewrite_header.next_index;
2852 if (rewrite_for_locally_received_packets)
2853 next0 = next0 && next0_override ? next0_override : next0;
2858 n_left_to_next -= 1;
2860 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2861 to_next, n_left_to_next,
2865 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2868 /* Need to do trace after rewrites to pick up new packet data. */
2869 if (node->flags & VLIB_NODE_FLAG_TRACE)
2870 ip4_forward_next_trace (vm, node, frame, adj_rx_tx);
2872 return frame->n_vectors;
2876 ip4_rewrite_transit (vlib_main_t * vm,
2877 vlib_node_runtime_t * node,
2878 vlib_frame_t * frame)
2880 return ip4_rewrite_inline (vm, node, frame,
2881 /* rewrite_for_locally_received_packets */ 0);
2885 ip4_rewrite_local (vlib_main_t * vm,
2886 vlib_node_runtime_t * node,
2887 vlib_frame_t * frame)
2889 return ip4_rewrite_inline (vm, node, frame,
2890 /* rewrite_for_locally_received_packets */ 1);
2893 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2894 .function = ip4_rewrite_transit,
2895 .name = "ip4-rewrite-transit",
2896 .vector_size = sizeof (u32),
2898 .format_trace = format_ip4_rewrite_trace,
2902 [IP4_REWRITE_NEXT_DROP] = "error-drop",
2903 [IP4_REWRITE_NEXT_ARP] = "ip4-arp",
2904 [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2908 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite_transit)
2910 VLIB_REGISTER_NODE (ip4_rewrite_local_node) = {
2911 .function = ip4_rewrite_local,
2912 .name = "ip4-rewrite-local",
2913 .vector_size = sizeof (u32),
2915 .sibling_of = "ip4-rewrite-transit",
2917 .format_trace = format_ip4_rewrite_trace,
2922 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_local_node, ip4_rewrite_local)
2924 static clib_error_t *
2925 add_del_interface_table (vlib_main_t * vm,
2926 unformat_input_t * input,
2927 vlib_cli_command_t * cmd)
2929 vnet_main_t * vnm = vnet_get_main();
2930 clib_error_t * error = 0;
2931 u32 sw_if_index, table_id;
2935 if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2937 error = clib_error_return (0, "unknown interface `%U'",
2938 format_unformat_error, input);
2942 if (unformat (input, "%d", &table_id))
2946 error = clib_error_return (0, "expected table id `%U'",
2947 format_unformat_error, input);
2952 ip4_main_t * im = &ip4_main;
2953 ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_id, IP4_ROUTE_FLAG_TABLE_ID);
2957 vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
2958 im->fib_index_by_sw_if_index[sw_if_index] = fib->index;
2966 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) = {
2967 .path = "set interface ip table",
2968 .function = add_del_interface_table,
2969 .short_help = "Add/delete FIB table id for interface",
2974 ip4_lookup_multicast (vlib_main_t * vm,
2975 vlib_node_runtime_t * node,
2976 vlib_frame_t * frame)
2978 ip4_main_t * im = &ip4_main;
2979 ip_lookup_main_t * lm = &im->lookup_main;
2980 vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
2981 u32 n_left_from, n_left_to_next, * from, * to_next;
2982 ip_lookup_next_t next;
2983 u32 cpu_index = os_get_cpu_number();
2985 from = vlib_frame_vector_args (frame);
2986 n_left_from = frame->n_vectors;
2987 next = node->cached_next_index;
2989 while (n_left_from > 0)
2991 vlib_get_next_frame (vm, node, next,
2992 to_next, n_left_to_next);
2994 while (n_left_from >= 4 && n_left_to_next >= 2)
2996 vlib_buffer_t * p0, * p1;
2997 u32 pi0, pi1, adj_index0, adj_index1, wrong_next;
2998 ip_lookup_next_t next0, next1;
2999 ip4_header_t * ip0, * ip1;
3000 ip_adjacency_t * adj0, * adj1;
3001 u32 fib_index0, fib_index1;
3002 u32 flow_hash_config0, flow_hash_config1;
3004 /* Prefetch next iteration. */
3006 vlib_buffer_t * p2, * p3;
3008 p2 = vlib_get_buffer (vm, from[2]);
3009 p3 = vlib_get_buffer (vm, from[3]);
3011 vlib_prefetch_buffer_header (p2, LOAD);
3012 vlib_prefetch_buffer_header (p3, LOAD);
3014 CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
3015 CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
3018 pi0 = to_next[0] = from[0];
3019 pi1 = to_next[1] = from[1];
3021 p0 = vlib_get_buffer (vm, pi0);
3022 p1 = vlib_get_buffer (vm, pi1);
3024 ip0 = vlib_buffer_get_current (p0);
3025 ip1 = vlib_buffer_get_current (p1);
3027 fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
3028 fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
3029 fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
3030 fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
3031 fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
3032 fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
3034 adj_index0 = ip4_fib_lookup_buffer (im, fib_index0,
3035 &ip0->dst_address, p0);
3036 adj_index1 = ip4_fib_lookup_buffer (im, fib_index1,
3037 &ip1->dst_address, p1);
3039 adj0 = ip_get_adjacency (lm, adj_index0);
3040 adj1 = ip_get_adjacency (lm, adj_index1);
3042 next0 = adj0->lookup_next_index;
3043 next1 = adj1->lookup_next_index;
3046 vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
3049 vec_elt_at_index (im->fibs, fib_index1)->flow_hash_config;
3051 vnet_buffer (p0)->ip.flow_hash = ip4_compute_flow_hash
3052 (ip0, flow_hash_config0);
3054 vnet_buffer (p1)->ip.flow_hash = ip4_compute_flow_hash
3055 (ip1, flow_hash_config1);
3057 ASSERT (adj0->n_adj > 0);
3058 ASSERT (adj1->n_adj > 0);
3059 ASSERT (is_pow2 (adj0->n_adj));
3060 ASSERT (is_pow2 (adj1->n_adj));
3061 adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
3062 adj_index1 += (vnet_buffer (p1)->ip.flow_hash & (adj1->n_adj - 1));
3064 vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
3065 vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
3067 if (1) /* $$$$$$ HACK FIXME */
3068 vlib_increment_combined_counter
3069 (cm, cpu_index, adj_index0, 1,
3070 vlib_buffer_length_in_chain (vm, p0));
3071 if (1) /* $$$$$$ HACK FIXME */
3072 vlib_increment_combined_counter
3073 (cm, cpu_index, adj_index1, 1,
3074 vlib_buffer_length_in_chain (vm, p1));
3078 n_left_to_next -= 2;
3081 wrong_next = (next0 != next) + 2*(next1 != next);
3082 if (PREDICT_FALSE (wrong_next != 0))
3090 n_left_to_next += 1;
3091 vlib_set_next_frame_buffer (vm, node, next0, pi0);
3097 n_left_to_next += 1;
3098 vlib_set_next_frame_buffer (vm, node, next1, pi1);
3104 n_left_to_next += 2;
3105 vlib_set_next_frame_buffer (vm, node, next0, pi0);
3106 vlib_set_next_frame_buffer (vm, node, next1, pi1);
3110 vlib_put_next_frame (vm, node, next, n_left_to_next);
3112 vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
3118 while (n_left_from > 0 && n_left_to_next > 0)
3122 u32 pi0, adj_index0;
3123 ip_lookup_next_t next0;
3124 ip_adjacency_t * adj0;
3126 u32 flow_hash_config0;
3131 p0 = vlib_get_buffer (vm, pi0);
3133 ip0 = vlib_buffer_get_current (p0);
3135 fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
3136 vnet_buffer (p0)->sw_if_index[VLIB_RX]);
3137 fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
3138 fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
3140 adj_index0 = ip4_fib_lookup_buffer (im, fib_index0,
3141 &ip0->dst_address, p0);
3143 adj0 = ip_get_adjacency (lm, adj_index0);
3145 next0 = adj0->lookup_next_index;
3148 vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
3150 vnet_buffer (p0)->ip.flow_hash =
3151 ip4_compute_flow_hash (ip0, flow_hash_config0);
3153 ASSERT (adj0->n_adj > 0);
3154 ASSERT (is_pow2 (adj0->n_adj));
3155 adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
3157 vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
3159 if (1) /* $$$$$$ HACK FIXME */
3160 vlib_increment_combined_counter
3161 (cm, cpu_index, adj_index0, 1,
3162 vlib_buffer_length_in_chain (vm, p0));
3166 n_left_to_next -= 1;
3169 if (PREDICT_FALSE (next0 != next))
3171 n_left_to_next += 1;
3172 vlib_put_next_frame (vm, node, next, n_left_to_next);
3174 vlib_get_next_frame (vm, node, next,
3175 to_next, n_left_to_next);
3178 n_left_to_next -= 1;
3182 vlib_put_next_frame (vm, node, next, n_left_to_next);
3185 if (node->flags & VLIB_NODE_FLAG_TRACE)
3186 ip4_forward_next_trace(vm, node, frame, VLIB_TX);
3188 return frame->n_vectors;
3191 VLIB_REGISTER_NODE (ip4_lookup_multicast_node,static) = {
3192 .function = ip4_lookup_multicast,
3193 .name = "ip4-lookup-multicast",
3194 .vector_size = sizeof (u32),
3195 .sibling_of = "ip4-lookup",
3196 .format_trace = format_ip4_lookup_trace,
3201 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_multicast_node, ip4_lookup_multicast)
3203 VLIB_REGISTER_NODE (ip4_multicast_node,static) = {
3204 .function = ip4_drop,
3205 .name = "ip4-multicast",
3206 .vector_size = sizeof (u32),
3208 .format_trace = format_ip4_forward_next_trace,
3216 int ip4_lookup_validate (ip4_address_t *a, u32 fib_index0)
3218 ip4_main_t * im = &ip4_main;
3219 ip4_fib_mtrie_t * mtrie0;
3220 ip4_fib_mtrie_leaf_t leaf0;
3223 mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
3225 leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
3226 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 0);
3227 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 1);
3228 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
3229 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
3231 /* Handle default route. */
3232 leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
3234 adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
3236 return adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
3238 /* no_default_route */ 0);
3241 static clib_error_t *
3242 test_lookup_command_fn (vlib_main_t * vm,
3243 unformat_input_t * input,
3244 vlib_cli_command_t * cmd)
3250 ip4_address_t ip4_base_address;
3253 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3254 if (unformat (input, "table %d", &table_id))
3256 else if (unformat (input, "count %f", &count))
3259 else if (unformat (input, "%U",
3260 unformat_ip4_address, &ip4_base_address))
3263 return clib_error_return (0, "unknown input `%U'",
3264 format_unformat_error, input);
3269 for (i = 0; i < n; i++)
3271 if (!ip4_lookup_validate (&ip4_base_address, table_id))
3274 ip4_base_address.as_u32 =
3275 clib_host_to_net_u32 (1 +
3276 clib_net_to_host_u32 (ip4_base_address.as_u32));
3280 vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
3282 vlib_cli_output (vm, "No errors in %d lookups\n", n);
3287 VLIB_CLI_COMMAND (lookup_test_command, static) = {
3288 .path = "test lookup",
3289 .short_help = "test lookup",
3290 .function = test_lookup_command_fn,
3293 int vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
3295 ip4_main_t * im4 = &ip4_main;
3297 uword * p = hash_get (im4->fib_index_by_table_id, table_id);
3300 return VNET_API_ERROR_NO_SUCH_FIB;
3302 fib = vec_elt_at_index (im4->fibs, p[0]);
3304 fib->flow_hash_config = flow_hash_config;
3308 static clib_error_t *
3309 set_ip_flow_hash_command_fn (vlib_main_t * vm,
3310 unformat_input_t * input,
3311 vlib_cli_command_t * cmd)
3315 u32 flow_hash_config = 0;
3318 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3319 if (unformat (input, "table %d", &table_id))
3322 else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3323 foreach_flow_hash_bit
3329 return clib_error_return (0, "unknown input `%U'",
3330 format_unformat_error, input);
3332 rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
3338 case VNET_API_ERROR_NO_SUCH_FIB:
3339 return clib_error_return (0, "no such FIB table %d", table_id);
3342 clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3349 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) = {
3350 .path = "set ip flow-hash",
3352 "set ip table flow-hash table <fib-id> src dst sport dport proto reverse",
3353 .function = set_ip_flow_hash_command_fn,
3356 int vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3359 vnet_main_t * vnm = vnet_get_main();
3360 vnet_interface_main_t * im = &vnm->interface_main;
3361 ip4_main_t * ipm = &ip4_main;
3362 ip_lookup_main_t * lm = &ipm->lookup_main;
3363 vnet_classify_main_t * cm = &vnet_classify_main;
3365 if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3366 return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3368 if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3369 return VNET_API_ERROR_NO_SUCH_ENTRY;
3371 vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3372 lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
3377 static clib_error_t *
3378 set_ip_classify_command_fn (vlib_main_t * vm,
3379 unformat_input_t * input,
3380 vlib_cli_command_t * cmd)
3382 u32 table_index = ~0;
3383 int table_index_set = 0;
3384 u32 sw_if_index = ~0;
3387 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3388 if (unformat (input, "table-index %d", &table_index))
3389 table_index_set = 1;
3390 else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3391 vnet_get_main(), &sw_if_index))
3397 if (table_index_set == 0)
3398 return clib_error_return (0, "classify table-index must be specified");
3400 if (sw_if_index == ~0)
3401 return clib_error_return (0, "interface / subif must be specified");
3403 rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
3410 case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3411 return clib_error_return (0, "No such interface");
3413 case VNET_API_ERROR_NO_SUCH_ENTRY:
3414 return clib_error_return (0, "No such classifier table");
3419 VLIB_CLI_COMMAND (set_ip_classify_command, static) = {
3420 .path = "set ip classify",
3422 "set ip classify intfc <int> table-index <index>",
3423 .function = set_ip_classify_command_fn,