2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/ip/ip.h>
17 #include <vnet/dpo/mpls_label_dpo.h>
18 #include <vnet/mpls/mpls.h>
19 #include <vnet/dpo/drop_dpo.h>
22 * pool of all MPLS Label DPOs
24 mpls_label_dpo_t *mpls_label_dpo_pool;
26 static mpls_label_dpo_t *
27 mpls_label_dpo_alloc (void)
29 mpls_label_dpo_t *mld;
31 pool_get_aligned(mpls_label_dpo_pool, mld, CLIB_CACHE_LINE_BYTES);
32 memset(mld, 0, sizeof(*mld));
34 dpo_reset(&mld->mld_dpo);
40 mpls_label_dpo_get_index (mpls_label_dpo_t *mld)
42 return (mld - mpls_label_dpo_pool);
46 mpls_label_dpo_create (mpls_label_t *label_stack,
50 dpo_proto_t payload_proto,
53 mpls_label_dpo_t *mld;
56 mld = mpls_label_dpo_alloc();
58 if (MPLS_LABEL_DPO_MAX_N_LABELS < vec_len(label_stack))
60 clib_warning("Label stack size exceeded");
61 dpo_stack(DPO_MPLS_LABEL,
62 mld->mld_payload_proto,
64 drop_dpo_get(DPO_PROTO_MPLS));
65 return (mpls_label_dpo_get_index(mld));
68 mld->mld_n_labels = vec_len(label_stack);
69 mld->mld_n_hdr_bytes = mld->mld_n_labels * sizeof(mld->mld_hdr[0]);
70 mld->mld_payload_proto = payload_proto;
73 * construct label rewrite headers for each value value passed.
74 * get the header in network byte order since we will paint it
75 * on a packet in the data-plane
78 for (ii = 0; ii < mld->mld_n_labels-1; ii++)
80 vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl, label_stack[ii]);
81 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl, 255);
82 vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl, 0);
83 vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl, MPLS_NON_EOS);
84 mld->mld_hdr[ii].label_exp_s_ttl =
85 clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
89 * the inner most label
91 ii = mld->mld_n_labels-1;
93 vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl, label_stack[ii]);
94 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl, ttl);
95 vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl, exp);
96 vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl, eos);
97 mld->mld_hdr[ii].label_exp_s_ttl =
98 clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
101 * stack this label objct on its parent.
103 dpo_stack(DPO_MPLS_LABEL,
104 mld->mld_payload_proto,
108 return (mpls_label_dpo_get_index(mld));
112 format_mpls_label_dpo (u8 *s, va_list *args)
114 index_t index = va_arg (*args, index_t);
115 u32 indent = va_arg (*args, u32);
116 mpls_unicast_header_t hdr;
117 mpls_label_dpo_t *mld;
120 s = format(s, "mpls-label:[%d]:", index);
122 if (pool_is_free_index(mpls_label_dpo_pool, index))
125 * the packet trace can be printed after the DPO has been deleted
130 mld = mpls_label_dpo_get(index);
132 for (ii = 0; ii < mld->mld_n_labels; ii++)
134 hdr.label_exp_s_ttl =
135 clib_net_to_host_u32(mld->mld_hdr[ii].label_exp_s_ttl);
136 s = format(s, "%U", format_mpls_header, hdr);
139 s = format(s, "\n%U", format_white_space, indent);
140 s = format(s, "%U", format_dpo_id, &mld->mld_dpo, indent+2);
146 mpls_label_dpo_lock (dpo_id_t *dpo)
148 mpls_label_dpo_t *mld;
150 mld = mpls_label_dpo_get(dpo->dpoi_index);
156 mpls_label_dpo_unlock (dpo_id_t *dpo)
158 mpls_label_dpo_t *mld;
160 mld = mpls_label_dpo_get(dpo->dpoi_index);
164 if (0 == mld->mld_locks)
166 dpo_reset(&mld->mld_dpo);
167 pool_put(mpls_label_dpo_pool, mld);
172 * @brief A struct to hold tracing information for the MPLS label imposition
175 typedef struct mpls_label_imposition_trace_t_
178 * The MPLS header imposed
180 mpls_unicast_header_t hdr;
181 } mpls_label_imposition_trace_t;
183 always_inline mpls_unicast_header_t *
184 mpls_label_paint (vlib_buffer_t * b0,
185 mpls_label_dpo_t *mld0,
188 mpls_unicast_header_t *hdr0;
190 vlib_buffer_advance(b0, -(mld0->mld_n_hdr_bytes));
192 hdr0 = vlib_buffer_get_current(b0);
194 if (1 == mld0->mld_n_labels)
196 /* optimise for the common case of one label */
197 *hdr0 = mld0->mld_hdr[0];
201 clib_memcpy(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes);
202 hdr0 = hdr0 + (mld0->mld_n_labels - 1);
204 /* fixup the TTL for the inner most label */
205 ((char*)hdr0)[3] = ttl0;
211 mpls_label_imposition_inline (vlib_main_t * vm,
212 vlib_node_runtime_t * node,
213 vlib_frame_t * from_frame,
216 u8 payload_is_ethernet)
218 u32 n_left_from, next_index, * from, * to_next;
220 from = vlib_frame_vector_args (from_frame);
221 n_left_from = from_frame->n_vectors;
223 next_index = node->cached_next_index;
225 while (n_left_from > 0)
229 vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
231 while (n_left_from >= 8 && n_left_to_next >= 4)
233 u32 bi0, mldi0, bi1, mldi1, bi2, mldi2, bi3, mldi3;
234 mpls_unicast_header_t *hdr0, *hdr1, *hdr2, *hdr3;
235 mpls_label_dpo_t *mld0, *mld1, *mld2, *mld3;
236 vlib_buffer_t * b0, *b1, * b2, *b3;
237 u32 next0, next1, next2, next3;
238 u8 ttl0, ttl1,ttl2, ttl3 ;
240 bi0 = to_next[0] = from[0];
241 bi1 = to_next[1] = from[1];
242 bi2 = to_next[2] = from[2];
243 bi3 = to_next[3] = from[3];
245 /* Prefetch next iteration. */
247 vlib_buffer_t * p2, * p3, *p4, *p5;
249 p2 = vlib_get_buffer (vm, from[2]);
250 p3 = vlib_get_buffer (vm, from[3]);
251 p4 = vlib_get_buffer (vm, from[4]);
252 p5 = vlib_get_buffer (vm, from[5]);
254 vlib_prefetch_buffer_header (p2, STORE);
255 vlib_prefetch_buffer_header (p3, STORE);
256 vlib_prefetch_buffer_header (p4, STORE);
257 vlib_prefetch_buffer_header (p5, STORE);
259 CLIB_PREFETCH (p2->data, sizeof (hdr0[0]), STORE);
260 CLIB_PREFETCH (p3->data, sizeof (hdr0[0]), STORE);
261 CLIB_PREFETCH (p4->data, sizeof (hdr0[0]), STORE);
262 CLIB_PREFETCH (p5->data, sizeof (hdr0[0]), STORE);
270 b0 = vlib_get_buffer (vm, bi0);
271 b1 = vlib_get_buffer (vm, bi1);
272 b2 = vlib_get_buffer (vm, bi2);
273 b3 = vlib_get_buffer (vm, bi3);
275 /* dst lookup was done by ip4 lookup */
276 mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
277 mldi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
278 mldi2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
279 mldi3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
280 mld0 = mpls_label_dpo_get(mldi0);
281 mld1 = mpls_label_dpo_get(mldi1);
282 mld2 = mpls_label_dpo_get(mldi2);
283 mld3 = mpls_label_dpo_get(mldi3);
288 * decrement the TTL on ingress to the LSP
290 ip4_header_t * ip0 = vlib_buffer_get_current(b0);
291 ip4_header_t * ip1 = vlib_buffer_get_current(b1);
292 ip4_header_t * ip2 = vlib_buffer_get_current(b2);
293 ip4_header_t * ip3 = vlib_buffer_get_current(b3);
299 checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
300 checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
301 checksum2 = ip2->checksum + clib_host_to_net_u16 (0x0100);
302 checksum3 = ip3->checksum + clib_host_to_net_u16 (0x0100);
304 checksum0 += checksum0 >= 0xffff;
305 checksum1 += checksum1 >= 0xffff;
306 checksum2 += checksum2 >= 0xffff;
307 checksum3 += checksum3 >= 0xffff;
309 ip0->checksum = checksum0;
310 ip1->checksum = checksum1;
311 ip2->checksum = checksum2;
312 ip3->checksum = checksum3;
324 else if (payload_is_ip6)
327 * decrement the TTL on ingress to the LSP
329 ip6_header_t * ip0 = vlib_buffer_get_current(b0);
330 ip6_header_t * ip1 = vlib_buffer_get_current(b1);
331 ip6_header_t * ip2 = vlib_buffer_get_current(b2);
332 ip6_header_t * ip3 = vlib_buffer_get_current(b3);
339 ttl0 = ip0->hop_limit;
340 ttl1 = ip1->hop_limit;
341 ttl2 = ip2->hop_limit;
342 ttl3 = ip3->hop_limit;
344 else if (payload_is_ethernet)
347 * nothing to chang ein the ethernet header
349 ttl0 = ttl1 = ttl2 = ttl3 = 255;
354 * else, the packet to be encapped is an MPLS packet
356 if (PREDICT_TRUE(vnet_buffer(b0)->mpls.first))
359 * The first label to be imposed on the packet. this is a label swap.
360 * in which case we stashed the TTL and EXP bits in the
361 * packet in the lookup node
363 ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
365 ttl0 = vnet_buffer(b0)->mpls.ttl - 1;
370 * not the first label. implying we are recusring down a chain of
372 * Each layer is considered a new LSP - hence the TTL is reset.
376 if (PREDICT_TRUE(vnet_buffer(b1)->mpls.first))
378 ASSERT(1 != vnet_buffer (b1)->mpls.ttl);
379 ttl1 = vnet_buffer(b1)->mpls.ttl - 1;
385 if (PREDICT_TRUE(vnet_buffer(b2)->mpls.first))
387 ASSERT(1 != vnet_buffer (b2)->mpls.ttl);
389 ttl2 = vnet_buffer(b2)->mpls.ttl - 1;
395 if (PREDICT_TRUE(vnet_buffer(b3)->mpls.first))
397 ASSERT(1 != vnet_buffer (b3)->mpls.ttl);
398 ttl3 = vnet_buffer(b3)->mpls.ttl - 1;
405 vnet_buffer(b0)->mpls.first = 0;
406 vnet_buffer(b1)->mpls.first = 0;
407 vnet_buffer(b2)->mpls.first = 0;
408 vnet_buffer(b3)->mpls.first = 0;
410 /* Paint the MPLS header */
411 hdr0 = mpls_label_paint(b0, mld0, ttl0);
412 hdr1 = mpls_label_paint(b1, mld1, ttl1);
413 hdr2 = mpls_label_paint(b2, mld2, ttl2);
414 hdr3 = mpls_label_paint(b3, mld3, ttl3);
416 next0 = mld0->mld_dpo.dpoi_next_node;
417 next1 = mld1->mld_dpo.dpoi_next_node;
418 next2 = mld2->mld_dpo.dpoi_next_node;
419 next3 = mld3->mld_dpo.dpoi_next_node;
420 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
421 vnet_buffer(b1)->ip.adj_index[VLIB_TX] = mld1->mld_dpo.dpoi_index;
422 vnet_buffer(b2)->ip.adj_index[VLIB_TX] = mld2->mld_dpo.dpoi_index;
423 vnet_buffer(b3)->ip.adj_index[VLIB_TX] = mld3->mld_dpo.dpoi_index;
425 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
427 mpls_label_imposition_trace_t *tr =
428 vlib_add_trace (vm, node, b0, sizeof (*tr));
431 if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
433 mpls_label_imposition_trace_t *tr =
434 vlib_add_trace (vm, node, b1, sizeof (*tr));
437 if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
439 mpls_label_imposition_trace_t *tr =
440 vlib_add_trace (vm, node, b2, sizeof (*tr));
443 if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
445 mpls_label_imposition_trace_t *tr =
446 vlib_add_trace (vm, node, b3, sizeof (*tr));
450 vlib_validate_buffer_enqueue_x4(vm, node, next_index, to_next,
453 next0, next1, next2, next3);
456 while (n_left_from > 0 && n_left_to_next > 0)
458 mpls_unicast_header_t *hdr0;
459 mpls_label_dpo_t *mld0;
472 b0 = vlib_get_buffer (vm, bi0);
474 /* dst lookup was done by ip4 lookup */
475 mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
476 mld0 = mpls_label_dpo_get(mldi0);
481 * decrement the TTL on ingress to the LSP
483 ip4_header_t * ip0 = vlib_buffer_get_current(b0);
486 checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
487 checksum0 += checksum0 >= 0xffff;
489 ip0->checksum = checksum0;
493 else if (payload_is_ip6)
496 * decrement the TTL on ingress to the LSP
498 ip6_header_t * ip0 = vlib_buffer_get_current(b0);
501 ttl = ip0->hop_limit;
506 * else, the packet to be encapped is an MPLS packet
508 if (vnet_buffer(b0)->mpls.first)
511 * The first label to be imposed on the packet. this is a label swap.
512 * in which case we stashed the TTL and EXP bits in the
513 * packet in the lookup node
515 ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
517 ttl = vnet_buffer(b0)->mpls.ttl - 1;
522 * not the first label. implying we are recusring down a chain of
524 * Each layer is considered a new LSP - hence the TTL is reset.
529 vnet_buffer(b0)->mpls.first = 0;
531 /* Paint the MPLS header */
532 vlib_buffer_advance(b0, -(mld0->mld_n_hdr_bytes));
533 hdr0 = vlib_buffer_get_current(b0);
534 clib_memcpy(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes);
536 /* fixup the TTL for the inner most label */
537 hdr0 = hdr0 + (mld0->mld_n_labels - 1);
538 ((char*)hdr0)[3] = ttl;
540 next0 = mld0->mld_dpo.dpoi_next_node;
541 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
543 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
545 mpls_label_imposition_trace_t *tr =
546 vlib_add_trace (vm, node, b0, sizeof (*tr));
550 vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
551 n_left_to_next, bi0, next0);
553 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
555 return from_frame->n_vectors;
559 format_mpls_label_imposition_trace (u8 * s, va_list * args)
561 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
562 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
563 mpls_label_imposition_trace_t * t;
564 mpls_unicast_header_t hdr;
567 t = va_arg (*args, mpls_label_imposition_trace_t *);
568 indent = format_get_indent (s);
569 hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
571 s = format (s, "%Umpls-header:%U",
572 format_white_space, indent,
573 format_mpls_header, hdr);
578 mpls_label_imposition (vlib_main_t * vm,
579 vlib_node_runtime_t * node,
580 vlib_frame_t * frame)
582 return (mpls_label_imposition_inline(vm, node, frame, 0, 0, 0));
585 VLIB_REGISTER_NODE (mpls_label_imposition_node) = {
586 .function = mpls_label_imposition,
587 .name = "mpls-label-imposition",
588 .vector_size = sizeof (u32),
590 .format_trace = format_mpls_label_imposition_trace,
596 VLIB_NODE_FUNCTION_MULTIARCH (mpls_label_imposition_node,
597 mpls_label_imposition)
600 ip4_mpls_label_imposition (vlib_main_t * vm,
601 vlib_node_runtime_t * node,
602 vlib_frame_t * frame)
604 return (mpls_label_imposition_inline(vm, node, frame, 1, 0, 0));
607 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_node) = {
608 .function = ip4_mpls_label_imposition,
609 .name = "ip4-mpls-label-imposition",
610 .vector_size = sizeof (u32),
612 .format_trace = format_mpls_label_imposition_trace,
618 VLIB_NODE_FUNCTION_MULTIARCH (ip4_mpls_label_imposition_node,
619 ip4_mpls_label_imposition)
622 ip6_mpls_label_imposition (vlib_main_t * vm,
623 vlib_node_runtime_t * node,
624 vlib_frame_t * frame)
626 return (mpls_label_imposition_inline(vm, node, frame, 0, 1, 0));
629 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_node) = {
630 .function = ip6_mpls_label_imposition,
631 .name = "ip6-mpls-label-imposition",
632 .vector_size = sizeof (u32),
634 .format_trace = format_mpls_label_imposition_trace,
640 VLIB_NODE_FUNCTION_MULTIARCH (ip6_mpls_label_imposition_node,
641 ip6_mpls_label_imposition)
644 ethernet_mpls_label_imposition (vlib_main_t * vm,
645 vlib_node_runtime_t * node,
646 vlib_frame_t * frame)
648 return (mpls_label_imposition_inline(vm, node, frame, 0, 0, 1));
651 VLIB_REGISTER_NODE (ethernet_mpls_label_imposition_node) = {
652 .function = ethernet_mpls_label_imposition,
653 .name = "ethernet-mpls-label-imposition",
654 .vector_size = sizeof (u32),
656 .format_trace = format_mpls_label_imposition_trace,
662 VLIB_NODE_FUNCTION_MULTIARCH (ethernet_mpls_label_imposition_node,
663 ethernet_mpls_label_imposition)
666 mpls_label_dpo_mem_show (void)
668 fib_show_memory_usage("MPLS label",
669 pool_elts(mpls_label_dpo_pool),
670 pool_len(mpls_label_dpo_pool),
671 sizeof(mpls_label_dpo_t));
674 const static dpo_vft_t mld_vft = {
675 .dv_lock = mpls_label_dpo_lock,
676 .dv_unlock = mpls_label_dpo_unlock,
677 .dv_format = format_mpls_label_dpo,
678 .dv_mem_show = mpls_label_dpo_mem_show,
681 const static char* const mpls_label_imp_ip4_nodes[] =
683 "ip4-mpls-label-imposition",
686 const static char* const mpls_label_imp_ip6_nodes[] =
688 "ip6-mpls-label-imposition",
691 const static char* const mpls_label_imp_mpls_nodes[] =
693 "mpls-label-imposition",
696 const static char* const mpls_label_imp_ethernet_nodes[] =
698 "ethernet-mpls-label-imposition",
702 const static char* const * const mpls_label_imp_nodes[DPO_PROTO_NUM] =
704 [DPO_PROTO_IP4] = mpls_label_imp_ip4_nodes,
705 [DPO_PROTO_IP6] = mpls_label_imp_ip6_nodes,
706 [DPO_PROTO_MPLS] = mpls_label_imp_mpls_nodes,
707 [DPO_PROTO_ETHERNET] = mpls_label_imp_ethernet_nodes,
712 mpls_label_dpo_module_init (void)
714 dpo_register(DPO_MPLS_LABEL, &mld_vft, mpls_label_imp_nodes);