2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/ip/ip.h>
17 #include <vnet/dpo/mpls_label_dpo.h>
18 #include <vnet/mpls/mpls.h>
21 * pool of all MPLS Label DPOs
23 mpls_label_dpo_t *mpls_label_dpo_pool;
25 static mpls_label_dpo_t *
26 mpls_label_dpo_alloc (void)
28 mpls_label_dpo_t *mld;
30 pool_get_aligned(mpls_label_dpo_pool, mld, CLIB_CACHE_LINE_BYTES);
31 memset(mld, 0, sizeof(*mld));
33 dpo_reset(&mld->mld_dpo);
39 mpls_label_dpo_get_index (mpls_label_dpo_t *mld)
41 return (mld - mpls_label_dpo_pool);
45 mpls_label_dpo_create (mpls_label_t *label_stack,
49 dpo_proto_t payload_proto,
52 mpls_label_dpo_t *mld;
55 mld = mpls_label_dpo_alloc();
56 mld->mld_n_labels = vec_len(label_stack);
57 mld->mld_n_hdr_bytes = mld->mld_n_labels * sizeof(mld->mld_hdr[0]);
58 mld->mld_payload_proto = payload_proto;
61 * construct label rewrite headers for each value value passed.
62 * get the header in network byte order since we will paint it
63 * on a packet in the data-plane
66 for (ii = 0; ii < mld->mld_n_labels-1; ii++)
68 vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl, label_stack[ii]);
69 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl, 255);
70 vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl, 0);
71 vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl, MPLS_NON_EOS);
72 mld->mld_hdr[ii].label_exp_s_ttl =
73 clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
77 * the inner most label
79 ii = mld->mld_n_labels-1;
81 vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl, label_stack[ii]);
82 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl, ttl);
83 vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl, exp);
84 vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl, eos);
85 mld->mld_hdr[ii].label_exp_s_ttl =
86 clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
89 * stack this label objct on its parent.
91 dpo_stack(DPO_MPLS_LABEL,
92 mld->mld_payload_proto,
96 return (mpls_label_dpo_get_index(mld));
100 format_mpls_label_dpo (u8 *s, va_list *args)
102 index_t index = va_arg (*args, index_t);
103 u32 indent = va_arg (*args, u32);
104 mpls_unicast_header_t hdr;
105 mpls_label_dpo_t *mld;
108 s = format(s, "mpls-label:[%d]:", index);
110 if (pool_is_free_index(mpls_label_dpo_pool, index))
113 * the packet trace can be printed after the DPO has been deleted
118 mld = mpls_label_dpo_get(index);
120 for (ii = 0; ii < mld->mld_n_labels; ii++)
122 hdr.label_exp_s_ttl =
123 clib_net_to_host_u32(mld->mld_hdr[ii].label_exp_s_ttl);
124 s = format(s, "%U", format_mpls_header, hdr);
127 s = format(s, "\n%U", format_white_space, indent);
128 s = format(s, "%U", format_dpo_id, &mld->mld_dpo, indent+2);
134 mpls_label_dpo_lock (dpo_id_t *dpo)
136 mpls_label_dpo_t *mld;
138 mld = mpls_label_dpo_get(dpo->dpoi_index);
144 mpls_label_dpo_unlock (dpo_id_t *dpo)
146 mpls_label_dpo_t *mld;
148 mld = mpls_label_dpo_get(dpo->dpoi_index);
152 if (0 == mld->mld_locks)
154 dpo_reset(&mld->mld_dpo);
155 pool_put(mpls_label_dpo_pool, mld);
160 * @brief A struct to hold tracing information for the MPLS label imposition
163 typedef struct mpls_label_imposition_trace_t_
166 * The MPLS header imposed
168 mpls_unicast_header_t hdr;
169 } mpls_label_imposition_trace_t;
171 always_inline mpls_unicast_header_t *
172 mpls_label_paint (vlib_buffer_t * b0,
173 mpls_label_dpo_t *mld0,
176 mpls_unicast_header_t *hdr0;
178 vlib_buffer_advance(b0, -(mld0->mld_n_hdr_bytes));
180 hdr0 = vlib_buffer_get_current(b0);
182 if (1 == mld0->mld_n_labels)
184 /* optimise for the common case of one label */
185 *hdr0 = mld0->mld_hdr[0];
189 clib_memcpy(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes);
190 hdr0 = hdr0 + (mld0->mld_n_labels - 1);
192 /* fixup the TTL for the inner most label */
193 ((char*)hdr0)[3] = ttl0;
199 mpls_label_imposition_inline (vlib_main_t * vm,
200 vlib_node_runtime_t * node,
201 vlib_frame_t * from_frame,
204 u8 payload_is_ethernet)
206 u32 n_left_from, next_index, * from, * to_next;
208 from = vlib_frame_vector_args (from_frame);
209 n_left_from = from_frame->n_vectors;
211 next_index = node->cached_next_index;
213 while (n_left_from > 0)
217 vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
219 while (n_left_from >= 8 && n_left_to_next >= 4)
221 u32 bi0, mldi0, bi1, mldi1, bi2, mldi2, bi3, mldi3;
222 mpls_unicast_header_t *hdr0, *hdr1, *hdr2, *hdr3;
223 mpls_label_dpo_t *mld0, *mld1, *mld2, *mld3;
224 vlib_buffer_t * b0, *b1, * b2, *b3;
225 u32 next0, next1, next2, next3;
226 u8 ttl0, ttl1,ttl2, ttl3 ;
228 bi0 = to_next[0] = from[0];
229 bi1 = to_next[1] = from[1];
230 bi2 = to_next[2] = from[2];
231 bi3 = to_next[3] = from[3];
233 /* Prefetch next iteration. */
235 vlib_buffer_t * p2, * p3, *p4, *p5;
237 p2 = vlib_get_buffer (vm, from[2]);
238 p3 = vlib_get_buffer (vm, from[3]);
239 p4 = vlib_get_buffer (vm, from[4]);
240 p5 = vlib_get_buffer (vm, from[5]);
242 vlib_prefetch_buffer_header (p2, STORE);
243 vlib_prefetch_buffer_header (p3, STORE);
244 vlib_prefetch_buffer_header (p4, STORE);
245 vlib_prefetch_buffer_header (p5, STORE);
247 CLIB_PREFETCH (p2->data, sizeof (hdr0[0]), STORE);
248 CLIB_PREFETCH (p3->data, sizeof (hdr0[0]), STORE);
249 CLIB_PREFETCH (p4->data, sizeof (hdr0[0]), STORE);
250 CLIB_PREFETCH (p5->data, sizeof (hdr0[0]), STORE);
258 b0 = vlib_get_buffer (vm, bi0);
259 b1 = vlib_get_buffer (vm, bi1);
260 b2 = vlib_get_buffer (vm, bi2);
261 b3 = vlib_get_buffer (vm, bi3);
263 /* dst lookup was done by ip4 lookup */
264 mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
265 mldi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
266 mldi2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
267 mldi3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
268 mld0 = mpls_label_dpo_get(mldi0);
269 mld1 = mpls_label_dpo_get(mldi1);
270 mld2 = mpls_label_dpo_get(mldi2);
271 mld3 = mpls_label_dpo_get(mldi3);
276 * decrement the TTL on ingress to the LSP
278 ip4_header_t * ip0 = vlib_buffer_get_current(b0);
279 ip4_header_t * ip1 = vlib_buffer_get_current(b1);
280 ip4_header_t * ip2 = vlib_buffer_get_current(b2);
281 ip4_header_t * ip3 = vlib_buffer_get_current(b3);
287 checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
288 checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
289 checksum2 = ip2->checksum + clib_host_to_net_u16 (0x0100);
290 checksum3 = ip3->checksum + clib_host_to_net_u16 (0x0100);
292 checksum0 += checksum0 >= 0xffff;
293 checksum1 += checksum1 >= 0xffff;
294 checksum2 += checksum2 >= 0xffff;
295 checksum3 += checksum3 >= 0xffff;
297 ip0->checksum = checksum0;
298 ip1->checksum = checksum1;
299 ip2->checksum = checksum2;
300 ip3->checksum = checksum3;
312 else if (payload_is_ip6)
315 * decrement the TTL on ingress to the LSP
317 ip6_header_t * ip0 = vlib_buffer_get_current(b0);
318 ip6_header_t * ip1 = vlib_buffer_get_current(b1);
319 ip6_header_t * ip2 = vlib_buffer_get_current(b2);
320 ip6_header_t * ip3 = vlib_buffer_get_current(b3);
327 ttl0 = ip0->hop_limit;
328 ttl1 = ip1->hop_limit;
329 ttl2 = ip2->hop_limit;
330 ttl3 = ip3->hop_limit;
332 else if (payload_is_ethernet)
335 * nothing to chang ein the ethernet header
337 ttl0 = ttl1 = ttl2 = ttl3 = 255;
342 * else, the packet to be encapped is an MPLS packet
344 if (PREDICT_TRUE(vnet_buffer(b0)->mpls.first))
347 * The first label to be imposed on the packet. this is a label swap.
348 * in which case we stashed the TTL and EXP bits in the
349 * packet in the lookup node
351 ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
353 ttl0 = vnet_buffer(b0)->mpls.ttl - 1;
358 * not the first label. implying we are recusring down a chain of
360 * Each layer is considered a new LSP - hence the TTL is reset.
364 if (PREDICT_TRUE(vnet_buffer(b1)->mpls.first))
366 ASSERT(1 != vnet_buffer (b1)->mpls.ttl);
367 ttl1 = vnet_buffer(b1)->mpls.ttl - 1;
373 if (PREDICT_TRUE(vnet_buffer(b2)->mpls.first))
375 ASSERT(1 != vnet_buffer (b2)->mpls.ttl);
377 ttl2 = vnet_buffer(b2)->mpls.ttl - 1;
383 if (PREDICT_TRUE(vnet_buffer(b3)->mpls.first))
385 ASSERT(1 != vnet_buffer (b3)->mpls.ttl);
386 ttl3 = vnet_buffer(b3)->mpls.ttl - 1;
393 vnet_buffer(b0)->mpls.first = 0;
394 vnet_buffer(b1)->mpls.first = 0;
395 vnet_buffer(b2)->mpls.first = 0;
396 vnet_buffer(b3)->mpls.first = 0;
398 /* Paint the MPLS header */
399 hdr0 = mpls_label_paint(b0, mld0, ttl0);
400 hdr1 = mpls_label_paint(b1, mld1, ttl1);
401 hdr2 = mpls_label_paint(b2, mld2, ttl2);
402 hdr3 = mpls_label_paint(b3, mld3, ttl3);
404 next0 = mld0->mld_dpo.dpoi_next_node;
405 next1 = mld1->mld_dpo.dpoi_next_node;
406 next2 = mld2->mld_dpo.dpoi_next_node;
407 next3 = mld3->mld_dpo.dpoi_next_node;
408 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
409 vnet_buffer(b1)->ip.adj_index[VLIB_TX] = mld1->mld_dpo.dpoi_index;
410 vnet_buffer(b2)->ip.adj_index[VLIB_TX] = mld2->mld_dpo.dpoi_index;
411 vnet_buffer(b3)->ip.adj_index[VLIB_TX] = mld3->mld_dpo.dpoi_index;
413 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
415 mpls_label_imposition_trace_t *tr =
416 vlib_add_trace (vm, node, b0, sizeof (*tr));
419 if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
421 mpls_label_imposition_trace_t *tr =
422 vlib_add_trace (vm, node, b1, sizeof (*tr));
425 if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
427 mpls_label_imposition_trace_t *tr =
428 vlib_add_trace (vm, node, b2, sizeof (*tr));
431 if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
433 mpls_label_imposition_trace_t *tr =
434 vlib_add_trace (vm, node, b3, sizeof (*tr));
438 vlib_validate_buffer_enqueue_x4(vm, node, next_index, to_next,
441 next0, next1, next2, next3);
444 while (n_left_from > 0 && n_left_to_next > 0)
446 mpls_unicast_header_t *hdr0;
447 mpls_label_dpo_t *mld0;
460 b0 = vlib_get_buffer (vm, bi0);
462 /* dst lookup was done by ip4 lookup */
463 mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
464 mld0 = mpls_label_dpo_get(mldi0);
469 * decrement the TTL on ingress to the LSP
471 ip4_header_t * ip0 = vlib_buffer_get_current(b0);
474 checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
475 checksum0 += checksum0 >= 0xffff;
477 ip0->checksum = checksum0;
481 else if (payload_is_ip6)
484 * decrement the TTL on ingress to the LSP
486 ip6_header_t * ip0 = vlib_buffer_get_current(b0);
489 ttl = ip0->hop_limit;
494 * else, the packet to be encapped is an MPLS packet
496 if (vnet_buffer(b0)->mpls.first)
499 * The first label to be imposed on the packet. this is a label swap.
500 * in which case we stashed the TTL and EXP bits in the
501 * packet in the lookup node
503 ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
505 ttl = vnet_buffer(b0)->mpls.ttl - 1;
510 * not the first label. implying we are recusring down a chain of
512 * Each layer is considered a new LSP - hence the TTL is reset.
517 vnet_buffer(b0)->mpls.first = 0;
519 /* Paint the MPLS header */
520 vlib_buffer_advance(b0, -(mld0->mld_n_hdr_bytes));
521 hdr0 = vlib_buffer_get_current(b0);
522 clib_memcpy(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes);
524 /* fixup the TTL for the inner most label */
525 hdr0 = hdr0 + (mld0->mld_n_labels - 1);
526 ((char*)hdr0)[3] = ttl;
528 next0 = mld0->mld_dpo.dpoi_next_node;
529 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
531 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
533 mpls_label_imposition_trace_t *tr =
534 vlib_add_trace (vm, node, b0, sizeof (*tr));
538 vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
539 n_left_to_next, bi0, next0);
541 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
543 return from_frame->n_vectors;
547 format_mpls_label_imposition_trace (u8 * s, va_list * args)
549 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
550 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
551 mpls_label_imposition_trace_t * t;
552 mpls_unicast_header_t hdr;
555 t = va_arg (*args, mpls_label_imposition_trace_t *);
556 indent = format_get_indent (s);
557 hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
559 s = format (s, "%Umpls-header:%U",
560 format_white_space, indent,
561 format_mpls_header, hdr);
566 mpls_label_imposition (vlib_main_t * vm,
567 vlib_node_runtime_t * node,
568 vlib_frame_t * frame)
570 return (mpls_label_imposition_inline(vm, node, frame, 0, 0, 0));
573 VLIB_REGISTER_NODE (mpls_label_imposition_node) = {
574 .function = mpls_label_imposition,
575 .name = "mpls-label-imposition",
576 .vector_size = sizeof (u32),
578 .format_trace = format_mpls_label_imposition_trace,
584 VLIB_NODE_FUNCTION_MULTIARCH (mpls_label_imposition_node,
585 mpls_label_imposition)
588 ip4_mpls_label_imposition (vlib_main_t * vm,
589 vlib_node_runtime_t * node,
590 vlib_frame_t * frame)
592 return (mpls_label_imposition_inline(vm, node, frame, 1, 0, 0));
595 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_node) = {
596 .function = ip4_mpls_label_imposition,
597 .name = "ip4-mpls-label-imposition",
598 .vector_size = sizeof (u32),
600 .format_trace = format_mpls_label_imposition_trace,
606 VLIB_NODE_FUNCTION_MULTIARCH (ip4_mpls_label_imposition_node,
607 ip4_mpls_label_imposition)
610 ip6_mpls_label_imposition (vlib_main_t * vm,
611 vlib_node_runtime_t * node,
612 vlib_frame_t * frame)
614 return (mpls_label_imposition_inline(vm, node, frame, 0, 1, 0));
617 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_node) = {
618 .function = ip6_mpls_label_imposition,
619 .name = "ip6-mpls-label-imposition",
620 .vector_size = sizeof (u32),
622 .format_trace = format_mpls_label_imposition_trace,
628 VLIB_NODE_FUNCTION_MULTIARCH (ip6_mpls_label_imposition_node,
629 ip6_mpls_label_imposition)
632 ethernet_mpls_label_imposition (vlib_main_t * vm,
633 vlib_node_runtime_t * node,
634 vlib_frame_t * frame)
636 return (mpls_label_imposition_inline(vm, node, frame, 0, 0, 1));
639 VLIB_REGISTER_NODE (ethernet_mpls_label_imposition_node) = {
640 .function = ethernet_mpls_label_imposition,
641 .name = "ethernet-mpls-label-imposition",
642 .vector_size = sizeof (u32),
644 .format_trace = format_mpls_label_imposition_trace,
650 VLIB_NODE_FUNCTION_MULTIARCH (ethernet_mpls_label_imposition_node,
651 ethernet_mpls_label_imposition)
654 mpls_label_dpo_mem_show (void)
656 fib_show_memory_usage("MPLS label",
657 pool_elts(mpls_label_dpo_pool),
658 pool_len(mpls_label_dpo_pool),
659 sizeof(mpls_label_dpo_t));
662 const static dpo_vft_t mld_vft = {
663 .dv_lock = mpls_label_dpo_lock,
664 .dv_unlock = mpls_label_dpo_unlock,
665 .dv_format = format_mpls_label_dpo,
666 .dv_mem_show = mpls_label_dpo_mem_show,
669 const static char* const mpls_label_imp_ip4_nodes[] =
671 "ip4-mpls-label-imposition",
674 const static char* const mpls_label_imp_ip6_nodes[] =
676 "ip6-mpls-label-imposition",
679 const static char* const mpls_label_imp_mpls_nodes[] =
681 "mpls-label-imposition",
684 const static char* const mpls_label_imp_ethernet_nodes[] =
686 "ethernet-mpls-label-imposition",
690 const static char* const * const mpls_label_imp_nodes[DPO_PROTO_NUM] =
692 [DPO_PROTO_IP4] = mpls_label_imp_ip4_nodes,
693 [DPO_PROTO_IP6] = mpls_label_imp_ip6_nodes,
694 [DPO_PROTO_MPLS] = mpls_label_imp_mpls_nodes,
695 [DPO_PROTO_ETHERNET] = mpls_label_imp_ethernet_nodes,
700 mpls_label_dpo_module_init (void)
702 dpo_register(DPO_MPLS_LABEL, &mld_vft, mpls_label_imp_nodes);