2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/ip/ip.h>
17 #include <vnet/dpo/mpls_label_dpo.h>
18 #include <vnet/mpls/mpls.h>
21 * pool of all MPLS Label DPOs
23 mpls_label_dpo_t *mpls_label_dpo_pool;
25 static mpls_label_dpo_t *
26 mpls_label_dpo_alloc (void)
28 mpls_label_dpo_t *mld;
30 pool_get_aligned(mpls_label_dpo_pool, mld, CLIB_CACHE_LINE_BYTES);
31 memset(mld, 0, sizeof(*mld));
33 dpo_reset(&mld->mld_dpo);
39 mpls_label_dpo_get_index (mpls_label_dpo_t *mld)
41 return (mld - mpls_label_dpo_pool);
45 mpls_label_dpo_create (mpls_label_t *label_stack,
49 dpo_proto_t payload_proto,
52 mpls_label_dpo_t *mld;
55 mld = mpls_label_dpo_alloc();
56 mld->mld_n_labels = vec_len(label_stack);
57 mld->mld_n_hdr_bytes = mld->mld_n_labels * sizeof(mld->mld_hdr[0]);
58 mld->mld_payload_proto = payload_proto;
61 * construct label rewrite headers for each value value passed.
62 * get the header in network byte order since we will paint it
63 * on a packet in the data-plane
66 for (ii = 0; ii < mld->mld_n_labels-1; ii++)
68 vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl, label_stack[ii]);
69 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl, 255);
70 vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl, 0);
71 vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl, MPLS_NON_EOS);
72 mld->mld_hdr[ii].label_exp_s_ttl =
73 clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
77 * the inner most label
79 ii = mld->mld_n_labels-1;
81 vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl, label_stack[ii]);
82 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl, ttl);
83 vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl, exp);
84 vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl, eos);
85 mld->mld_hdr[ii].label_exp_s_ttl =
86 clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
89 * stack this label objct on its parent.
91 dpo_stack(DPO_MPLS_LABEL,
92 mld->mld_payload_proto,
96 return (mpls_label_dpo_get_index(mld));
100 format_mpls_label_dpo (u8 *s, va_list *args)
102 index_t index = va_arg (*args, index_t);
103 u32 indent = va_arg (*args, u32);
104 mpls_unicast_header_t hdr;
105 mpls_label_dpo_t *mld;
108 mld = mpls_label_dpo_get(index);
110 s = format(s, "mpls-label:[%d]:", index);
112 for (ii = 0; ii < mld->mld_n_labels; ii++)
114 hdr.label_exp_s_ttl =
115 clib_net_to_host_u32(mld->mld_hdr[ii].label_exp_s_ttl);
116 s = format(s, "%U", format_mpls_header, hdr);
119 s = format(s, "\n%U", format_white_space, indent);
120 s = format(s, "%U", format_dpo_id, &mld->mld_dpo, indent+2);
126 mpls_label_dpo_lock (dpo_id_t *dpo)
128 mpls_label_dpo_t *mld;
130 mld = mpls_label_dpo_get(dpo->dpoi_index);
136 mpls_label_dpo_unlock (dpo_id_t *dpo)
138 mpls_label_dpo_t *mld;
140 mld = mpls_label_dpo_get(dpo->dpoi_index);
144 if (0 == mld->mld_locks)
146 dpo_reset(&mld->mld_dpo);
147 pool_put(mpls_label_dpo_pool, mld);
152 * @brief A struct to hold tracing information for the MPLS label imposition
155 typedef struct mpls_label_imposition_trace_t_
158 * The MPLS header imposed
160 mpls_unicast_header_t hdr;
161 } mpls_label_imposition_trace_t;
163 always_inline mpls_unicast_header_t *
164 mpls_label_paint (vlib_buffer_t * b0,
165 mpls_label_dpo_t *mld0,
168 mpls_unicast_header_t *hdr0;
170 vlib_buffer_advance(b0, -(mld0->mld_n_hdr_bytes));
172 hdr0 = vlib_buffer_get_current(b0);
174 if (PREDICT_TRUE(1 == mld0->mld_n_labels))
176 /* optimise for the common case of one label */
177 *hdr0 = mld0->mld_hdr[0];
181 clib_memcpy(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes);
182 hdr0 = hdr0 + (mld0->mld_n_labels - 1);
184 /* fixup the TTL for the inner most label */
185 ((char*)hdr0)[3] = ttl0;
191 mpls_label_imposition_inline (vlib_main_t * vm,
192 vlib_node_runtime_t * node,
193 vlib_frame_t * from_frame,
197 u32 n_left_from, next_index, * from, * to_next;
199 from = vlib_frame_vector_args (from_frame);
200 n_left_from = from_frame->n_vectors;
202 next_index = node->cached_next_index;
204 while (n_left_from > 0)
208 vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
210 while (n_left_from >= 8 && n_left_to_next >= 4)
212 u32 bi0, mldi0, bi1, mldi1, bi2, mldi2, bi3, mldi3;
213 mpls_unicast_header_t *hdr0, *hdr1, *hdr2, *hdr3;
214 mpls_label_dpo_t *mld0, *mld1, *mld2, *mld3;
215 vlib_buffer_t * b0, *b1, * b2, *b3;
216 u32 next0, next1, next2, next3;
217 u8 ttl0, ttl1,ttl2, ttl3 ;
219 bi0 = to_next[0] = from[0];
220 bi1 = to_next[1] = from[1];
221 bi2 = to_next[2] = from[2];
222 bi3 = to_next[3] = from[3];
224 /* Prefetch next iteration. */
226 vlib_buffer_t * p2, * p3, *p4, *p5;
228 p2 = vlib_get_buffer (vm, from[2]);
229 p3 = vlib_get_buffer (vm, from[3]);
230 p4 = vlib_get_buffer (vm, from[4]);
231 p5 = vlib_get_buffer (vm, from[5]);
233 vlib_prefetch_buffer_header (p2, STORE);
234 vlib_prefetch_buffer_header (p3, STORE);
235 vlib_prefetch_buffer_header (p4, STORE);
236 vlib_prefetch_buffer_header (p5, STORE);
238 CLIB_PREFETCH (p2->data, sizeof (hdr0[0]), STORE);
239 CLIB_PREFETCH (p3->data, sizeof (hdr0[0]), STORE);
240 CLIB_PREFETCH (p4->data, sizeof (hdr0[0]), STORE);
241 CLIB_PREFETCH (p5->data, sizeof (hdr0[0]), STORE);
249 b0 = vlib_get_buffer (vm, bi0);
250 b1 = vlib_get_buffer (vm, bi1);
251 b2 = vlib_get_buffer (vm, bi2);
252 b3 = vlib_get_buffer (vm, bi3);
254 /* dst lookup was done by ip4 lookup */
255 mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
256 mldi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
257 mldi2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
258 mldi3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
259 mld0 = mpls_label_dpo_get(mldi0);
260 mld1 = mpls_label_dpo_get(mldi1);
261 mld2 = mpls_label_dpo_get(mldi2);
262 mld3 = mpls_label_dpo_get(mldi3);
267 * decrement the TTL on ingress to the LSP
269 ip4_header_t * ip0 = vlib_buffer_get_current(b0);
270 ip4_header_t * ip1 = vlib_buffer_get_current(b1);
271 ip4_header_t * ip2 = vlib_buffer_get_current(b2);
272 ip4_header_t * ip3 = vlib_buffer_get_current(b3);
278 checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
279 checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
280 checksum2 = ip2->checksum + clib_host_to_net_u16 (0x0100);
281 checksum3 = ip3->checksum + clib_host_to_net_u16 (0x0100);
283 checksum0 += checksum0 >= 0xffff;
284 checksum1 += checksum1 >= 0xffff;
285 checksum2 += checksum2 >= 0xffff;
286 checksum3 += checksum3 >= 0xffff;
288 ip0->checksum = checksum0;
289 ip1->checksum = checksum1;
290 ip2->checksum = checksum2;
291 ip3->checksum = checksum3;
303 else if (payload_is_ip6)
306 * decrement the TTL on ingress to the LSP
308 ip6_header_t * ip0 = vlib_buffer_get_current(b0);
309 ip6_header_t * ip1 = vlib_buffer_get_current(b1);
310 ip6_header_t * ip2 = vlib_buffer_get_current(b2);
311 ip6_header_t * ip3 = vlib_buffer_get_current(b3);
318 ttl0 = ip0->hop_limit;
319 ttl1 = ip1->hop_limit;
320 ttl2 = ip2->hop_limit;
321 ttl3 = ip3->hop_limit;
326 * else, the packet to be encapped is an MPLS packet
328 if (PREDICT_TRUE(vnet_buffer(b0)->mpls.first))
331 * The first label to be imposed on the packet. this is a label swap.
332 * in which case we stashed the TTL and EXP bits in the
333 * packet in the lookup node
335 ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
337 ttl0 = vnet_buffer(b0)->mpls.ttl - 1;
342 * not the first label. implying we are recusring down a chain of
344 * Each layer is considered a new LSP - hence the TTL is reset.
348 if (PREDICT_TRUE(vnet_buffer(b1)->mpls.first))
350 ASSERT(1 != vnet_buffer (b1)->mpls.ttl);
351 ttl1 = vnet_buffer(b1)->mpls.ttl - 1;
357 if (PREDICT_TRUE(vnet_buffer(b2)->mpls.first))
359 ASSERT(2 != vnet_buffer (b2)->mpls.ttl);
361 ttl2 = vnet_buffer(b2)->mpls.ttl - 1;
367 if (PREDICT_TRUE(vnet_buffer(b3)->mpls.first))
369 ASSERT(1 != vnet_buffer (b3)->mpls.ttl);
370 ttl3 = vnet_buffer(b3)->mpls.ttl - 1;
377 vnet_buffer(b0)->mpls.first = 0;
378 vnet_buffer(b1)->mpls.first = 0;
379 vnet_buffer(b2)->mpls.first = 0;
380 vnet_buffer(b3)->mpls.first = 0;
382 /* Paint the MPLS header */
383 hdr0 = mpls_label_paint(b0, mld0, ttl0);
384 hdr1 = mpls_label_paint(b1, mld1, ttl1);
385 hdr2 = mpls_label_paint(b2, mld2, ttl2);
386 hdr3 = mpls_label_paint(b3, mld3, ttl3);
388 next0 = mld0->mld_dpo.dpoi_next_node;
389 next1 = mld1->mld_dpo.dpoi_next_node;
390 next2 = mld2->mld_dpo.dpoi_next_node;
391 next3 = mld3->mld_dpo.dpoi_next_node;
392 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
393 vnet_buffer(b1)->ip.adj_index[VLIB_TX] = mld1->mld_dpo.dpoi_index;
394 vnet_buffer(b2)->ip.adj_index[VLIB_TX] = mld2->mld_dpo.dpoi_index;
395 vnet_buffer(b3)->ip.adj_index[VLIB_TX] = mld3->mld_dpo.dpoi_index;
397 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
399 mpls_label_imposition_trace_t *tr =
400 vlib_add_trace (vm, node, b0, sizeof (*tr));
403 if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
405 mpls_label_imposition_trace_t *tr =
406 vlib_add_trace (vm, node, b1, sizeof (*tr));
409 if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
411 mpls_label_imposition_trace_t *tr =
412 vlib_add_trace (vm, node, b2, sizeof (*tr));
415 if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
417 mpls_label_imposition_trace_t *tr =
418 vlib_add_trace (vm, node, b3, sizeof (*tr));
422 vlib_validate_buffer_enqueue_x4(vm, node, next_index, to_next,
425 next0, next1, next2, next3);
428 while (n_left_from > 0 && n_left_to_next > 0)
430 mpls_unicast_header_t *hdr0;
431 mpls_label_dpo_t *mld0;
444 b0 = vlib_get_buffer (vm, bi0);
446 /* dst lookup was done by ip4 lookup */
447 mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
448 mld0 = mpls_label_dpo_get(mldi0);
453 * decrement the TTL on ingress to the LSP
455 ip4_header_t * ip0 = vlib_buffer_get_current(b0);
458 checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
459 checksum0 += checksum0 >= 0xffff;
461 ip0->checksum = checksum0;
465 else if (payload_is_ip6)
468 * decrement the TTL on ingress to the LSP
470 ip6_header_t * ip0 = vlib_buffer_get_current(b0);
473 ttl = ip0->hop_limit;
478 * else, the packet to be encapped is an MPLS packet
480 if (vnet_buffer(b0)->mpls.first)
483 * The first label to be imposed on the packet. this is a label swap.
484 * in which case we stashed the TTL and EXP bits in the
485 * packet in the lookup node
487 ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
489 ttl = vnet_buffer(b0)->mpls.ttl - 1;
494 * not the first label. implying we are recusring down a chain of
496 * Each layer is considered a new LSP - hence the TTL is reset.
501 vnet_buffer(b0)->mpls.first = 0;
503 /* Paint the MPLS header */
504 vlib_buffer_advance(b0, -(mld0->mld_n_hdr_bytes));
505 hdr0 = vlib_buffer_get_current(b0);
506 clib_memcpy(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes);
508 /* fixup the TTL for the inner most label */
509 hdr0 = hdr0 + (mld0->mld_n_labels - 1);
510 ((char*)hdr0)[3] = ttl;
512 next0 = mld0->mld_dpo.dpoi_next_node;
513 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
515 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
517 mpls_label_imposition_trace_t *tr =
518 vlib_add_trace (vm, node, b0, sizeof (*tr));
522 vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
523 n_left_to_next, bi0, next0);
525 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
527 return from_frame->n_vectors;
531 format_mpls_label_imposition_trace (u8 * s, va_list * args)
533 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
534 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
535 mpls_label_imposition_trace_t * t;
536 mpls_unicast_header_t hdr;
539 t = va_arg (*args, mpls_label_imposition_trace_t *);
540 indent = format_get_indent (s);
541 hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
543 s = format (s, "%Umpls-header:%U",
544 format_white_space, indent,
545 format_mpls_header, hdr);
550 mpls_label_imposition (vlib_main_t * vm,
551 vlib_node_runtime_t * node,
552 vlib_frame_t * frame)
554 return (mpls_label_imposition_inline(vm, node, frame, 0, 0));
557 VLIB_REGISTER_NODE (mpls_label_imposition_node) = {
558 .function = mpls_label_imposition,
559 .name = "mpls-label-imposition",
560 .vector_size = sizeof (u32),
562 .format_trace = format_mpls_label_imposition_trace,
568 VLIB_NODE_FUNCTION_MULTIARCH (mpls_label_imposition_node,
569 mpls_label_imposition)
572 ip4_mpls_label_imposition (vlib_main_t * vm,
573 vlib_node_runtime_t * node,
574 vlib_frame_t * frame)
576 return (mpls_label_imposition_inline(vm, node, frame, 1, 0));
579 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_node) = {
580 .function = ip4_mpls_label_imposition,
581 .name = "ip4-mpls-label-imposition",
582 .vector_size = sizeof (u32),
584 .format_trace = format_mpls_label_imposition_trace,
590 VLIB_NODE_FUNCTION_MULTIARCH (ip4_mpls_label_imposition_node,
591 ip4_mpls_label_imposition)
594 ip6_mpls_label_imposition (vlib_main_t * vm,
595 vlib_node_runtime_t * node,
596 vlib_frame_t * frame)
598 return (mpls_label_imposition_inline(vm, node, frame, 0, 1));
601 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_node) = {
602 .function = ip6_mpls_label_imposition,
603 .name = "ip6-mpls-label-imposition",
604 .vector_size = sizeof (u32),
606 .format_trace = format_mpls_label_imposition_trace,
612 VLIB_NODE_FUNCTION_MULTIARCH (ip6_mpls_label_imposition_node,
613 ip6_mpls_label_imposition)
616 mpls_label_dpo_mem_show (void)
618 fib_show_memory_usage("MPLS label",
619 pool_elts(mpls_label_dpo_pool),
620 pool_len(mpls_label_dpo_pool),
621 sizeof(mpls_label_dpo_t));
624 const static dpo_vft_t mld_vft = {
625 .dv_lock = mpls_label_dpo_lock,
626 .dv_unlock = mpls_label_dpo_unlock,
627 .dv_format = format_mpls_label_dpo,
628 .dv_mem_show = mpls_label_dpo_mem_show,
631 const static char* const mpls_label_imp_ip4_nodes[] =
633 "ip4-mpls-label-imposition",
636 const static char* const mpls_label_imp_ip6_nodes[] =
638 "ip6-mpls-label-imposition",
641 const static char* const mpls_label_imp_mpls_nodes[] =
643 "mpls-label-imposition",
646 const static char* const * const mpls_label_imp_nodes[DPO_PROTO_NUM] =
648 [DPO_PROTO_IP4] = mpls_label_imp_ip4_nodes,
649 [DPO_PROTO_IP6] = mpls_label_imp_ip6_nodes,
650 [DPO_PROTO_MPLS] = mpls_label_imp_mpls_nodes,
655 mpls_label_dpo_module_init (void)
657 dpo_register(DPO_MPLS_LABEL, &mld_vft, mpls_label_imp_nodes);