src/vnet/dpo/mpls_label_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/mpls_label_dpo.h>
  18 #include <vnet/mpls/mpls.h>
  19 #include <vnet/dpo/drop_dpo.h>
  20
  21 #ifndef CLIB_MARCH_VARIANT
  22 /*
  23  * pool of all MPLS Label DPOs
  24  */
  25 mpls_label_dpo_t *mpls_label_dpo_pool;
  26
  27 /**
  28  * Strings for the flags
  29  */
  30 const char* mpls_label_dpo_attr_names[] = MPLS_LABEL_DPO_ATTR_NAMES;
  31
  32 /**
  33  * registered DPO types for each of the label sub-types. And there's a
  34  * subtype for each of the flag combinations.
  35  */
  36 static dpo_type_t mpls_label_dpo_types[1 << MPLS_LABEL_DPO_ATTR_MAX];
  37
  38 static mpls_label_dpo_t *
  39 mpls_label_dpo_alloc (void)
  40 {
  41     mpls_label_dpo_t *mld;
  42
  43     pool_get_aligned(mpls_label_dpo_pool, mld, CLIB_CACHE_LINE_BYTES);
  44     clib_memset(mld, 0, sizeof(*mld));
  45
  46     dpo_reset(&mld->mld_dpo);
  47
  48     return (mld);
  49 }
  50
  51 static index_t
  52 mpls_label_dpo_get_index (mpls_label_dpo_t *mld)
  53 {
  54     return (mld - mpls_label_dpo_pool);
  55 }
  56
  57 void
  58 mpls_label_dpo_create (fib_mpls_label_t *label_stack,
  59                        mpls_eos_bit_t eos,
  60                        dpo_proto_t payload_proto,
  61                        mpls_label_dpo_flags_t flags,
  62                        const dpo_id_t *parent,
  63                        dpo_id_t *dpo)
  64 {
  65     mpls_label_dpo_t *mld;
  66     dpo_type_t dtype;
  67     u32 ii;
  68
  69     if ((DPO_PROTO_IP4 != payload_proto) &&
  70         (DPO_PROTO_IP6 != payload_proto))
  71     {
  72         /*
  73          * remove unsupported configuration
  74          */
  75         flags &= ~MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR;
  76     }
  77
  78     mld = mpls_label_dpo_alloc();
  79     mld->mld_flags = flags;
  80     dtype = mpls_label_dpo_types[flags];
  81
  82     if (MPLS_LABEL_DPO_MAX_N_LABELS < vec_len(label_stack))
  83     {
  84         clib_warning("Label stack size exceeded");
  85         dpo_stack(dtype,
  86                   mld->mld_payload_proto,
  87                   &mld->mld_dpo,
  88                   drop_dpo_get(DPO_PROTO_MPLS));
  89     }
  90     else
  91     {
  92         mld->mld_n_labels = vec_len(label_stack);
  93         mld->mld_n_hdr_bytes = mld->mld_n_labels * sizeof(mld->mld_hdr[0]);
  94         mld->mld_payload_proto = payload_proto;
  95
  96         /*
  97          * construct label rewrite headers for each value passed.
  98          * get the header in network byte order since we will paint it
  99          * on a packet in the data-plane
 100          */
 101         for (ii = 0; ii < mld->mld_n_labels-1; ii++)
 102         {
 103             vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl,
 104                                    label_stack[ii].fml_value);
 105             vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl,
 106                                  label_stack[ii].fml_exp);
 107             vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl,
 108                                MPLS_NON_EOS);
 109             if (0 != label_stack[ii].fml_ttl)
 110             {
 111                 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 112                                      label_stack[ii].fml_ttl);
 113             }
 114             else
 115             {
 116                 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 117                                      MPLS_LABEL_DEFAULT_TTL);
 118             }
 119             mld->mld_hdr[ii].label_exp_s_ttl =
 120                 clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
 121         }
 122
 123         /*
 124          * the inner most label
 125          */
 126         ii = mld->mld_n_labels-1;
 127
 128         vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl,
 129                                label_stack[ii].fml_value);
 130         vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl,
 131                              label_stack[ii].fml_exp);
 132         vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl, eos);
 133         if (0 != label_stack[ii].fml_ttl)
 134         {
 135             vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 136                                  label_stack[ii].fml_ttl);
 137         }
 138         else
 139         {
 140             vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 141                                  MPLS_LABEL_DEFAULT_TTL);
 142         }
 143         mld->mld_hdr[ii].label_exp_s_ttl =
 144             clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
 145
 146         /*
 147          * pipe/uniform mode is only supported for the bottom of stack label
 148          */
 149         if (FIB_MPLS_LSP_MODE_UNIFORM == label_stack[ii].fml_mode)
 150         {
 151             mld->mld_flags |= MPLS_LABEL_DPO_FLAG_UNIFORM_MODE;
 152         }
 153         else
 154         {
 155             mld->mld_flags &= ~MPLS_LABEL_DPO_FLAG_UNIFORM_MODE;
 156         }
 157         dtype = mpls_label_dpo_types[mld->mld_flags];
 158
 159         /*
 160          * stack this label object on its parent.
 161          */
 162         dpo_stack(dtype,
 163                   mld->mld_payload_proto,
 164                   &mld->mld_dpo,
 165                   parent);
 166     }
 167
 168     dpo_set(dpo,
 169             dtype,
 170             mld->mld_payload_proto,
 171             mpls_label_dpo_get_index(mld));
 172 }
 173
 174 u8*
 175 format_mpls_label_dpo_flags (u8 *s, va_list *args)
 176 {
 177     mpls_label_dpo_flags_t flags = va_arg (*args, int);
 178     mpls_label_dpo_attr_t attr;
 179
 180     FOR_EACH_MPLS_LABEL_DPO_ATTR(attr)
 181     {
 182         if ((1 << attr) & flags)
 183         {
 184             s = format(s, "%s,", mpls_label_dpo_attr_names[attr]);
 185         }
 186     }
 187
 188     return (s);
 189 }
 190
 191 u8*
 192 format_mpls_label_dpo (u8 *s, va_list *args)
 193 {
 194     index_t index = va_arg (*args, index_t);
 195     u32 indent = va_arg (*args, u32);
 196     mpls_unicast_header_t hdr;
 197     mpls_label_dpo_t *mld;
 198     u32 ii;
 199
 200     if (pool_is_free_index(mpls_label_dpo_pool, index))
 201     {
 202         /*
 203          * the packet trace can be printed after the DPO has been deleted
 204          */
 205         return (format(s, "mpls-label[???,%d]:", index));
 206     }
 207
 208     mld = mpls_label_dpo_get(index);
 209     s = format(s, "mpls-label[%U@%d]:",
 210                format_mpls_label_dpo_flags,
 211                (int) mld->mld_flags, index);
 212
 213     for (ii = 0; ii < mld->mld_n_labels; ii++)
 214     {
 215         hdr.label_exp_s_ttl =
 216             clib_net_to_host_u32(mld->mld_hdr[ii].label_exp_s_ttl);
 217         s = format(s, "%U", format_mpls_header, hdr);
 218     }
 219
 220     s = format(s, "\n%U", format_white_space, indent);
 221     s = format(s, "%U", format_dpo_id, &mld->mld_dpo, indent+2);
 222
 223     return (s);
 224 }
 225
 226 static void
 227 mpls_label_dpo_lock (dpo_id_t *dpo)
 228 {
 229     mpls_label_dpo_t *mld;
 230
 231     mld = mpls_label_dpo_get(dpo->dpoi_index);
 232
 233     mld->mld_locks++;
 234 }
 235
 236 static void
 237 mpls_label_dpo_unlock (dpo_id_t *dpo)
 238 {
 239     mpls_label_dpo_t *mld;
 240
 241     mld = mpls_label_dpo_get(dpo->dpoi_index);
 242
 243     mld->mld_locks--;
 244
 245     if (0 == mld->mld_locks)
 246     {
 247         dpo_reset(&mld->mld_dpo);
 248         pool_put(mpls_label_dpo_pool, mld);
 249     }
 250 }
 251 #endif /* CLIB_MARCH_VARIANT */
 252
 253 /**
 254  * @brief A struct to hold tracing information for the MPLS label imposition
 255  * node.
 256  */
 257 typedef struct mpls_label_imposition_trace_t_
 258 {
 259     /**
 260      * The MPLS header imposed
 261      */
 262     mpls_unicast_header_t hdr;
 263
 264     /**
 265      * TTL imposed - only valid for uniform LSPs
 266      */
 267     u8 ttl;
 268
 269     /**
 270      * TTL imposed - only valid for uniform LSPs
 271      */
 272     u8 exp;
 273 } mpls_label_imposition_trace_t;
 274
 275 always_inline mpls_unicast_header_t *
 276 mpls_label_paint (vlib_buffer_t * b0,
 277                   mpls_label_dpo_t *mld0)
 278 {
 279     mpls_unicast_header_t *hdr0;
 280
 281     vlib_buffer_advance(b0, -(mld0->mld_n_hdr_bytes));
 282
 283     hdr0 = vlib_buffer_get_current(b0);
 284
 285     if (1 == mld0->mld_n_labels)
 286     {
 287         /* optimise for the common case of one label */
 288         *hdr0 = mld0->mld_hdr[0];
 289     }
 290     else
 291     {
 292         clib_memcpy_fast(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes);
 293         hdr0 = hdr0 + (mld0->mld_n_labels - 1);
 294     }
 295
 296     return (hdr0);
 297 }
 298
 299 /**
 300  * Paint on an MPLS label and fixup the TTL
 301  */
 302 always_inline mpls_unicast_header_t *
 303 mpls_label_paint_w_ttl (vlib_buffer_t * b0,
 304                         mpls_label_dpo_t *mld0,
 305                         u8 ttl0)
 306 {
 307     mpls_unicast_header_t *hdr0;
 308
 309     hdr0 = mpls_label_paint(b0, mld0);
 310
 311     /* fixup the TTL for the inner most label */
 312     ((char*)hdr0)[3] = ttl0;
 313
 314     return (hdr0);
 315 }
 316
 317 /**
 318  * Paint on an MPLS label and fixup the TTL and EXP bits.
 319  */
 320 always_inline mpls_unicast_header_t *
 321 mpls_label_paint_w_ttl_exp (vlib_buffer_t * b0,
 322                             mpls_label_dpo_t *mld0,
 323                             u8 ttl0,
 324                             u8 exp0)
 325 {
 326     mpls_unicast_header_t *hdr0;
 327
 328     hdr0 = mpls_label_paint_w_ttl(b0, mld0, ttl0);
 329
 330     /* fixup the EXP for the inner most label */
 331     ((char*)hdr0)[2] |= (exp0 << 1);
 332
 333     return (hdr0);
 334 }
 335
 336 /**
 337  * Paint on an MPLS label and fixup the TTL and EXP bits
 338  * When the EXP bits are *already* bit shift to the correct place in
 339  * in the 2nd byte (i.e. they were read from another label)
 340  */
 341 always_inline mpls_unicast_header_t *
 342 mpls_label_paint_w_ttl_mpls_exp (vlib_buffer_t * b0,
 343                                  mpls_label_dpo_t *mld0,
 344                                  u8 ttl0,
 345                                  u8 exp0)
 346 {
 347     mpls_unicast_header_t *hdr0;
 348
 349     hdr0 = mpls_label_paint_w_ttl(b0, mld0, ttl0);
 350
 351     /* fixup the EXP for the inner most label */
 352     ((char*)hdr0)[2] |= exp0;
 353
 354     return (hdr0);
 355 }
 356
 357 always_inline uword
 358 mpls_label_imposition_inline (vlib_main_t * vm,
 359                               vlib_node_runtime_t * node,
 360                               vlib_frame_t * from_frame,
 361                               const dpo_proto_t dproto,
 362                               const mpls_label_dpo_flags_t flags)
 363 {
 364     u32 n_left_from, next_index, * from, * to_next;
 365
 366     from = vlib_frame_vector_args (from_frame);
 367     n_left_from = from_frame->n_vectors;
 368
 369     next_index = node->cached_next_index;
 370
 371     while (n_left_from > 0)
 372     {
 373         u32 n_left_to_next;
 374
 375         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 376
 377         while (n_left_from >= 8 && n_left_to_next >= 4)
 378         {
 379             u32 bi0, mldi0, bi1, mldi1, bi2, mldi2, bi3, mldi3;
 380             mpls_unicast_header_t *hdr0, *hdr1, *hdr2, *hdr3;
 381             mpls_label_dpo_t *mld0, *mld1, *mld2, *mld3;
 382             vlib_buffer_t * b0, *b1, * b2, *b3;
 383             u32 next0, next1, next2, next3;
 384             u8 ttl0, ttl1, ttl2, ttl3;
 385             u8 exp0, exp1, exp2, exp3;
 386
 387             bi0 = to_next[0] = from[0];
 388             bi1 = to_next[1] = from[1];
 389             bi2 = to_next[2] = from[2];
 390             bi3 = to_next[3] = from[3];
 391
 392             /* Prefetch next iteration. */
 393             {
 394                 vlib_buffer_t * p2, * p3, *p4, *p5;
 395
 396                 p2 = vlib_get_buffer (vm, from[2]);
 397                 p3 = vlib_get_buffer (vm, from[3]);
 398                 p4 = vlib_get_buffer (vm, from[4]);
 399                 p5 = vlib_get_buffer (vm, from[5]);
 400
 401                 vlib_prefetch_buffer_header (p2, STORE);
 402                 vlib_prefetch_buffer_header (p3, STORE);
 403                 vlib_prefetch_buffer_header (p4, STORE);
 404                 vlib_prefetch_buffer_header (p5, STORE);
 405
 406                 CLIB_PREFETCH (p2->data, sizeof (hdr0[0]), STORE);
 407                 CLIB_PREFETCH (p3->data, sizeof (hdr0[0]), STORE);
 408                 CLIB_PREFETCH (p4->data, sizeof (hdr0[0]), STORE);
 409                 CLIB_PREFETCH (p5->data, sizeof (hdr0[0]), STORE);
 410             }
 411
 412             from += 4;
 413             to_next += 4;
 414             n_left_from -= 4;
 415             n_left_to_next -= 4;
 416
 417             b0 = vlib_get_buffer (vm, bi0);
 418             b1 = vlib_get_buffer (vm, bi1);
 419             b2 = vlib_get_buffer (vm, bi2);
 420             b3 = vlib_get_buffer (vm, bi3);
 421
 422             /* dst lookup was done by ip4 lookup */
 423             mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 424             mldi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 425             mldi2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
 426             mldi3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
 427             mld0 = mpls_label_dpo_get(mldi0);
 428             mld1 = mpls_label_dpo_get(mldi1);
 429             mld2 = mpls_label_dpo_get(mldi2);
 430             mld3 = mpls_label_dpo_get(mldi3);
 431
 432             if (DPO_PROTO_MPLS != dproto)
 433             {
 434                 /*
 435                  * These are the non-MPLS payload imposition cases
 436                  */
 437                 if (DPO_PROTO_IP4 == dproto)
 438                 {
 439                     ip4_header_t * ip0 = vlib_buffer_get_current(b0);
 440                     ip4_header_t * ip1 = vlib_buffer_get_current(b1);
 441                     ip4_header_t * ip2 = vlib_buffer_get_current(b2);
 442                     ip4_header_t * ip3 = vlib_buffer_get_current(b3);
 443
 444                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 445                     {
 446                         /*
 447                          * decrement the TTL on ingress to the LSP
 448                          */
 449                         u32 checksum0;
 450                         u32 checksum1;
 451                         u32 checksum2;
 452                         u32 checksum3;
 453
 454                         checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
 455                         checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
 456                         checksum2 = ip2->checksum + clib_host_to_net_u16 (0x0100);
 457                         checksum3 = ip3->checksum + clib_host_to_net_u16 (0x0100);
 458
 459                         checksum0 += checksum0 >= 0xffff;
 460                         checksum1 += checksum1 >= 0xffff;
 461                         checksum2 += checksum2 >= 0xffff;
 462                         checksum3 += checksum3 >= 0xffff;
 463
 464                         ip0->checksum = checksum0;
 465                         ip1->checksum = checksum1;
 466                         ip2->checksum = checksum2;
 467                         ip3->checksum = checksum3;
 468
 469                         ip0->ttl -= 1;
 470                         ip1->ttl -= 1;
 471                         ip2->ttl -= 1;
 472                         ip3->ttl -= 1;
 473                     }
 474
 475                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 476                     {
 477                         ttl1 = ip1->ttl;
 478                         ttl0 = ip0->ttl;
 479                         ttl3 = ip3->ttl;
 480                         ttl2 = ip2->ttl;
 481                         /* by default copy the 3 most significant bits */
 482                         exp0 = ip_dscp_to_mpls_exp(ip0->tos);
 483                         exp1 = ip_dscp_to_mpls_exp(ip1->tos);
 484                         exp2 = ip_dscp_to_mpls_exp(ip2->tos);
 485                         exp3 = ip_dscp_to_mpls_exp(ip3->tos);
 486                     }
 487                 }
 488                 else if (DPO_PROTO_IP6 == dproto)
 489                 {
 490                     /*
 491                      * decrement the TTL on ingress to the LSP
 492                      */
 493                     ip6_header_t * ip0 = vlib_buffer_get_current(b0);
 494                     ip6_header_t * ip1 = vlib_buffer_get_current(b1);
 495                     ip6_header_t * ip2 = vlib_buffer_get_current(b2);
 496                     ip6_header_t * ip3 = vlib_buffer_get_current(b3);
 497
 498                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 499                     {
 500                         ip0->hop_limit -= 1;
 501                         ip1->hop_limit -= 1;
 502                         ip2->hop_limit -= 1;
 503                         ip3->hop_limit -= 1;
 504                     }
 505                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 506                     {
 507                         ttl0 = ip0->hop_limit;
 508                         ttl1 = ip1->hop_limit;
 509                         ttl2 = ip2->hop_limit;
 510                         ttl3 = ip3->hop_limit;
 511                         /* by default copy the 3 most significant bits */
 512                         exp0 = ip_dscp_to_mpls_exp(
 513                             ip6_traffic_class_network_order(ip0));
 514                         exp1 = ip_dscp_to_mpls_exp(
 515                             ip6_traffic_class_network_order(ip1));
 516                         exp2 = ip_dscp_to_mpls_exp(
 517                             ip6_traffic_class_network_order(ip2));
 518                         exp3 = ip_dscp_to_mpls_exp(
 519                             ip6_traffic_class_network_order(ip3));
 520                     }
 521                 }
 522                 else
 523                 {
 524                     /*
 525                      * nothing to change in the ethernet header
 526                      */
 527                     ttl0 = ttl1 = ttl2 = ttl3 = MPLS_LABEL_DEFAULT_TTL;
 528                     exp0 = exp1 = exp2 = exp3 = MPLS_LABEL_DEFAULT_EXP;
 529                 }
 530                 /*
 531                  * These are the non-MPLS payload imposition cases.
 532                  * Based on the LSP mode either, for uniform, copy down the TTL
 533                  * and EXP from the payload or, for pipe mode, slap on the value
 534                  * requested from config
 535                  */
 536                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 537                 {
 538                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 539                     hdr1 = mpls_label_paint_w_ttl_exp(b1, mld1, ttl1, exp1);
 540                     hdr2 = mpls_label_paint_w_ttl_exp(b2, mld2, ttl2, exp2);
 541                     hdr3 = mpls_label_paint_w_ttl_exp(b3, mld3, ttl3, exp3);
 542                 }
 543                 else
 544                 {
 545                     hdr0 = mpls_label_paint(b0, mld0);
 546                     hdr1 = mpls_label_paint(b1, mld1);
 547                     hdr2 = mpls_label_paint(b2, mld2);
 548                     hdr3 = mpls_label_paint(b3, mld3);
 549                 }
 550             }
 551             else
 552             {
 553                 /*
 554                  * else, the packet to be encapped is an MPLS packet
 555                  * there are two cases to consider:
 556                  *  1 - this is an MPLS label swap at an LSP midpoint.
 557                  *      recognisable because mpls.first = 1. In this case the
 558                  *      TTL must be set to the current value -1.
 559                  *  2 - The MPLS packet is recursing (or being injected into)
 560                  *      this LSP, in which case the pipe/uniform rules apply
 561                  *
 562                  */
 563                 if (PREDICT_TRUE(vnet_buffer(b0)->mpls.first))
 564                 {
 565                     /*
 566                      * The first label to be imposed on the packet. this is a
 567                      * label swap.in which case we stashed the TTL and EXP bits
 568                      * in the packet in the lookup node
 569                      */
 570                     ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
 571
 572                     ttl0 = vnet_buffer(b0)->mpls.ttl - 1;
 573                     exp0 = vnet_buffer(b0)->mpls.exp;
 574                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 575                 }
 576                 else
 577                 {
 578                     /*
 579                      * not the first label. implying we are recusring down a
 580                      * chain of output labels. Each layer is considered a new
 581                      * LSP - hence the TTL/EXP are pipe/uniform handled
 582                      */
 583                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 584                     {
 585                         hdr0 = vlib_buffer_get_current(b0);
 586                         ttl0 = ((u8*)hdr0)[3];
 587                         exp0 = ((u8*)hdr0)[2] & 0xe;
 588                         hdr0 = mpls_label_paint_w_ttl_mpls_exp(b0, mld0, ttl0, exp0);
 589                     }
 590                     else
 591                     {
 592                         hdr0 = mpls_label_paint(b0, mld0);
 593                     }
 594                 }
 595                 if (PREDICT_TRUE(vnet_buffer(b1)->mpls.first))
 596                 {
 597                     ASSERT(0 != vnet_buffer (b1)->mpls.ttl);
 598
 599                     ttl1 = vnet_buffer(b1)->mpls.ttl - 1;
 600                     exp1 = vnet_buffer(b1)->mpls.exp;
 601                     hdr1 = mpls_label_paint_w_ttl_exp(b1, mld1, ttl1, exp1);
 602                 }
 603                 else
 604                 {
 605                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 606                     {
 607                         hdr1 = vlib_buffer_get_current(b1);
 608                         ttl1 = ((u8*)hdr1)[3];
 609                         exp1 = ((u8*)hdr1)[2] & 0xe;
 610                         hdr1 = mpls_label_paint_w_ttl_mpls_exp(b1, mld1, ttl1, exp1);
 611                     }
 612                     else
 613                     {
 614                         hdr1 = mpls_label_paint(b1, mld1);
 615                     }
 616                 }
 617                 if (PREDICT_TRUE(vnet_buffer(b2)->mpls.first))
 618                 {
 619                     ASSERT(0 != vnet_buffer (b2)->mpls.ttl);
 620
 621                     ttl2 = vnet_buffer(b2)->mpls.ttl - 1;
 622                     exp2 = vnet_buffer(b2)->mpls.exp;
 623                     hdr2 = mpls_label_paint_w_ttl_exp(b2, mld2, ttl2, exp2);
 624                 }
 625                 else
 626                 {
 627                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 628                     {
 629                         hdr2 = vlib_buffer_get_current(b2);
 630                         ttl2 = ((u8*)hdr2)[3];
 631                         exp2 = ((u8*)hdr2)[2] & 0xe;
 632                         hdr2 = mpls_label_paint_w_ttl_mpls_exp(b2, mld2, ttl2, exp2);
 633                     }
 634                     else
 635                     {
 636                         hdr2 = mpls_label_paint(b2, mld2);
 637                     }
 638                 }
 639                 if (PREDICT_TRUE(vnet_buffer(b3)->mpls.first))
 640                 {
 641                     ASSERT(0 != vnet_buffer (b3)->mpls.ttl);
 642
 643                     ttl3 = vnet_buffer(b3)->mpls.ttl - 1;
 644                     exp3 = vnet_buffer(b3)->mpls.exp;
 645                     hdr3 = mpls_label_paint_w_ttl_exp(b3, mld3, ttl3, exp3);
 646                 }
 647                 else
 648                 {
 649                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 650                     {
 651                         hdr3 = vlib_buffer_get_current(b3);
 652                         ttl3 = ((u8*)hdr3)[3];
 653                         exp3 = ((u8*)hdr3)[2] & 0xe;
 654                         hdr3 = mpls_label_paint_w_ttl_mpls_exp(b3, mld3, ttl3, exp3);
 655                     }
 656                     else
 657                     {
 658                         hdr3 = mpls_label_paint(b3, mld3);
 659                     }
 660                 }
 661
 662                 vnet_buffer(b0)->mpls.first = 0;
 663                 vnet_buffer(b1)->mpls.first = 0;
 664                 vnet_buffer(b2)->mpls.first = 0;
 665                 vnet_buffer(b3)->mpls.first = 0;
 666             }
 667
 668             next0 = mld0->mld_dpo.dpoi_next_node;
 669             next1 = mld1->mld_dpo.dpoi_next_node;
 670             next2 = mld2->mld_dpo.dpoi_next_node;
 671             next3 = mld3->mld_dpo.dpoi_next_node;
 672
 673             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
 674             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = mld1->mld_dpo.dpoi_index;
 675             vnet_buffer(b2)->ip.adj_index[VLIB_TX] = mld2->mld_dpo.dpoi_index;
 676             vnet_buffer(b3)->ip.adj_index[VLIB_TX] = mld3->mld_dpo.dpoi_index;
 677
 678             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 679             {
 680                 mpls_label_imposition_trace_t *tr =
 681                     vlib_add_trace (vm, node, b0, sizeof (*tr));
 682                 tr->hdr = *hdr0;
 683                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 684                 {
 685                     tr->ttl = ttl0;
 686                     tr->exp = exp0;
 687                 }
 688                 else
 689                 {
 690                     tr->ttl = tr->exp = 0;
 691                 }
 692             }
 693             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 694             {
 695                 mpls_label_imposition_trace_t *tr =
 696                     vlib_add_trace (vm, node, b1, sizeof (*tr));
 697                 tr->hdr = *hdr1;
 698                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 699                 {
 700                     tr->ttl = ttl1;
 701                     tr->exp = exp1;
 702                 }
 703                 else
 704                 {
 705                     tr->ttl = tr->exp = 0;
 706                 }
 707             }
 708             if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
 709             {
 710                 mpls_label_imposition_trace_t *tr =
 711                     vlib_add_trace (vm, node, b2, sizeof (*tr));
 712                 tr->hdr = *hdr2;
 713                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 714                 {
 715                     tr->ttl = ttl2;
 716                     tr->exp = exp2;
 717                 }
 718                 else
 719                 {
 720                     tr->ttl = tr->exp = 0;
 721                 }
 722             }
 723             if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
 724             {
 725                 mpls_label_imposition_trace_t *tr =
 726                     vlib_add_trace (vm, node, b3, sizeof (*tr));
 727                 tr->hdr = *hdr3;
 728                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 729                 {
 730                     tr->ttl = ttl3;
 731                     tr->exp = exp3;
 732                 }
 733                 else
 734                 {
 735                     tr->ttl = tr->exp = 0;
 736                 }
 737             }
 738
 739             vlib_validate_buffer_enqueue_x4(vm, node, next_index, to_next,
 740                                             n_left_to_next,
 741                                             bi0, bi1, bi2, bi3,
 742                                             next0, next1, next2, next3);
 743         }
 744
 745         while (n_left_from > 0 && n_left_to_next > 0)
 746         {
 747             mpls_unicast_header_t *hdr0;
 748             mpls_label_dpo_t *mld0;
 749             vlib_buffer_t * b0;
 750             u32 bi0, mldi0;
 751             u8 ttl0, exp0;
 752             u32 next0;
 753
 754             bi0 = from[0];
 755             to_next[0] = bi0;
 756             from += 1;
 757             to_next += 1;
 758             n_left_from -= 1;
 759             n_left_to_next -= 1;
 760
 761             b0 = vlib_get_buffer (vm, bi0);
 762
 763             /* dst lookup was done by ip4 lookup */
 764             mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 765             mld0 = mpls_label_dpo_get(mldi0);
 766
 767             if (DPO_PROTO_MPLS != dproto)
 768             {
 769                 if (DPO_PROTO_IP4 == dproto)
 770                 {
 771                     /*
 772                      * decrement the TTL on ingress to the LSP
 773                      */
 774                     ip4_header_t * ip0 = vlib_buffer_get_current(b0);
 775                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 776                     {
 777                         u32 checksum0;
 778
 779                         checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
 780                         checksum0 += checksum0 >= 0xffff;
 781
 782                         ip0->checksum = checksum0;
 783                         ip0->ttl -= 1;
 784                     }
 785                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 786                     {
 787                         ttl0 = ip0->ttl;
 788                         exp0 = ip_dscp_to_mpls_exp(ip0->tos);
 789                     }
 790                 }
 791                 else if (DPO_PROTO_IP6 == dproto)
 792                 {
 793                     /*
 794                      * decrement the TTL on ingress to the LSP
 795                      */
 796                     ip6_header_t * ip0 = vlib_buffer_get_current(b0);
 797
 798                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 799                     {
 800                         ip0->hop_limit -= 1;
 801                     }
 802                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 803                     {
 804                         ttl0 = ip0->hop_limit;
 805                         exp0 = ip_dscp_to_mpls_exp(
 806                             ip6_traffic_class_network_order(ip0));
 807                     }
 808                 }
 809                 else
 810                 {
 811                     /*
 812                      * nothing to change in the ethernet header
 813                      */
 814                     ttl0 = MPLS_LABEL_DEFAULT_TTL;
 815                     exp0 = MPLS_LABEL_DEFAULT_EXP;
 816                 }
 817
 818                 /*
 819                  * These are the non-MPLS payload imposition cases.
 820                  * Based on the LSP mode either, for uniform, copy down the TTL
 821                  * from the payload or, for pipe mode, slap on the value
 822                  * requested from config
 823                  */
 824                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 825                 {
 826                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 827                 }
 828                 else
 829                 {
 830                     hdr0 = mpls_label_paint(b0, mld0);
 831                 }
 832             }
 833             else
 834             {
 835                 if (PREDICT_TRUE(vnet_buffer(b0)->mpls.first))
 836                 {
 837                     ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
 838
 839                     ttl0 = vnet_buffer(b0)->mpls.ttl - 1;
 840                     exp0 = vnet_buffer(b0)->mpls.exp;
 841                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 842                 }
 843                 else
 844                 {
 845                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 846                     {
 847                         hdr0 = vlib_buffer_get_current(b0);
 848                         ttl0 = ((u8*)hdr0)[3];
 849                         exp0 = ((u8*)hdr0)[2] & 0xe;
 850                         hdr0 = mpls_label_paint_w_ttl_mpls_exp(b0, mld0, ttl0, exp0);
 851                     }
 852                     else
 853                     {
 854                         hdr0 = mpls_label_paint(b0, mld0);
 855                     }
 856                 }
 857
 858                 vnet_buffer(b0)->mpls.first = 0;
 859             }
 860
 861             next0 = mld0->mld_dpo.dpoi_next_node;
 862             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
 863
 864             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 865             {
 866                 mpls_label_imposition_trace_t *tr =
 867                     vlib_add_trace (vm, node, b0, sizeof (*tr));
 868                 tr->hdr = *hdr0;
 869                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 870                 {
 871                     tr->ttl = ttl0;
 872                     tr->exp = exp0;
 873                 }
 874                 else
 875                 {
 876                     tr->ttl = tr->exp = 0;
 877                 }
 878            }
 879
 880             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 881                                             n_left_to_next, bi0, next0);
 882         }
 883         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 884     }
 885     return from_frame->n_vectors;
 886 }
 887
 888 static u8 *
 889 format_mpls_label_imposition_trace (u8 * s, va_list * args)
 890 {
 891     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 892     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 893     mpls_label_imposition_trace_t * t;
 894     mpls_unicast_header_t hdr;
 895     u32 indent;
 896
 897     t = va_arg (*args, mpls_label_imposition_trace_t *);
 898     indent = format_get_indent (s);
 899     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
 900
 901     s = format (s, "%Umpls-header:%U",
 902                 format_white_space, indent,
 903                 format_mpls_header, hdr);
 904     return (s);
 905 }
 906
 907 VLIB_NODE_FN (mpls_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 908                                  vlib_node_runtime_t * node,
 909                                  vlib_frame_t * frame)
 910 {
 911     return (mpls_label_imposition_inline(vm, node, frame,
 912                                          DPO_PROTO_MPLS,
 913                                          MPLS_LABEL_DPO_FLAG_NONE));
 914 }
 915
 916 VLIB_REGISTER_NODE (mpls_mpls_label_imposition_pipe_node) = {
 917     .name = "mpls-label-imposition-pipe",
 918     .vector_size = sizeof (u32),
 919
 920     .format_trace = format_mpls_label_imposition_trace,
 921     .n_next_nodes = 1,
 922     .next_nodes = {
 923         [0] = "mpls-drop",
 924     }
 925 };
 926
 927 VLIB_NODE_FN (ip4_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 928                                 vlib_node_runtime_t * node,
 929                                 vlib_frame_t * frame)
 930 {
 931     return (mpls_label_imposition_inline(vm, node, frame,
 932                                          DPO_PROTO_IP4,
 933                                          MPLS_LABEL_DPO_FLAG_NONE));
 934 }
 935
 936 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_pipe_node) = {
 937     .name = "ip4-mpls-label-imposition-pipe",
 938     .vector_size = sizeof (u32),
 939
 940     .format_trace = format_mpls_label_imposition_trace,
 941     .n_next_nodes = 1,
 942     .next_nodes = {
 943         [0] = "ip4-drop",
 944     }
 945 };
 946
 947 VLIB_NODE_FN (ip6_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 948                                 vlib_node_runtime_t * node,
 949                                 vlib_frame_t * frame)
 950 {
 951     return (mpls_label_imposition_inline(vm, node, frame,
 952                                          DPO_PROTO_IP6,
 953                                          MPLS_LABEL_DPO_FLAG_NONE));
 954 }
 955
 956 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_pipe_node) = {
 957     .name = "ip6-mpls-label-imposition-pipe",
 958     .vector_size = sizeof (u32),
 959
 960     .format_trace = format_mpls_label_imposition_trace,
 961     .n_next_nodes = 1,
 962     .next_nodes = {
 963         [0] = "ip6-drop",
 964     }
 965 };
 966
 967 VLIB_NODE_FN (ethernet_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 968                                      vlib_node_runtime_t * node,
 969                                      vlib_frame_t * frame)
 970 {
 971     return (mpls_label_imposition_inline(vm, node, frame,
 972                                          DPO_PROTO_ETHERNET,
 973                                          MPLS_LABEL_DPO_FLAG_NONE));
 974 }
 975
 976 VLIB_REGISTER_NODE (ethernet_mpls_label_imposition_pipe_node) = {
 977     .name = "ethernet-mpls-label-imposition-pipe",
 978     .vector_size = sizeof (u32),
 979
 980     .format_trace = format_mpls_label_imposition_trace,
 981     .n_next_nodes = 1,
 982     .next_nodes = {
 983         [0] = "error-drop",
 984     }
 985 };
 986
 987 VLIB_NODE_FN (mpls_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
 988                                     vlib_node_runtime_t * node,
 989                                     vlib_frame_t * frame)
 990 {
 991     return (mpls_label_imposition_inline(vm, node, frame,
 992                                          DPO_PROTO_MPLS,
 993                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
 994 }
 995
 996 VLIB_REGISTER_NODE (mpls_mpls_label_imposition_uniform_node) = {
 997     .name = "mpls-label-imposition-uniform",
 998     .vector_size = sizeof (u32),
 999
1000     .format_trace = format_mpls_label_imposition_trace,
1001     .n_next_nodes = 1,
1002     .next_nodes = {
1003         [0] = "mpls-drop",
1004     }
1005 };
1006
1007 VLIB_NODE_FN (ip4_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
1008                                    vlib_node_runtime_t * node,
1009                                    vlib_frame_t * frame)
1010 {
1011     return (mpls_label_imposition_inline(vm, node, frame,
1012                                          DPO_PROTO_IP4,
1013                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
1014 }
1015
1016 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_uniform_node) = {
1017     .name = "ip4-mpls-label-imposition-uniform",
1018     .vector_size = sizeof (u32),
1019
1020     .format_trace = format_mpls_label_imposition_trace,
1021     .n_next_nodes = 1,
1022     .next_nodes = {
1023         [0] = "ip4-drop",
1024     }
1025 };
1026
1027 VLIB_NODE_FN (ip6_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
1028                                    vlib_node_runtime_t * node,
1029                                    vlib_frame_t * frame)
1030 {
1031     return (mpls_label_imposition_inline(vm, node, frame,
1032                                          DPO_PROTO_IP6,
1033                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
1034 }
1035
1036 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_uniform_node) = {
1037     .name = "ip6-mpls-label-imposition-uniform",
1038     .vector_size = sizeof (u32),
1039
1040     .format_trace = format_mpls_label_imposition_trace,
1041     .n_next_nodes = 1,
1042     .next_nodes = {
1043         [0] = "ip6-drop",
1044     }
1045 };
1046
1047 VLIB_NODE_FN (ethernet_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
1048                                         vlib_node_runtime_t * node,
1049                                         vlib_frame_t * frame)
1050 {
1051     return (mpls_label_imposition_inline(vm, node, frame,
1052                                          DPO_PROTO_ETHERNET,
1053                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
1054 }
1055
1056 VLIB_REGISTER_NODE (ethernet_mpls_label_imposition_uniform_node) = {
1057     .name = "ethernet-mpls-label-imposition-uniform",
1058     .vector_size = sizeof (u32),
1059
1060     .format_trace = format_mpls_label_imposition_trace,
1061     .n_next_nodes = 1,
1062     .next_nodes = {
1063         [0] = "error-drop",
1064     }
1065 };
1066
1067
1068 VLIB_NODE_FN (ip4_mpls_label_imposition_pipe_no_ip_ttl_decr_node) (vlib_main_t * vm,
1069                                                vlib_node_runtime_t * node,
1070                                                vlib_frame_t * frame)
1071 {
1072     return (mpls_label_imposition_inline(vm, node, frame,
1073                                          DPO_PROTO_IP4,
1074                                          MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR));
1075 }
1076
1077 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_pipe_no_ip_ttl_decr_node) = {
1078     .name = "ip4-mpls-label-imposition-pipe-no-ip-ttl-decr",
1079     .vector_size = sizeof (u32),
1080
1081     .format_trace = format_mpls_label_imposition_trace,
1082     .n_next_nodes = 1,
1083     .next_nodes = {
1084         [0] = "ip4-drop",
1085     }
1086 };
1087
1088 VLIB_NODE_FN (ip6_mpls_label_imposition_pipe_no_ip_ttl_decr_node) (vlib_main_t * vm,
1089                                                vlib_node_runtime_t * node,
1090                                                vlib_frame_t * frame)
1091 {
1092     return (mpls_label_imposition_inline(vm, node, frame,
1093                                          DPO_PROTO_IP6,
1094                                          MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR));
1095 }
1096
1097 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_pipe_no_ip_ttl_decr_node) = {
1098     .name = "ip6-mpls-label-imposition-pipe-no-ip-ttl-decr",
1099     .vector_size = sizeof (u32),
1100
1101     .format_trace = format_mpls_label_imposition_trace,
1102     .n_next_nodes = 1,
1103     .next_nodes = {
1104         [0] = "ip6-drop",
1105     }
1106 };
1107
1108 VLIB_NODE_FN (ip4_mpls_label_imposition_uniform_no_ip_ttl_decr_node) (vlib_main_t * vm,
1109                                                   vlib_node_runtime_t * node,
1110                                                   vlib_frame_t * frame)
1111 {
1112     return (mpls_label_imposition_inline(vm, node, frame,
1113                                          DPO_PROTO_IP4,
1114                                          (MPLS_LABEL_DPO_FLAG_UNIFORM_MODE |
1115                                           MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR)));
1116 }
1117
1118 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_uniform_no_ip_ttl_decr_node) = {
1119     .name = "ip4-mpls-label-imposition-uniform-no-ip-ttl-decr",
1120     .vector_size = sizeof (u32),
1121
1122     .format_trace = format_mpls_label_imposition_trace,
1123     .n_next_nodes = 1,
1124     .next_nodes = {
1125         [0] = "ip4-drop",
1126     }
1127 };
1128
1129 VLIB_NODE_FN (ip6_mpls_label_imposition_uniform_no_ip_ttl_decr_node) (vlib_main_t * vm,
1130                                                   vlib_node_runtime_t * node,
1131                                                   vlib_frame_t * frame)
1132 {
1133     return (mpls_label_imposition_inline(vm, node, frame,
1134                                          DPO_PROTO_IP6,
1135                                          (MPLS_LABEL_DPO_FLAG_UNIFORM_MODE |
1136                                           MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR)));
1137 }
1138
1139 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_uniform_no_ip_ttl_decr_node) = {
1140     .name = "ip6-mpls-label-imposition-uniform-no-ip-ttl-decr",
1141     .vector_size = sizeof (u32),
1142
1143     .format_trace = format_mpls_label_imposition_trace,
1144     .n_next_nodes = 1,
1145     .next_nodes = {
1146         [0] = "ip6-drop",
1147     }
1148 };
1149
1150
1151 #ifndef CLIB_MARCH_VARIANT
1152 static void
1153 mpls_label_dpo_mem_show (void)
1154 {
1155     fib_show_memory_usage("MPLS label",
1156                           pool_elts(mpls_label_dpo_pool),
1157                           pool_len(mpls_label_dpo_pool),
1158                           sizeof(mpls_label_dpo_t));
1159 }
1160
1161 /**
1162  * Interpose a label DPO. used in the FIB unit tests
1163  */
1164 static void
1165 mpls_label_interpose (const dpo_id_t *original,
1166                       const dpo_id_t *parent,
1167                       dpo_id_t *clone)
1168 {
1169     mpls_label_dpo_t *mld, *mld_clone;
1170
1171     mld_clone = mpls_label_dpo_alloc();
1172     mld = mpls_label_dpo_get(original->dpoi_index);
1173
1174     mld_clone->mld_locks = 0;
1175     clib_memcpy_fast(&mld_clone->mld_hdr,
1176                 &mld->mld_hdr,
1177                 sizeof(mld_clone->mld_hdr));
1178     mld_clone->mld_payload_proto = mld->mld_payload_proto;
1179     mld_clone->mld_n_labels = mld->mld_n_labels;
1180     mld_clone->mld_n_hdr_bytes = mld->mld_n_hdr_bytes;
1181
1182     dpo_stack(mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NONE],
1183               mld_clone->mld_payload_proto,
1184               &mld_clone->mld_dpo,
1185               parent);
1186
1187     dpo_set(clone,
1188             mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NONE],
1189             mld_clone->mld_payload_proto,
1190             mpls_label_dpo_get_index(mld_clone));
1191 }
1192
1193 const static dpo_vft_t mld_vft = {
1194     .dv_lock = mpls_label_dpo_lock,
1195     .dv_unlock = mpls_label_dpo_unlock,
1196     .dv_format = format_mpls_label_dpo,
1197     .dv_mem_show = mpls_label_dpo_mem_show,
1198     .dv_mk_interpose = mpls_label_interpose,
1199 };
1200
1201 const static char* const mpls_label_imp_pipe_ip4_nodes[] =
1202 {
1203     "ip4-mpls-label-imposition-pipe",
1204     NULL,
1205 };
1206 const static char* const mpls_label_imp_pipe_ip6_nodes[] =
1207 {
1208     "ip6-mpls-label-imposition-pipe",
1209     NULL,
1210 };
1211 const static char* const mpls_label_imp_pipe_mpls_nodes[] =
1212 {
1213     "mpls-label-imposition-pipe",
1214     NULL,
1215 };
1216 const static char* const mpls_label_imp_pipe_ethernet_nodes[] =
1217 {
1218     "ethernet-mpls-label-imposition-pipe",
1219     NULL,
1220 };
1221
1222 const static char* const * const mpls_label_imp_pipe_nodes[DPO_PROTO_NUM] =
1223 {
1224     [DPO_PROTO_IP4]  = mpls_label_imp_pipe_ip4_nodes,
1225     [DPO_PROTO_IP6]  = mpls_label_imp_pipe_ip6_nodes,
1226     [DPO_PROTO_MPLS] = mpls_label_imp_pipe_mpls_nodes,
1227     [DPO_PROTO_ETHERNET] = mpls_label_imp_pipe_ethernet_nodes,
1228 };
1229
1230 const static char* const mpls_label_imp_uniform_ip4_nodes[] =
1231 {
1232     "ip4-mpls-label-imposition-uniform",
1233     NULL,
1234 };
1235 const static char* const mpls_label_imp_uniform_ip6_nodes[] =
1236 {
1237     "ip6-mpls-label-imposition-uniform",
1238     NULL,
1239 };
1240 const static char* const mpls_label_imp_uniform_mpls_nodes[] =
1241 {
1242     "mpls-label-imposition-uniform",
1243     NULL,
1244 };
1245 const static char* const mpls_label_imp_uniform_ethernet_nodes[] =
1246 {
1247     "ethernet-mpls-label-imposition-uniform",
1248     NULL,
1249 };
1250
1251 const static char* const * const mpls_label_imp_uniform_nodes[DPO_PROTO_NUM] =
1252 {
1253     [DPO_PROTO_IP4]  = mpls_label_imp_uniform_ip4_nodes,
1254     [DPO_PROTO_IP6]  = mpls_label_imp_uniform_ip6_nodes,
1255     [DPO_PROTO_MPLS] = mpls_label_imp_uniform_mpls_nodes,
1256     [DPO_PROTO_ETHERNET] = mpls_label_imp_uniform_ethernet_nodes,
1257 };
1258
1259 const static char* const mpls_label_imp_pipe_no_ip_tll_decr_ip4_nodes[] =
1260 {
1261     "ip4-mpls-label-imposition-pipe-no-ip-ttl-decr",
1262     NULL,
1263 };
1264 const static char* const mpls_label_imp_pipe_no_ip_tll_decr_ip6_nodes[] =
1265 {
1266     "ip6-mpls-label-imposition-pipe-no-ip-ttl-decr",
1267     NULL,
1268 };
1269
1270 const static char* const * const mpls_label_imp_pipe_no_ip_tll_decr_nodes[DPO_PROTO_NUM] =
1271 {
1272     [DPO_PROTO_IP4]  = mpls_label_imp_pipe_no_ip_tll_decr_ip4_nodes,
1273     [DPO_PROTO_IP6]  = mpls_label_imp_pipe_no_ip_tll_decr_ip6_nodes,
1274 };
1275
1276 const static char* const mpls_label_imp_uniform_no_ip_tll_decr_ip4_nodes[] =
1277 {
1278     "ip4-mpls-label-imposition-uniform-no-ip-ttl-decr",
1279     NULL,
1280 };
1281 const static char* const mpls_label_imp_uniform_no_ip_tll_decr_ip6_nodes[] =
1282 {
1283     "ip6-mpls-label-imposition-uniform-no-ip-ttl-decr",
1284     NULL,
1285 };
1286
1287 const static char* const * const mpls_label_imp_uniform_no_ip_tll_decr_nodes[DPO_PROTO_NUM] =
1288 {
1289     [DPO_PROTO_IP4]  = mpls_label_imp_uniform_no_ip_tll_decr_ip4_nodes,
1290     [DPO_PROTO_IP6]  = mpls_label_imp_uniform_no_ip_tll_decr_ip6_nodes,
1291 };
1292
1293 void
1294 mpls_label_dpo_module_init (void)
1295 {
1296     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NONE] =
1297         dpo_register_new_type(&mld_vft,
1298                               mpls_label_imp_pipe_nodes);
1299     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR] =
1300         dpo_register_new_type(&mld_vft,
1301                               mpls_label_imp_pipe_no_ip_tll_decr_nodes);
1302     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_UNIFORM_MODE] =
1303         dpo_register_new_type(&mld_vft,
1304                               mpls_label_imp_uniform_nodes);
1305     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_UNIFORM_MODE |
1306                          MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR] =
1307         dpo_register_new_type(&mld_vft,
1308                               mpls_label_imp_uniform_no_ip_tll_decr_nodes);
1309 }
1310
1311 dpo_type_t
1312 mpls_label_dpo_get_type (mpls_label_dpo_flags_t flags)
1313 {
1314     return (mpls_label_dpo_types[flags]);
1315 }
1316 #endif /* CLIB_MARCH_VARIANT */