src/vnet/dpo/mpls_label_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/mpls_label_dpo.h>
  18 #include <vnet/mpls/mpls.h>
  19 #include <vnet/dpo/drop_dpo.h>
  20
  21 #ifndef CLIB_MARCH_VARIANT
  22 /*
  23  * pool of all MPLS Label DPOs
  24  */
  25 mpls_label_dpo_t *mpls_label_dpo_pool;
  26
  27 /**
  28  * Strings for the flags
  29  */
  30 const char* mpls_label_dpo_attr_names[] = MPLS_LABEL_DPO_ATTR_NAMES;
  31
  32 /**
  33  * registered DPO types for each of the label sub-types. And there's a
  34  * subtype for each of the flag combinations.
  35  */
  36 static dpo_type_t mpls_label_dpo_types[1 << MPLS_LABEL_DPO_ATTR_MAX];
  37
  38 static mpls_label_dpo_t *
  39 mpls_label_dpo_alloc (void)
  40 {
  41     mpls_label_dpo_t *mld;
  42     vlib_main_t *vm;
  43     u8 did_barrier_sync;
  44
  45     dpo_pool_barrier_sync (vm, mpls_label_dpo_pool, did_barrier_sync);
  46     pool_get_aligned(mpls_label_dpo_pool, mld, CLIB_CACHE_LINE_BYTES);
  47     dpo_pool_barrier_release (vm, did_barrier_sync);
  48
  49     clib_memset(mld, 0, sizeof(*mld));
  50
  51     dpo_reset(&mld->mld_dpo);
  52
  53     return (mld);
  54 }
  55
  56 static index_t
  57 mpls_label_dpo_get_index (mpls_label_dpo_t *mld)
  58 {
  59     return (mld - mpls_label_dpo_pool);
  60 }
  61
  62 void
  63 mpls_label_dpo_create (fib_mpls_label_t *label_stack,
  64                        mpls_eos_bit_t eos,
  65                        dpo_proto_t payload_proto,
  66                        mpls_label_dpo_flags_t flags,
  67                        const dpo_id_t *parent,
  68                        dpo_id_t *dpo)
  69 {
  70     mpls_label_dpo_t *mld;
  71     dpo_type_t dtype;
  72     u32 ii;
  73
  74     if ((DPO_PROTO_IP4 != payload_proto) &&
  75         (DPO_PROTO_IP6 != payload_proto))
  76     {
  77         /*
  78          * remove unsupported configuration
  79          */
  80         flags &= ~MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR;
  81     }
  82
  83     mld = mpls_label_dpo_alloc();
  84     mld->mld_flags = flags;
  85     dtype = mpls_label_dpo_types[flags];
  86
  87     if (MPLS_LABEL_DPO_MAX_N_LABELS < vec_len(label_stack))
  88     {
  89         clib_warning("Label stack size exceeded");
  90         dpo_stack(dtype,
  91                   mld->mld_payload_proto,
  92                   &mld->mld_dpo,
  93                   drop_dpo_get(DPO_PROTO_MPLS));
  94     }
  95     else
  96     {
  97         mld->mld_n_labels = vec_len(label_stack);
  98         mld->mld_n_hdr_bytes = mld->mld_n_labels * sizeof(mld->mld_hdr[0]);
  99         mld->mld_payload_proto = payload_proto;
 100
 101         /*
 102          * construct label rewrite headers for each value passed.
 103          * get the header in network byte order since we will paint it
 104          * on a packet in the data-plane
 105          */
 106         for (ii = 0; ii < mld->mld_n_labels-1; ii++)
 107         {
 108             vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl,
 109                                    label_stack[ii].fml_value);
 110             vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl,
 111                                  label_stack[ii].fml_exp);
 112             vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl,
 113                                MPLS_NON_EOS);
 114             if (0 != label_stack[ii].fml_ttl)
 115             {
 116                 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 117                                      label_stack[ii].fml_ttl);
 118             }
 119             else
 120             {
 121                 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 122                                      MPLS_LABEL_DEFAULT_TTL);
 123             }
 124             mld->mld_hdr[ii].label_exp_s_ttl =
 125                 clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
 126         }
 127
 128         /*
 129          * the inner most label
 130          */
 131         ii = mld->mld_n_labels-1;
 132
 133         vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl,
 134                                label_stack[ii].fml_value);
 135         vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl,
 136                              label_stack[ii].fml_exp);
 137         vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl, eos);
 138         if (0 != label_stack[ii].fml_ttl)
 139         {
 140             vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 141                                  label_stack[ii].fml_ttl);
 142         }
 143         else
 144         {
 145             vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 146                                  MPLS_LABEL_DEFAULT_TTL);
 147         }
 148         mld->mld_hdr[ii].label_exp_s_ttl =
 149             clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
 150
 151         /*
 152          * pipe/uniform mode is only supported for the bottom of stack label
 153          */
 154         if (FIB_MPLS_LSP_MODE_UNIFORM == label_stack[ii].fml_mode)
 155         {
 156             mld->mld_flags |= MPLS_LABEL_DPO_FLAG_UNIFORM_MODE;
 157         }
 158         else
 159         {
 160             mld->mld_flags &= ~MPLS_LABEL_DPO_FLAG_UNIFORM_MODE;
 161         }
 162         dtype = mpls_label_dpo_types[mld->mld_flags];
 163
 164         /*
 165          * stack this label object on its parent.
 166          */
 167         dpo_stack(dtype,
 168                   mld->mld_payload_proto,
 169                   &mld->mld_dpo,
 170                   parent);
 171     }
 172
 173     dpo_set(dpo,
 174             dtype,
 175             mld->mld_payload_proto,
 176             mpls_label_dpo_get_index(mld));
 177 }
 178
 179 u8*
 180 format_mpls_label_dpo_flags (u8 *s, va_list *args)
 181 {
 182     mpls_label_dpo_flags_t flags = va_arg (*args, int);
 183     mpls_label_dpo_attr_t attr;
 184
 185     FOR_EACH_MPLS_LABEL_DPO_ATTR(attr)
 186     {
 187         if ((1 << attr) & flags)
 188         {
 189             s = format(s, "%s,", mpls_label_dpo_attr_names[attr]);
 190         }
 191     }
 192
 193     return (s);
 194 }
 195
 196 u8*
 197 format_mpls_label_dpo (u8 *s, va_list *args)
 198 {
 199     index_t index = va_arg (*args, index_t);
 200     u32 indent = va_arg (*args, u32);
 201     mpls_unicast_header_t hdr;
 202     mpls_label_dpo_t *mld;
 203     u32 ii;
 204
 205     if (pool_is_free_index(mpls_label_dpo_pool, index))
 206     {
 207         /*
 208          * the packet trace can be printed after the DPO has been deleted
 209          */
 210         return (format(s, "mpls-label[???,%d]:", index));
 211     }
 212
 213     mld = mpls_label_dpo_get(index);
 214     s = format(s, "mpls-label[%U@%d]:",
 215                format_mpls_label_dpo_flags,
 216                (int) mld->mld_flags, index);
 217
 218     for (ii = 0; ii < mld->mld_n_labels; ii++)
 219     {
 220         hdr.label_exp_s_ttl =
 221             clib_net_to_host_u32(mld->mld_hdr[ii].label_exp_s_ttl);
 222         s = format(s, "%U", format_mpls_header, hdr);
 223     }
 224
 225     s = format(s, "\n%U", format_white_space, indent);
 226     s = format(s, "%U", format_dpo_id, &mld->mld_dpo, indent+2);
 227
 228     return (s);
 229 }
 230
 231 static void
 232 mpls_label_dpo_lock (dpo_id_t *dpo)
 233 {
 234     mpls_label_dpo_t *mld;
 235
 236     mld = mpls_label_dpo_get(dpo->dpoi_index);
 237
 238     mld->mld_locks++;
 239 }
 240
 241 static void
 242 mpls_label_dpo_unlock (dpo_id_t *dpo)
 243 {
 244     mpls_label_dpo_t *mld;
 245
 246     mld = mpls_label_dpo_get(dpo->dpoi_index);
 247
 248     mld->mld_locks--;
 249
 250     if (0 == mld->mld_locks)
 251     {
 252         dpo_reset(&mld->mld_dpo);
 253         pool_put(mpls_label_dpo_pool, mld);
 254     }
 255 }
 256 #endif /* CLIB_MARCH_VARIANT */
 257
 258 /**
 259  * @brief A struct to hold tracing information for the MPLS label imposition
 260  * node.
 261  */
 262 typedef struct mpls_label_imposition_trace_t_
 263 {
 264     /**
 265      * The MPLS header imposed
 266      */
 267     mpls_unicast_header_t hdr;
 268
 269     /**
 270      * TTL imposed - only valid for uniform LSPs
 271      */
 272     u8 ttl;
 273
 274     /**
 275      * TTL imposed - only valid for uniform LSPs
 276      */
 277     u8 exp;
 278 } mpls_label_imposition_trace_t;
 279
 280 always_inline mpls_unicast_header_t *
 281 mpls_label_paint (vlib_buffer_t * b0,
 282                   mpls_label_dpo_t *mld0)
 283 {
 284     mpls_unicast_header_t *hdr0;
 285
 286     vlib_buffer_advance(b0, -(mld0->mld_n_hdr_bytes));
 287
 288     hdr0 = vlib_buffer_get_current(b0);
 289
 290     if (1 == mld0->mld_n_labels)
 291     {
 292         /* optimise for the common case of one label */
 293         *hdr0 = mld0->mld_hdr[0];
 294     }
 295     else
 296     {
 297         clib_memcpy_fast(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes);
 298         hdr0 = hdr0 + (mld0->mld_n_labels - 1);
 299     }
 300
 301     return (hdr0);
 302 }
 303
 304 /**
 305  * Paint on an MPLS label and fixup the TTL
 306  */
 307 always_inline mpls_unicast_header_t *
 308 mpls_label_paint_w_ttl (vlib_buffer_t * b0,
 309                         mpls_label_dpo_t *mld0,
 310                         u8 ttl0)
 311 {
 312     mpls_unicast_header_t *hdr0;
 313
 314     hdr0 = mpls_label_paint(b0, mld0);
 315
 316     /* fixup the TTL for the inner most label */
 317     ((char*)hdr0)[3] = ttl0;
 318
 319     return (hdr0);
 320 }
 321
 322 /**
 323  * Paint on an MPLS label and fixup the TTL and EXP bits.
 324  */
 325 always_inline mpls_unicast_header_t *
 326 mpls_label_paint_w_ttl_exp (vlib_buffer_t * b0,
 327                             mpls_label_dpo_t *mld0,
 328                             u8 ttl0,
 329                             u8 exp0)
 330 {
 331     mpls_unicast_header_t *hdr0;
 332
 333     hdr0 = mpls_label_paint_w_ttl(b0, mld0, ttl0);
 334
 335     /* fixup the EXP for the inner most label */
 336     ((char*)hdr0)[2] |= (exp0 << 1);
 337
 338     return (hdr0);
 339 }
 340
 341 /**
 342  * Paint on an MPLS label and fixup the TTL and EXP bits
 343  * When the EXP bits are *already* bit shift to the correct place in
 344  * in the 2nd byte (i.e. they were read from another label)
 345  */
 346 always_inline mpls_unicast_header_t *
 347 mpls_label_paint_w_ttl_mpls_exp (vlib_buffer_t * b0,
 348                                  mpls_label_dpo_t *mld0,
 349                                  u8 ttl0,
 350                                  u8 exp0)
 351 {
 352     mpls_unicast_header_t *hdr0;
 353
 354     hdr0 = mpls_label_paint_w_ttl(b0, mld0, ttl0);
 355
 356     /* fixup the EXP for the inner most label */
 357     ((char*)hdr0)[2] |= exp0;
 358
 359     return (hdr0);
 360 }
 361
 362 always_inline uword
 363 mpls_label_imposition_inline (vlib_main_t * vm,
 364                               vlib_node_runtime_t * node,
 365                               vlib_frame_t * from_frame,
 366                               const dpo_proto_t dproto,
 367                               const mpls_label_dpo_flags_t flags)
 368 {
 369     u32 n_left_from, next_index, * from, * to_next;
 370
 371     from = vlib_frame_vector_args (from_frame);
 372     n_left_from = from_frame->n_vectors;
 373
 374     next_index = node->cached_next_index;
 375
 376     while (n_left_from > 0)
 377     {
 378         u32 n_left_to_next;
 379
 380         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 381
 382         while (n_left_from >= 8 && n_left_to_next >= 4)
 383         {
 384             u32 bi0, mldi0, bi1, mldi1, bi2, mldi2, bi3, mldi3;
 385             mpls_unicast_header_t *hdr0, *hdr1, *hdr2, *hdr3;
 386             mpls_label_dpo_t *mld0, *mld1, *mld2, *mld3;
 387             vlib_buffer_t * b0, *b1, * b2, *b3;
 388             u32 next0, next1, next2, next3;
 389             u8 ttl0, ttl1, ttl2, ttl3;
 390             u8 exp0, exp1, exp2, exp3;
 391
 392             bi0 = to_next[0] = from[0];
 393             bi1 = to_next[1] = from[1];
 394             bi2 = to_next[2] = from[2];
 395             bi3 = to_next[3] = from[3];
 396
 397             /* Prefetch next iteration. */
 398             {
 399                 vlib_buffer_t * p2, * p3, *p4, *p5;
 400
 401                 p2 = vlib_get_buffer (vm, from[2]);
 402                 p3 = vlib_get_buffer (vm, from[3]);
 403                 p4 = vlib_get_buffer (vm, from[4]);
 404                 p5 = vlib_get_buffer (vm, from[5]);
 405
 406                 vlib_prefetch_buffer_header (p2, STORE);
 407                 vlib_prefetch_buffer_header (p3, STORE);
 408                 vlib_prefetch_buffer_header (p4, STORE);
 409                 vlib_prefetch_buffer_header (p5, STORE);
 410
 411                 CLIB_PREFETCH (p2->data, sizeof (hdr0[0]), STORE);
 412                 CLIB_PREFETCH (p3->data, sizeof (hdr0[0]), STORE);
 413                 CLIB_PREFETCH (p4->data, sizeof (hdr0[0]), STORE);
 414                 CLIB_PREFETCH (p5->data, sizeof (hdr0[0]), STORE);
 415             }
 416
 417             from += 4;
 418             to_next += 4;
 419             n_left_from -= 4;
 420             n_left_to_next -= 4;
 421
 422             b0 = vlib_get_buffer (vm, bi0);
 423             b1 = vlib_get_buffer (vm, bi1);
 424             b2 = vlib_get_buffer (vm, bi2);
 425             b3 = vlib_get_buffer (vm, bi3);
 426
 427             /* dst lookup was done by ip4 lookup */
 428             mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 429             mldi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 430             mldi2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
 431             mldi3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
 432             mld0 = mpls_label_dpo_get(mldi0);
 433             mld1 = mpls_label_dpo_get(mldi1);
 434             mld2 = mpls_label_dpo_get(mldi2);
 435             mld3 = mpls_label_dpo_get(mldi3);
 436
 437             if (DPO_PROTO_MPLS != dproto)
 438             {
 439                 /*
 440                  * These are the non-MPLS payload imposition cases
 441                  */
 442                 if (DPO_PROTO_IP4 == dproto)
 443                 {
 444                     ip4_header_t * ip0 = vlib_buffer_get_current(b0);
 445                     ip4_header_t * ip1 = vlib_buffer_get_current(b1);
 446                     ip4_header_t * ip2 = vlib_buffer_get_current(b2);
 447                     ip4_header_t * ip3 = vlib_buffer_get_current(b3);
 448
 449                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 450                     {
 451                         /*
 452                          * decrement the TTL on ingress to the LSP
 453                          */
 454                         u32 checksum0;
 455                         u32 checksum1;
 456                         u32 checksum2;
 457                         u32 checksum3;
 458
 459                         checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
 460                         checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
 461                         checksum2 = ip2->checksum + clib_host_to_net_u16 (0x0100);
 462                         checksum3 = ip3->checksum + clib_host_to_net_u16 (0x0100);
 463
 464                         checksum0 += checksum0 >= 0xffff;
 465                         checksum1 += checksum1 >= 0xffff;
 466                         checksum2 += checksum2 >= 0xffff;
 467                         checksum3 += checksum3 >= 0xffff;
 468
 469                         ip0->checksum = checksum0;
 470                         ip1->checksum = checksum1;
 471                         ip2->checksum = checksum2;
 472                         ip3->checksum = checksum3;
 473
 474                         ip0->ttl -= 1;
 475                         ip1->ttl -= 1;
 476                         ip2->ttl -= 1;
 477                         ip3->ttl -= 1;
 478                     }
 479
 480                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 481                     {
 482                         ttl1 = ip1->ttl;
 483                         ttl0 = ip0->ttl;
 484                         ttl3 = ip3->ttl;
 485                         ttl2 = ip2->ttl;
 486                         /* by default copy the 3 most significant bits */
 487                         exp0 = ip_dscp_to_mpls_exp(ip0->tos);
 488                         exp1 = ip_dscp_to_mpls_exp(ip1->tos);
 489                         exp2 = ip_dscp_to_mpls_exp(ip2->tos);
 490                         exp3 = ip_dscp_to_mpls_exp(ip3->tos);
 491                     }
 492
 493                     /* save the payload proto information in mpls opaque */
 494                     vnet_buffer(b0)->mpls.pyld_proto = DPO_PROTO_IP4;
 495                     vnet_buffer(b1)->mpls.pyld_proto = DPO_PROTO_IP4;
 496                     vnet_buffer(b2)->mpls.pyld_proto = DPO_PROTO_IP4;
 497                     vnet_buffer(b3)->mpls.pyld_proto = DPO_PROTO_IP4;
 498                 }
 499                 else if (DPO_PROTO_IP6 == dproto)
 500                 {
 501                     /*
 502                      * decrement the TTL on ingress to the LSP
 503                      */
 504                     ip6_header_t * ip0 = vlib_buffer_get_current(b0);
 505                     ip6_header_t * ip1 = vlib_buffer_get_current(b1);
 506                     ip6_header_t * ip2 = vlib_buffer_get_current(b2);
 507                     ip6_header_t * ip3 = vlib_buffer_get_current(b3);
 508
 509                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 510                     {
 511                         ip0->hop_limit -= 1;
 512                         ip1->hop_limit -= 1;
 513                         ip2->hop_limit -= 1;
 514                         ip3->hop_limit -= 1;
 515                     }
 516                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 517                     {
 518                         ttl0 = ip0->hop_limit;
 519                         ttl1 = ip1->hop_limit;
 520                         ttl2 = ip2->hop_limit;
 521                         ttl3 = ip3->hop_limit;
 522                         /* by default copy the 3 most significant bits */
 523                         exp0 = ip_dscp_to_mpls_exp(
 524                             ip6_traffic_class_network_order(ip0));
 525                         exp1 = ip_dscp_to_mpls_exp(
 526                             ip6_traffic_class_network_order(ip1));
 527                         exp2 = ip_dscp_to_mpls_exp(
 528                             ip6_traffic_class_network_order(ip2));
 529                         exp3 = ip_dscp_to_mpls_exp(
 530                             ip6_traffic_class_network_order(ip3));
 531                     }
 532
 533                     /* save the payload proto information in mpls opaque */
 534                     vnet_buffer(b0)->mpls.pyld_proto = DPO_PROTO_IP6;
 535                     vnet_buffer(b1)->mpls.pyld_proto = DPO_PROTO_IP6;
 536                     vnet_buffer(b2)->mpls.pyld_proto = DPO_PROTO_IP6;
 537                     vnet_buffer(b3)->mpls.pyld_proto = DPO_PROTO_IP6;
 538                 }
 539                 else
 540                 {
 541                     /*
 542                      * nothing to change in the ethernet header
 543                      */
 544                     ttl0 = ttl1 = ttl2 = ttl3 = MPLS_LABEL_DEFAULT_TTL;
 545                     exp0 = exp1 = exp2 = exp3 = MPLS_LABEL_DEFAULT_EXP;
 546                 }
 547                 /*
 548                  * These are the non-MPLS payload imposition cases.
 549                  * Based on the LSP mode either, for uniform, copy down the TTL
 550                  * and EXP from the payload or, for pipe mode, slap on the value
 551                  * requested from config
 552                  */
 553                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 554                 {
 555                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 556                     hdr1 = mpls_label_paint_w_ttl_exp(b1, mld1, ttl1, exp1);
 557                     hdr2 = mpls_label_paint_w_ttl_exp(b2, mld2, ttl2, exp2);
 558                     hdr3 = mpls_label_paint_w_ttl_exp(b3, mld3, ttl3, exp3);
 559                 }
 560                 else
 561                 {
 562                     hdr0 = mpls_label_paint(b0, mld0);
 563                     hdr1 = mpls_label_paint(b1, mld1);
 564                     hdr2 = mpls_label_paint(b2, mld2);
 565                     hdr3 = mpls_label_paint(b3, mld3);
 566                 }
 567             }
 568             else
 569             {
 570                 /*
 571                  * else, the packet to be encapped is an MPLS packet
 572                  * there are two cases to consider:
 573                  *  1 - this is an MPLS label swap at an LSP midpoint.
 574                  *      recognisable because mpls.first = 1. In this case the
 575                  *      TTL must be set to the current value -1.
 576                  *  2 - The MPLS packet is recursing (or being injected into)
 577                  *      this LSP, in which case the pipe/uniform rules apply
 578                  *
 579                  */
 580                 if (PREDICT_TRUE(vnet_buffer(b0)->mpls.first))
 581                 {
 582                     /*
 583                      * The first label to be imposed on the packet. this is a
 584                      * label swap.in which case we stashed the TTL and EXP bits
 585                      * in the packet in the lookup node
 586                      */
 587                     ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
 588
 589                     ttl0 = vnet_buffer(b0)->mpls.ttl - 1;
 590                     exp0 = vnet_buffer(b0)->mpls.exp;
 591                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 592                 }
 593                 else
 594                 {
 595                     /*
 596                      * not the first label. implying we are recusring down a
 597                      * chain of output labels. Each layer is considered a new
 598                      * LSP - hence the TTL/EXP are pipe/uniform handled
 599                      */
 600                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 601                     {
 602                         hdr0 = vlib_buffer_get_current(b0);
 603                         ttl0 = ((u8*)hdr0)[3];
 604                         exp0 = ((u8*)hdr0)[2] & 0xe;
 605                         hdr0 = mpls_label_paint_w_ttl_mpls_exp(b0, mld0, ttl0, exp0);
 606                     }
 607                     else
 608                     {
 609                         hdr0 = mpls_label_paint(b0, mld0);
 610                     }
 611                 }
 612                 if (PREDICT_TRUE(vnet_buffer(b1)->mpls.first))
 613                 {
 614                     ASSERT(0 != vnet_buffer (b1)->mpls.ttl);
 615
 616                     ttl1 = vnet_buffer(b1)->mpls.ttl - 1;
 617                     exp1 = vnet_buffer(b1)->mpls.exp;
 618                     hdr1 = mpls_label_paint_w_ttl_exp(b1, mld1, ttl1, exp1);
 619                 }
 620                 else
 621                 {
 622                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 623                     {
 624                         hdr1 = vlib_buffer_get_current(b1);
 625                         ttl1 = ((u8*)hdr1)[3];
 626                         exp1 = ((u8*)hdr1)[2] & 0xe;
 627                         hdr1 = mpls_label_paint_w_ttl_mpls_exp(b1, mld1, ttl1, exp1);
 628                     }
 629                     else
 630                     {
 631                         hdr1 = mpls_label_paint(b1, mld1);
 632                     }
 633                 }
 634                 if (PREDICT_TRUE(vnet_buffer(b2)->mpls.first))
 635                 {
 636                     ASSERT(0 != vnet_buffer (b2)->mpls.ttl);
 637
 638                     ttl2 = vnet_buffer(b2)->mpls.ttl - 1;
 639                     exp2 = vnet_buffer(b2)->mpls.exp;
 640                     hdr2 = mpls_label_paint_w_ttl_exp(b2, mld2, ttl2, exp2);
 641                 }
 642                 else
 643                 {
 644                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 645                     {
 646                         hdr2 = vlib_buffer_get_current(b2);
 647                         ttl2 = ((u8*)hdr2)[3];
 648                         exp2 = ((u8*)hdr2)[2] & 0xe;
 649                         hdr2 = mpls_label_paint_w_ttl_mpls_exp(b2, mld2, ttl2, exp2);
 650                     }
 651                     else
 652                     {
 653                         hdr2 = mpls_label_paint(b2, mld2);
 654                     }
 655                 }
 656                 if (PREDICT_TRUE(vnet_buffer(b3)->mpls.first))
 657                 {
 658                     ASSERT(0 != vnet_buffer (b3)->mpls.ttl);
 659
 660                     ttl3 = vnet_buffer(b3)->mpls.ttl - 1;
 661                     exp3 = vnet_buffer(b3)->mpls.exp;
 662                     hdr3 = mpls_label_paint_w_ttl_exp(b3, mld3, ttl3, exp3);
 663                 }
 664                 else
 665                 {
 666                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 667                     {
 668                         hdr3 = vlib_buffer_get_current(b3);
 669                         ttl3 = ((u8*)hdr3)[3];
 670                         exp3 = ((u8*)hdr3)[2] & 0xe;
 671                         hdr3 = mpls_label_paint_w_ttl_mpls_exp(b3, mld3, ttl3, exp3);
 672                     }
 673                     else
 674                     {
 675                         hdr3 = mpls_label_paint(b3, mld3);
 676                     }
 677                 }
 678
 679                 vnet_buffer(b0)->mpls.first = 0;
 680                 vnet_buffer(b1)->mpls.first = 0;
 681                 vnet_buffer(b2)->mpls.first = 0;
 682                 vnet_buffer(b3)->mpls.first = 0;
 683             }
 684
 685             next0 = mld0->mld_dpo.dpoi_next_node;
 686             next1 = mld1->mld_dpo.dpoi_next_node;
 687             next2 = mld2->mld_dpo.dpoi_next_node;
 688             next3 = mld3->mld_dpo.dpoi_next_node;
 689
 690             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
 691             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = mld1->mld_dpo.dpoi_index;
 692             vnet_buffer(b2)->ip.adj_index[VLIB_TX] = mld2->mld_dpo.dpoi_index;
 693             vnet_buffer(b3)->ip.adj_index[VLIB_TX] = mld3->mld_dpo.dpoi_index;
 694
 695             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 696             {
 697                 mpls_label_imposition_trace_t *tr =
 698                     vlib_add_trace (vm, node, b0, sizeof (*tr));
 699                 tr->hdr = *hdr0;
 700                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 701                 {
 702                     tr->ttl = ttl0;
 703                     tr->exp = exp0;
 704                 }
 705                 else
 706                 {
 707                     tr->ttl = tr->exp = 0;
 708                 }
 709             }
 710             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 711             {
 712                 mpls_label_imposition_trace_t *tr =
 713                     vlib_add_trace (vm, node, b1, sizeof (*tr));
 714                 tr->hdr = *hdr1;
 715                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 716                 {
 717                     tr->ttl = ttl1;
 718                     tr->exp = exp1;
 719                 }
 720                 else
 721                 {
 722                     tr->ttl = tr->exp = 0;
 723                 }
 724             }
 725             if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
 726             {
 727                 mpls_label_imposition_trace_t *tr =
 728                     vlib_add_trace (vm, node, b2, sizeof (*tr));
 729                 tr->hdr = *hdr2;
 730                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 731                 {
 732                     tr->ttl = ttl2;
 733                     tr->exp = exp2;
 734                 }
 735                 else
 736                 {
 737                     tr->ttl = tr->exp = 0;
 738                 }
 739             }
 740             if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
 741             {
 742                 mpls_label_imposition_trace_t *tr =
 743                     vlib_add_trace (vm, node, b3, sizeof (*tr));
 744                 tr->hdr = *hdr3;
 745                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 746                 {
 747                     tr->ttl = ttl3;
 748                     tr->exp = exp3;
 749                 }
 750                 else
 751                 {
 752                     tr->ttl = tr->exp = 0;
 753                 }
 754             }
 755
 756             vlib_validate_buffer_enqueue_x4(vm, node, next_index, to_next,
 757                                             n_left_to_next,
 758                                             bi0, bi1, bi2, bi3,
 759                                             next0, next1, next2, next3);
 760         }
 761
 762         while (n_left_from > 0 && n_left_to_next > 0)
 763         {
 764             mpls_unicast_header_t *hdr0;
 765             mpls_label_dpo_t *mld0;
 766             vlib_buffer_t * b0;
 767             u32 bi0, mldi0;
 768             u8 ttl0, exp0;
 769             u32 next0;
 770
 771             bi0 = from[0];
 772             to_next[0] = bi0;
 773             from += 1;
 774             to_next += 1;
 775             n_left_from -= 1;
 776             n_left_to_next -= 1;
 777
 778             b0 = vlib_get_buffer (vm, bi0);
 779
 780             /* dst lookup was done by ip4 lookup */
 781             mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 782             mld0 = mpls_label_dpo_get(mldi0);
 783
 784             if (DPO_PROTO_MPLS != dproto)
 785             {
 786                 if (DPO_PROTO_IP4 == dproto)
 787                 {
 788                     /*
 789                      * decrement the TTL on ingress to the LSP
 790                      */
 791                     ip4_header_t * ip0 = vlib_buffer_get_current(b0);
 792                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 793                     {
 794                         u32 checksum0;
 795
 796                         checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
 797                         checksum0 += checksum0 >= 0xffff;
 798
 799                         ip0->checksum = checksum0;
 800                         ip0->ttl -= 1;
 801                     }
 802                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 803                     {
 804                         ttl0 = ip0->ttl;
 805                         exp0 = ip_dscp_to_mpls_exp(ip0->tos);
 806                     }
 807
 808                     /* save the payload proto information in mpls opaque */
 809                     vnet_buffer(b0)->mpls.pyld_proto = DPO_PROTO_IP4;
 810                 }
 811                 else if (DPO_PROTO_IP6 == dproto)
 812                 {
 813                     /*
 814                      * decrement the TTL on ingress to the LSP
 815                      */
 816                     ip6_header_t * ip0 = vlib_buffer_get_current(b0);
 817
 818                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 819                     {
 820                         ip0->hop_limit -= 1;
 821                     }
 822                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 823                     {
 824                         ttl0 = ip0->hop_limit;
 825                         exp0 = ip_dscp_to_mpls_exp(
 826                             ip6_traffic_class_network_order(ip0));
 827                     }
 828
 829                     /* save the payload proto information in mpls opaque */
 830                     vnet_buffer(b0)->mpls.pyld_proto = DPO_PROTO_IP6;
 831                 }
 832                 else
 833                 {
 834                     /*
 835                      * nothing to change in the ethernet header
 836                      */
 837                     ttl0 = MPLS_LABEL_DEFAULT_TTL;
 838                     exp0 = MPLS_LABEL_DEFAULT_EXP;
 839                 }
 840
 841                 /*
 842                  * These are the non-MPLS payload imposition cases.
 843                  * Based on the LSP mode either, for uniform, copy down the TTL
 844                  * from the payload or, for pipe mode, slap on the value
 845                  * requested from config
 846                  */
 847                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 848                 {
 849                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 850                 }
 851                 else
 852                 {
 853                     hdr0 = mpls_label_paint(b0, mld0);
 854                 }
 855             }
 856             else
 857             {
 858                 if (PREDICT_TRUE(vnet_buffer(b0)->mpls.first))
 859                 {
 860                     ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
 861
 862                     ttl0 = vnet_buffer(b0)->mpls.ttl - 1;
 863                     exp0 = vnet_buffer(b0)->mpls.exp;
 864                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 865                 }
 866                 else
 867                 {
 868                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 869                     {
 870                         hdr0 = vlib_buffer_get_current(b0);
 871                         ttl0 = ((u8*)hdr0)[3];
 872                         exp0 = ((u8*)hdr0)[2] & 0xe;
 873                         hdr0 = mpls_label_paint_w_ttl_mpls_exp(b0, mld0, ttl0, exp0);
 874                     }
 875                     else
 876                     {
 877                         hdr0 = mpls_label_paint(b0, mld0);
 878                     }
 879                 }
 880
 881                 vnet_buffer(b0)->mpls.first = 0;
 882             }
 883
 884             next0 = mld0->mld_dpo.dpoi_next_node;
 885             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
 886
 887             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 888             {
 889                 mpls_label_imposition_trace_t *tr =
 890                     vlib_add_trace (vm, node, b0, sizeof (*tr));
 891                 tr->hdr = *hdr0;
 892                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 893                 {
 894                     tr->ttl = ttl0;
 895                     tr->exp = exp0;
 896                 }
 897                 else
 898                 {
 899                     tr->ttl = tr->exp = 0;
 900                 }
 901            }
 902
 903             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 904                                             n_left_to_next, bi0, next0);
 905         }
 906         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 907     }
 908     return from_frame->n_vectors;
 909 }
 910
 911 static u8 *
 912 format_mpls_label_imposition_trace (u8 * s, va_list * args)
 913 {
 914     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 915     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 916     mpls_label_imposition_trace_t * t;
 917     mpls_unicast_header_t hdr;
 918     u32 indent;
 919
 920     t = va_arg (*args, mpls_label_imposition_trace_t *);
 921     indent = format_get_indent (s);
 922     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
 923
 924     s = format (s, "%Umpls-header:%U",
 925                 format_white_space, indent,
 926                 format_mpls_header, hdr);
 927     return (s);
 928 }
 929
 930 VLIB_NODE_FN (mpls_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 931                                  vlib_node_runtime_t * node,
 932                                  vlib_frame_t * frame)
 933 {
 934     return (mpls_label_imposition_inline(vm, node, frame,
 935                                          DPO_PROTO_MPLS,
 936                                          MPLS_LABEL_DPO_FLAG_NONE));
 937 }
 938
 939 VLIB_REGISTER_NODE (mpls_mpls_label_imposition_pipe_node) = {
 940     .name = "mpls-label-imposition-pipe",
 941     .vector_size = sizeof (u32),
 942
 943     .format_trace = format_mpls_label_imposition_trace,
 944     .n_next_nodes = 1,
 945     .next_nodes = {
 946         [0] = "mpls-drop",
 947     }
 948 };
 949
 950 VLIB_NODE_FN (ip4_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 951                                 vlib_node_runtime_t * node,
 952                                 vlib_frame_t * frame)
 953 {
 954     return (mpls_label_imposition_inline(vm, node, frame,
 955                                          DPO_PROTO_IP4,
 956                                          MPLS_LABEL_DPO_FLAG_NONE));
 957 }
 958
 959 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_pipe_node) = {
 960     .name = "ip4-mpls-label-imposition-pipe",
 961     .vector_size = sizeof (u32),
 962
 963     .format_trace = format_mpls_label_imposition_trace,
 964     .n_next_nodes = 1,
 965     .next_nodes = {
 966         [0] = "ip4-drop",
 967     }
 968 };
 969
 970 VLIB_NODE_FN (ip6_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 971                                 vlib_node_runtime_t * node,
 972                                 vlib_frame_t * frame)
 973 {
 974     return (mpls_label_imposition_inline(vm, node, frame,
 975                                          DPO_PROTO_IP6,
 976                                          MPLS_LABEL_DPO_FLAG_NONE));
 977 }
 978
 979 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_pipe_node) = {
 980     .name = "ip6-mpls-label-imposition-pipe",
 981     .vector_size = sizeof (u32),
 982
 983     .format_trace = format_mpls_label_imposition_trace,
 984     .n_next_nodes = 1,
 985     .next_nodes = {
 986         [0] = "ip6-drop",
 987     }
 988 };
 989
 990 VLIB_NODE_FN (ethernet_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 991                                      vlib_node_runtime_t * node,
 992                                      vlib_frame_t * frame)
 993 {
 994     return (mpls_label_imposition_inline(vm, node, frame,
 995                                          DPO_PROTO_ETHERNET,
 996                                          MPLS_LABEL_DPO_FLAG_NONE));
 997 }
 998
 999 VLIB_REGISTER_NODE (ethernet_mpls_label_imposition_pipe_node) = {
1000     .name = "ethernet-mpls-label-imposition-pipe",
1001     .vector_size = sizeof (u32),
1002
1003     .format_trace = format_mpls_label_imposition_trace,
1004     .n_next_nodes = 1,
1005     .next_nodes = {
1006         [0] = "error-drop",
1007     }
1008 };
1009
1010 VLIB_NODE_FN (mpls_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
1011                                     vlib_node_runtime_t * node,
1012                                     vlib_frame_t * frame)
1013 {
1014     return (mpls_label_imposition_inline(vm, node, frame,
1015                                          DPO_PROTO_MPLS,
1016                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
1017 }
1018
1019 VLIB_REGISTER_NODE (mpls_mpls_label_imposition_uniform_node) = {
1020     .name = "mpls-label-imposition-uniform",
1021     .vector_size = sizeof (u32),
1022
1023     .format_trace = format_mpls_label_imposition_trace,
1024     .n_next_nodes = 1,
1025     .next_nodes = {
1026         [0] = "mpls-drop",
1027     }
1028 };
1029
1030 VLIB_NODE_FN (ip4_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
1031                                    vlib_node_runtime_t * node,
1032                                    vlib_frame_t * frame)
1033 {
1034     return (mpls_label_imposition_inline(vm, node, frame,
1035                                          DPO_PROTO_IP4,
1036                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
1037 }
1038
1039 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_uniform_node) = {
1040     .name = "ip4-mpls-label-imposition-uniform",
1041     .vector_size = sizeof (u32),
1042
1043     .format_trace = format_mpls_label_imposition_trace,
1044     .n_next_nodes = 1,
1045     .next_nodes = {
1046         [0] = "ip4-drop",
1047     }
1048 };
1049
1050 VLIB_NODE_FN (ip6_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
1051                                    vlib_node_runtime_t * node,
1052                                    vlib_frame_t * frame)
1053 {
1054     return (mpls_label_imposition_inline(vm, node, frame,
1055                                          DPO_PROTO_IP6,
1056                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
1057 }
1058
1059 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_uniform_node) = {
1060     .name = "ip6-mpls-label-imposition-uniform",
1061     .vector_size = sizeof (u32),
1062
1063     .format_trace = format_mpls_label_imposition_trace,
1064     .n_next_nodes = 1,
1065     .next_nodes = {
1066         [0] = "ip6-drop",
1067     }
1068 };
1069
1070 VLIB_NODE_FN (ethernet_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
1071                                         vlib_node_runtime_t * node,
1072                                         vlib_frame_t * frame)
1073 {
1074     return (mpls_label_imposition_inline(vm, node, frame,
1075                                          DPO_PROTO_ETHERNET,
1076                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
1077 }
1078
1079 VLIB_REGISTER_NODE (ethernet_mpls_label_imposition_uniform_node) = {
1080     .name = "ethernet-mpls-label-imposition-uniform",
1081     .vector_size = sizeof (u32),
1082
1083     .format_trace = format_mpls_label_imposition_trace,
1084     .n_next_nodes = 1,
1085     .next_nodes = {
1086         [0] = "error-drop",
1087     }
1088 };
1089
1090
1091 VLIB_NODE_FN (ip4_mpls_label_imposition_pipe_no_ip_ttl_decr_node) (vlib_main_t * vm,
1092                                                vlib_node_runtime_t * node,
1093                                                vlib_frame_t * frame)
1094 {
1095     return (mpls_label_imposition_inline(vm, node, frame,
1096                                          DPO_PROTO_IP4,
1097                                          MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR));
1098 }
1099
1100 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_pipe_no_ip_ttl_decr_node) = {
1101     .name = "ip4-mpls-label-imposition-pipe-no-ip-ttl-decr",
1102     .vector_size = sizeof (u32),
1103
1104     .format_trace = format_mpls_label_imposition_trace,
1105     .n_next_nodes = 1,
1106     .next_nodes = {
1107         [0] = "ip4-drop",
1108     }
1109 };
1110
1111 VLIB_NODE_FN (ip6_mpls_label_imposition_pipe_no_ip_ttl_decr_node) (vlib_main_t * vm,
1112                                                vlib_node_runtime_t * node,
1113                                                vlib_frame_t * frame)
1114 {
1115     return (mpls_label_imposition_inline(vm, node, frame,
1116                                          DPO_PROTO_IP6,
1117                                          MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR));
1118 }
1119
1120 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_pipe_no_ip_ttl_decr_node) = {
1121     .name = "ip6-mpls-label-imposition-pipe-no-ip-ttl-decr",
1122     .vector_size = sizeof (u32),
1123
1124     .format_trace = format_mpls_label_imposition_trace,
1125     .n_next_nodes = 1,
1126     .next_nodes = {
1127         [0] = "ip6-drop",
1128     }
1129 };
1130
1131 VLIB_NODE_FN (ip4_mpls_label_imposition_uniform_no_ip_ttl_decr_node) (vlib_main_t * vm,
1132                                                   vlib_node_runtime_t * node,
1133                                                   vlib_frame_t * frame)
1134 {
1135     return (mpls_label_imposition_inline(vm, node, frame,
1136                                          DPO_PROTO_IP4,
1137                                          (MPLS_LABEL_DPO_FLAG_UNIFORM_MODE |
1138                                           MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR)));
1139 }
1140
1141 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_uniform_no_ip_ttl_decr_node) = {
1142     .name = "ip4-mpls-label-imposition-uniform-no-ip-ttl-decr",
1143     .vector_size = sizeof (u32),
1144
1145     .format_trace = format_mpls_label_imposition_trace,
1146     .n_next_nodes = 1,
1147     .next_nodes = {
1148         [0] = "ip4-drop",
1149     }
1150 };
1151
1152 VLIB_NODE_FN (ip6_mpls_label_imposition_uniform_no_ip_ttl_decr_node) (vlib_main_t * vm,
1153                                                   vlib_node_runtime_t * node,
1154                                                   vlib_frame_t * frame)
1155 {
1156     return (mpls_label_imposition_inline(vm, node, frame,
1157                                          DPO_PROTO_IP6,
1158                                          (MPLS_LABEL_DPO_FLAG_UNIFORM_MODE |
1159                                           MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR)));
1160 }
1161
1162 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_uniform_no_ip_ttl_decr_node) = {
1163     .name = "ip6-mpls-label-imposition-uniform-no-ip-ttl-decr",
1164     .vector_size = sizeof (u32),
1165
1166     .format_trace = format_mpls_label_imposition_trace,
1167     .n_next_nodes = 1,
1168     .next_nodes = {
1169         [0] = "ip6-drop",
1170     }
1171 };
1172
1173
1174 #ifndef CLIB_MARCH_VARIANT
1175 static void
1176 mpls_label_dpo_mem_show (void)
1177 {
1178     fib_show_memory_usage("MPLS label",
1179                           pool_elts(mpls_label_dpo_pool),
1180                           pool_len(mpls_label_dpo_pool),
1181                           sizeof(mpls_label_dpo_t));
1182 }
1183
1184 /**
1185  * Interpose a label DPO. used in the FIB unit tests
1186  */
1187 static void
1188 mpls_label_interpose (const dpo_id_t *original,
1189                       const dpo_id_t *parent,
1190                       dpo_id_t *clone)
1191 {
1192     mpls_label_dpo_t *mld, *mld_clone;
1193
1194     mld_clone = mpls_label_dpo_alloc();
1195     mld = mpls_label_dpo_get(original->dpoi_index);
1196
1197     mld_clone->mld_locks = 0;
1198     clib_memcpy_fast(&mld_clone->mld_hdr,
1199                 &mld->mld_hdr,
1200                 sizeof(mld_clone->mld_hdr));
1201     mld_clone->mld_payload_proto = mld->mld_payload_proto;
1202     mld_clone->mld_n_labels = mld->mld_n_labels;
1203     mld_clone->mld_n_hdr_bytes = mld->mld_n_hdr_bytes;
1204
1205     dpo_stack(mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NONE],
1206               mld_clone->mld_payload_proto,
1207               &mld_clone->mld_dpo,
1208               parent);
1209
1210     dpo_set(clone,
1211             mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NONE],
1212             mld_clone->mld_payload_proto,
1213             mpls_label_dpo_get_index(mld_clone));
1214 }
1215
1216 const static dpo_vft_t mld_vft = {
1217     .dv_lock = mpls_label_dpo_lock,
1218     .dv_unlock = mpls_label_dpo_unlock,
1219     .dv_format = format_mpls_label_dpo,
1220     .dv_mem_show = mpls_label_dpo_mem_show,
1221     .dv_mk_interpose = mpls_label_interpose,
1222 };
1223
1224 const static char* const mpls_label_imp_pipe_ip4_nodes[] =
1225 {
1226     "ip4-mpls-label-imposition-pipe",
1227     NULL,
1228 };
1229 const static char* const mpls_label_imp_pipe_ip6_nodes[] =
1230 {
1231     "ip6-mpls-label-imposition-pipe",
1232     NULL,
1233 };
1234 const static char* const mpls_label_imp_pipe_mpls_nodes[] =
1235 {
1236     "mpls-label-imposition-pipe",
1237     NULL,
1238 };
1239 const static char* const mpls_label_imp_pipe_ethernet_nodes[] =
1240 {
1241     "ethernet-mpls-label-imposition-pipe",
1242     NULL,
1243 };
1244
1245 const static char* const * const mpls_label_imp_pipe_nodes[DPO_PROTO_NUM] =
1246 {
1247     [DPO_PROTO_IP4]  = mpls_label_imp_pipe_ip4_nodes,
1248     [DPO_PROTO_IP6]  = mpls_label_imp_pipe_ip6_nodes,
1249     [DPO_PROTO_MPLS] = mpls_label_imp_pipe_mpls_nodes,
1250     [DPO_PROTO_ETHERNET] = mpls_label_imp_pipe_ethernet_nodes,
1251 };
1252
1253 const static char* const mpls_label_imp_uniform_ip4_nodes[] =
1254 {
1255     "ip4-mpls-label-imposition-uniform",
1256     NULL,
1257 };
1258 const static char* const mpls_label_imp_uniform_ip6_nodes[] =
1259 {
1260     "ip6-mpls-label-imposition-uniform",
1261     NULL,
1262 };
1263 const static char* const mpls_label_imp_uniform_mpls_nodes[] =
1264 {
1265     "mpls-label-imposition-uniform",
1266     NULL,
1267 };
1268 const static char* const mpls_label_imp_uniform_ethernet_nodes[] =
1269 {
1270     "ethernet-mpls-label-imposition-uniform",
1271     NULL,
1272 };
1273
1274 const static char* const * const mpls_label_imp_uniform_nodes[DPO_PROTO_NUM] =
1275 {
1276     [DPO_PROTO_IP4]  = mpls_label_imp_uniform_ip4_nodes,
1277     [DPO_PROTO_IP6]  = mpls_label_imp_uniform_ip6_nodes,
1278     [DPO_PROTO_MPLS] = mpls_label_imp_uniform_mpls_nodes,
1279     [DPO_PROTO_ETHERNET] = mpls_label_imp_uniform_ethernet_nodes,
1280 };
1281
1282 const static char* const mpls_label_imp_pipe_no_ip_tll_decr_ip4_nodes[] =
1283 {
1284     "ip4-mpls-label-imposition-pipe-no-ip-ttl-decr",
1285     NULL,
1286 };
1287 const static char* const mpls_label_imp_pipe_no_ip_tll_decr_ip6_nodes[] =
1288 {
1289     "ip6-mpls-label-imposition-pipe-no-ip-ttl-decr",
1290     NULL,
1291 };
1292
1293 const static char* const * const mpls_label_imp_pipe_no_ip_tll_decr_nodes[DPO_PROTO_NUM] =
1294 {
1295     [DPO_PROTO_IP4]  = mpls_label_imp_pipe_no_ip_tll_decr_ip4_nodes,
1296     [DPO_PROTO_IP6]  = mpls_label_imp_pipe_no_ip_tll_decr_ip6_nodes,
1297 };
1298
1299 const static char* const mpls_label_imp_uniform_no_ip_tll_decr_ip4_nodes[] =
1300 {
1301     "ip4-mpls-label-imposition-uniform-no-ip-ttl-decr",
1302     NULL,
1303 };
1304 const static char* const mpls_label_imp_uniform_no_ip_tll_decr_ip6_nodes[] =
1305 {
1306     "ip6-mpls-label-imposition-uniform-no-ip-ttl-decr",
1307     NULL,
1308 };
1309
1310 const static char* const * const mpls_label_imp_uniform_no_ip_tll_decr_nodes[DPO_PROTO_NUM] =
1311 {
1312     [DPO_PROTO_IP4]  = mpls_label_imp_uniform_no_ip_tll_decr_ip4_nodes,
1313     [DPO_PROTO_IP6]  = mpls_label_imp_uniform_no_ip_tll_decr_ip6_nodes,
1314 };
1315
1316 void
1317 mpls_label_dpo_module_init (void)
1318 {
1319     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NONE] =
1320         dpo_register_new_type(&mld_vft,
1321                               mpls_label_imp_pipe_nodes);
1322     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR] =
1323         dpo_register_new_type(&mld_vft,
1324                               mpls_label_imp_pipe_no_ip_tll_decr_nodes);
1325     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_UNIFORM_MODE] =
1326         dpo_register_new_type(&mld_vft,
1327                               mpls_label_imp_uniform_nodes);
1328     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_UNIFORM_MODE |
1329                          MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR] =
1330         dpo_register_new_type(&mld_vft,
1331                               mpls_label_imp_uniform_no_ip_tll_decr_nodes);
1332 }
1333
1334 dpo_type_t
1335 mpls_label_dpo_get_type (mpls_label_dpo_flags_t flags)
1336 {
1337     return (mpls_label_dpo_types[flags]);
1338 }
1339 #endif /* CLIB_MARCH_VARIANT */