src/vnet/dpo/mpls_label_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/mpls_label_dpo.h>
  18 #include <vnet/mpls/mpls.h>
  19 #include <vnet/dpo/drop_dpo.h>
  20
  21 #ifndef CLIB_MARCH_VARIANT
  22 /*
  23  * pool of all MPLS Label DPOs
  24  */
  25 mpls_label_dpo_t *mpls_label_dpo_pool;
  26
  27 /**
  28  * Strings for the flags
  29  */
  30 const char* mpls_label_dpo_attr_names[] = MPLS_LABEL_DPO_ATTR_NAMES;
  31
  32 /**
  33  * registered DPO types for each of the label sub-types. And there's a
  34  * subtype for each of the flag combinations.
  35  */
  36 static dpo_type_t mpls_label_dpo_types[1 << MPLS_LABEL_DPO_ATTR_MAX];
  37
  38 static mpls_label_dpo_t *
  39 mpls_label_dpo_alloc (void)
  40 {
  41     mpls_label_dpo_t *mld;
  42
  43     pool_get_aligned(mpls_label_dpo_pool, mld, CLIB_CACHE_LINE_BYTES);
  44     clib_memset(mld, 0, sizeof(*mld));
  45
  46     dpo_reset(&mld->mld_dpo);
  47
  48     return (mld);
  49 }
  50
  51 static index_t
  52 mpls_label_dpo_get_index (mpls_label_dpo_t *mld)
  53 {
  54     return (mld - mpls_label_dpo_pool);
  55 }
  56
  57 void
  58 mpls_label_dpo_create (fib_mpls_label_t *label_stack,
  59                        mpls_eos_bit_t eos,
  60                        dpo_proto_t payload_proto,
  61                        mpls_label_dpo_flags_t flags,
  62                        const dpo_id_t *parent,
  63                        dpo_id_t *dpo)
  64 {
  65     mpls_label_dpo_t *mld;
  66     dpo_type_t dtype;
  67     u32 ii;
  68
  69     if ((DPO_PROTO_IP4 != payload_proto) &&
  70         (DPO_PROTO_IP6 != payload_proto))
  71     {
  72         /*
  73          * remove unsupported configuration
  74          */
  75         flags &= ~MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR;
  76     }
  77
  78     mld = mpls_label_dpo_alloc();
  79     mld->mld_flags = flags;
  80     dtype = mpls_label_dpo_types[flags];
  81
  82     if (MPLS_LABEL_DPO_MAX_N_LABELS < vec_len(label_stack))
  83     {
  84         clib_warning("Label stack size exceeded");
  85         dpo_stack(dtype,
  86                   mld->mld_payload_proto,
  87                   &mld->mld_dpo,
  88                   drop_dpo_get(DPO_PROTO_MPLS));
  89     }
  90     else
  91     {
  92         mld->mld_n_labels = vec_len(label_stack);
  93         mld->mld_n_hdr_bytes = mld->mld_n_labels * sizeof(mld->mld_hdr[0]);
  94         mld->mld_payload_proto = payload_proto;
  95
  96         /*
  97          * construct label rewrite headers for each value passed.
  98          * get the header in network byte order since we will paint it
  99          * on a packet in the data-plane
 100          */
 101         for (ii = 0; ii < mld->mld_n_labels-1; ii++)
 102         {
 103             vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl,
 104                                    label_stack[ii].fml_value);
 105             vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl,
 106                                  label_stack[ii].fml_exp);
 107             vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl,
 108                                MPLS_NON_EOS);
 109             if (0 != label_stack[ii].fml_ttl)
 110             {
 111                 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 112                                      label_stack[ii].fml_ttl);
 113             }
 114             else
 115             {
 116                 vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 117                                      MPLS_LABEL_DEFAULT_TTL);
 118             }
 119             mld->mld_hdr[ii].label_exp_s_ttl =
 120                 clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
 121         }
 122
 123         /*
 124          * the inner most label
 125          */
 126         ii = mld->mld_n_labels-1;
 127
 128         vnet_mpls_uc_set_label(&mld->mld_hdr[ii].label_exp_s_ttl,
 129                                label_stack[ii].fml_value);
 130         vnet_mpls_uc_set_exp(&mld->mld_hdr[ii].label_exp_s_ttl,
 131                              label_stack[ii].fml_exp);
 132         vnet_mpls_uc_set_s(&mld->mld_hdr[ii].label_exp_s_ttl, eos);
 133         if (0 != label_stack[ii].fml_ttl)
 134         {
 135             vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 136                                  label_stack[ii].fml_ttl);
 137         }
 138         else
 139         {
 140             vnet_mpls_uc_set_ttl(&mld->mld_hdr[ii].label_exp_s_ttl,
 141                                  MPLS_LABEL_DEFAULT_TTL);
 142         }
 143         mld->mld_hdr[ii].label_exp_s_ttl =
 144             clib_host_to_net_u32(mld->mld_hdr[ii].label_exp_s_ttl);
 145
 146         /*
 147          * pipe/uniform mode is only supported for the bottom of stack label
 148          */
 149         if (FIB_MPLS_LSP_MODE_UNIFORM == label_stack[ii].fml_mode)
 150         {
 151             mld->mld_flags |= MPLS_LABEL_DPO_FLAG_UNIFORM_MODE;
 152         }
 153         else
 154         {
 155             mld->mld_flags &= ~MPLS_LABEL_DPO_FLAG_UNIFORM_MODE;
 156         }
 157         dtype = mpls_label_dpo_types[mld->mld_flags];
 158
 159         /*
 160          * stack this label object on its parent.
 161          */
 162         dpo_stack(dtype,
 163                   mld->mld_payload_proto,
 164                   &mld->mld_dpo,
 165                   parent);
 166     }
 167
 168     dpo_set(dpo,
 169             dtype,
 170             mld->mld_payload_proto,
 171             mpls_label_dpo_get_index(mld));
 172 }
 173
 174 u8*
 175 format_mpls_label_dpo_flags (u8 *s, va_list *args)
 176 {
 177     mpls_label_dpo_flags_t flags = va_arg (*args, int);
 178     mpls_label_dpo_attr_t attr;
 179
 180     FOR_EACH_MPLS_LABEL_DPO_ATTR(attr)
 181     {
 182         if ((1 << attr) & flags)
 183         {
 184             s = format(s, "%s,", mpls_label_dpo_attr_names[attr]);
 185         }
 186     }
 187
 188     return (s);
 189 }
 190
 191 u8*
 192 format_mpls_label_dpo (u8 *s, va_list *args)
 193 {
 194     index_t index = va_arg (*args, index_t);
 195     u32 indent = va_arg (*args, u32);
 196     mpls_unicast_header_t hdr;
 197     mpls_label_dpo_t *mld;
 198     u32 ii;
 199
 200     if (pool_is_free_index(mpls_label_dpo_pool, index))
 201     {
 202         /*
 203          * the packet trace can be printed after the DPO has been deleted
 204          */
 205         return (format(s, "mpls-label[???,%d]:", index));
 206     }
 207
 208     mld = mpls_label_dpo_get(index);
 209     s = format(s, "mpls-label[%U@%d]:",
 210                format_mpls_label_dpo_flags,
 211                (int) mld->mld_flags, index);
 212
 213     for (ii = 0; ii < mld->mld_n_labels; ii++)
 214     {
 215         hdr.label_exp_s_ttl =
 216             clib_net_to_host_u32(mld->mld_hdr[ii].label_exp_s_ttl);
 217         s = format(s, "%U", format_mpls_header, hdr);
 218     }
 219
 220     s = format(s, "\n%U", format_white_space, indent);
 221     s = format(s, "%U", format_dpo_id, &mld->mld_dpo, indent+2);
 222
 223     return (s);
 224 }
 225
 226 static void
 227 mpls_label_dpo_lock (dpo_id_t *dpo)
 228 {
 229     mpls_label_dpo_t *mld;
 230
 231     mld = mpls_label_dpo_get(dpo->dpoi_index);
 232
 233     mld->mld_locks++;
 234 }
 235
 236 static void
 237 mpls_label_dpo_unlock (dpo_id_t *dpo)
 238 {
 239     mpls_label_dpo_t *mld;
 240
 241     mld = mpls_label_dpo_get(dpo->dpoi_index);
 242
 243     mld->mld_locks--;
 244
 245     if (0 == mld->mld_locks)
 246     {
 247         dpo_reset(&mld->mld_dpo);
 248         pool_put(mpls_label_dpo_pool, mld);
 249     }
 250 }
 251 #endif /* CLIB_MARCH_VARIANT */
 252
 253 /**
 254  * @brief A struct to hold tracing information for the MPLS label imposition
 255  * node.
 256  */
 257 typedef struct mpls_label_imposition_trace_t_
 258 {
 259     /**
 260      * The MPLS header imposed
 261      */
 262     mpls_unicast_header_t hdr;
 263
 264     /**
 265      * TTL imposed - only valid for uniform LSPs
 266      */
 267     u8 ttl;
 268
 269     /**
 270      * TTL imposed - only valid for uniform LSPs
 271      */
 272     u8 exp;
 273 } mpls_label_imposition_trace_t;
 274
 275 always_inline mpls_unicast_header_t *
 276 mpls_label_paint (vlib_buffer_t * b0,
 277                   mpls_label_dpo_t *mld0)
 278 {
 279     mpls_unicast_header_t *hdr0;
 280
 281     vlib_buffer_advance(b0, -(mld0->mld_n_hdr_bytes));
 282
 283     hdr0 = vlib_buffer_get_current(b0);
 284
 285     if (1 == mld0->mld_n_labels)
 286     {
 287         /* optimise for the common case of one label */
 288         *hdr0 = mld0->mld_hdr[0];
 289     }
 290     else
 291     {
 292         clib_memcpy_fast(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes);
 293         hdr0 = hdr0 + (mld0->mld_n_labels - 1);
 294     }
 295
 296     return (hdr0);
 297 }
 298
 299 /**
 300  * Paint on an MPLS label and fixup the TTL
 301  */
 302 always_inline mpls_unicast_header_t *
 303 mpls_label_paint_w_ttl (vlib_buffer_t * b0,
 304                         mpls_label_dpo_t *mld0,
 305                         u8 ttl0)
 306 {
 307     mpls_unicast_header_t *hdr0;
 308
 309     hdr0 = mpls_label_paint(b0, mld0);
 310
 311     /* fixup the TTL for the inner most label */
 312     ((char*)hdr0)[3] = ttl0;
 313
 314     return (hdr0);
 315 }
 316
 317 /**
 318  * Paint on an MPLS label and fixup the TTL and EXP bits.
 319  */
 320 always_inline mpls_unicast_header_t *
 321 mpls_label_paint_w_ttl_exp (vlib_buffer_t * b0,
 322                             mpls_label_dpo_t *mld0,
 323                             u8 ttl0,
 324                             u8 exp0)
 325 {
 326     mpls_unicast_header_t *hdr0;
 327
 328     hdr0 = mpls_label_paint_w_ttl(b0, mld0, ttl0);
 329
 330     /* fixup the EXP for the inner most label */
 331     ((char*)hdr0)[2] |= (exp0 << 1);
 332
 333     return (hdr0);
 334 }
 335
 336 /**
 337  * Paint on an MPLS label and fixup the TTL and EXP bits
 338  * When the EXP bits are *already* bit shift to the correct place in
 339  * in the 2nd byte (i.e. they were read from another label)
 340  */
 341 always_inline mpls_unicast_header_t *
 342 mpls_label_paint_w_ttl_mpls_exp (vlib_buffer_t * b0,
 343                                  mpls_label_dpo_t *mld0,
 344                                  u8 ttl0,
 345                                  u8 exp0)
 346 {
 347     mpls_unicast_header_t *hdr0;
 348
 349     hdr0 = mpls_label_paint_w_ttl(b0, mld0, ttl0);
 350
 351     /* fixup the EXP for the inner most label */
 352     ((char*)hdr0)[2] |= exp0;
 353
 354     return (hdr0);
 355 }
 356
 357 always_inline uword
 358 mpls_label_imposition_inline (vlib_main_t * vm,
 359                               vlib_node_runtime_t * node,
 360                               vlib_frame_t * from_frame,
 361                               const dpo_proto_t dproto,
 362                               const mpls_label_dpo_flags_t flags)
 363 {
 364     u32 n_left_from, next_index, * from, * to_next;
 365
 366     from = vlib_frame_vector_args (from_frame);
 367     n_left_from = from_frame->n_vectors;
 368
 369     next_index = node->cached_next_index;
 370
 371     while (n_left_from > 0)
 372     {
 373         u32 n_left_to_next;
 374
 375         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 376
 377         while (n_left_from >= 8 && n_left_to_next >= 4)
 378         {
 379             u32 bi0, mldi0, bi1, mldi1, bi2, mldi2, bi3, mldi3;
 380             mpls_unicast_header_t *hdr0, *hdr1, *hdr2, *hdr3;
 381             mpls_label_dpo_t *mld0, *mld1, *mld2, *mld3;
 382             vlib_buffer_t * b0, *b1, * b2, *b3;
 383             u32 next0, next1, next2, next3;
 384             u8 ttl0, ttl1, ttl2, ttl3;
 385             u8 exp0, exp1, exp2, exp3;
 386
 387             bi0 = to_next[0] = from[0];
 388             bi1 = to_next[1] = from[1];
 389             bi2 = to_next[2] = from[2];
 390             bi3 = to_next[3] = from[3];
 391
 392             /* Prefetch next iteration. */
 393             {
 394                 vlib_buffer_t * p2, * p3, *p4, *p5;
 395
 396                 p2 = vlib_get_buffer (vm, from[2]);
 397                 p3 = vlib_get_buffer (vm, from[3]);
 398                 p4 = vlib_get_buffer (vm, from[4]);
 399                 p5 = vlib_get_buffer (vm, from[5]);
 400
 401                 vlib_prefetch_buffer_header (p2, STORE);
 402                 vlib_prefetch_buffer_header (p3, STORE);
 403                 vlib_prefetch_buffer_header (p4, STORE);
 404                 vlib_prefetch_buffer_header (p5, STORE);
 405
 406                 CLIB_PREFETCH (p2->data, sizeof (hdr0[0]), STORE);
 407                 CLIB_PREFETCH (p3->data, sizeof (hdr0[0]), STORE);
 408                 CLIB_PREFETCH (p4->data, sizeof (hdr0[0]), STORE);
 409                 CLIB_PREFETCH (p5->data, sizeof (hdr0[0]), STORE);
 410             }
 411
 412             from += 4;
 413             to_next += 4;
 414             n_left_from -= 4;
 415             n_left_to_next -= 4;
 416
 417             b0 = vlib_get_buffer (vm, bi0);
 418             b1 = vlib_get_buffer (vm, bi1);
 419             b2 = vlib_get_buffer (vm, bi2);
 420             b3 = vlib_get_buffer (vm, bi3);
 421
 422             /* dst lookup was done by ip4 lookup */
 423             mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 424             mldi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 425             mldi2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
 426             mldi3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
 427             mld0 = mpls_label_dpo_get(mldi0);
 428             mld1 = mpls_label_dpo_get(mldi1);
 429             mld2 = mpls_label_dpo_get(mldi2);
 430             mld3 = mpls_label_dpo_get(mldi3);
 431
 432             if (DPO_PROTO_MPLS != dproto)
 433             {
 434                 /*
 435                  * These are the non-MPLS payload imposition cases
 436                  */
 437                 if (DPO_PROTO_IP4 == dproto)
 438                 {
 439                     ip4_header_t * ip0 = vlib_buffer_get_current(b0);
 440                     ip4_header_t * ip1 = vlib_buffer_get_current(b1);
 441                     ip4_header_t * ip2 = vlib_buffer_get_current(b2);
 442                     ip4_header_t * ip3 = vlib_buffer_get_current(b3);
 443
 444                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 445                     {
 446                         /*
 447                          * decrement the TTL on ingress to the LSP
 448                          */
 449                         u32 checksum0;
 450                         u32 checksum1;
 451                         u32 checksum2;
 452                         u32 checksum3;
 453
 454                         checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
 455                         checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
 456                         checksum2 = ip2->checksum + clib_host_to_net_u16 (0x0100);
 457                         checksum3 = ip3->checksum + clib_host_to_net_u16 (0x0100);
 458
 459                         checksum0 += checksum0 >= 0xffff;
 460                         checksum1 += checksum1 >= 0xffff;
 461                         checksum2 += checksum2 >= 0xffff;
 462                         checksum3 += checksum3 >= 0xffff;
 463
 464                         ip0->checksum = checksum0;
 465                         ip1->checksum = checksum1;
 466                         ip2->checksum = checksum2;
 467                         ip3->checksum = checksum3;
 468
 469                         ip0->ttl -= 1;
 470                         ip1->ttl -= 1;
 471                         ip2->ttl -= 1;
 472                         ip3->ttl -= 1;
 473                     }
 474
 475                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 476                     {
 477                         ttl1 = ip1->ttl;
 478                         ttl0 = ip0->ttl;
 479                         ttl3 = ip3->ttl;
 480                         ttl2 = ip2->ttl;
 481                         /* by default copy the 3 most significant bits */
 482                         exp0 = ip_dscp_to_mpls_exp(ip0->tos);
 483                         exp1 = ip_dscp_to_mpls_exp(ip1->tos);
 484                         exp2 = ip_dscp_to_mpls_exp(ip2->tos);
 485                         exp3 = ip_dscp_to_mpls_exp(ip3->tos);
 486                     }
 487
 488                     /* save the payload proto information in mpls opaque */
 489                     vnet_buffer(b0)->mpls.pyld_proto = DPO_PROTO_IP4;
 490                     vnet_buffer(b1)->mpls.pyld_proto = DPO_PROTO_IP4;
 491                     vnet_buffer(b2)->mpls.pyld_proto = DPO_PROTO_IP4;
 492                     vnet_buffer(b3)->mpls.pyld_proto = DPO_PROTO_IP4;
 493                 }
 494                 else if (DPO_PROTO_IP6 == dproto)
 495                 {
 496                     /*
 497                      * decrement the TTL on ingress to the LSP
 498                      */
 499                     ip6_header_t * ip0 = vlib_buffer_get_current(b0);
 500                     ip6_header_t * ip1 = vlib_buffer_get_current(b1);
 501                     ip6_header_t * ip2 = vlib_buffer_get_current(b2);
 502                     ip6_header_t * ip3 = vlib_buffer_get_current(b3);
 503
 504                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 505                     {
 506                         ip0->hop_limit -= 1;
 507                         ip1->hop_limit -= 1;
 508                         ip2->hop_limit -= 1;
 509                         ip3->hop_limit -= 1;
 510                     }
 511                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 512                     {
 513                         ttl0 = ip0->hop_limit;
 514                         ttl1 = ip1->hop_limit;
 515                         ttl2 = ip2->hop_limit;
 516                         ttl3 = ip3->hop_limit;
 517                         /* by default copy the 3 most significant bits */
 518                         exp0 = ip_dscp_to_mpls_exp(
 519                             ip6_traffic_class_network_order(ip0));
 520                         exp1 = ip_dscp_to_mpls_exp(
 521                             ip6_traffic_class_network_order(ip1));
 522                         exp2 = ip_dscp_to_mpls_exp(
 523                             ip6_traffic_class_network_order(ip2));
 524                         exp3 = ip_dscp_to_mpls_exp(
 525                             ip6_traffic_class_network_order(ip3));
 526                     }
 527
 528                     /* save the payload proto information in mpls opaque */
 529                     vnet_buffer(b0)->mpls.pyld_proto = DPO_PROTO_IP6;
 530                     vnet_buffer(b1)->mpls.pyld_proto = DPO_PROTO_IP6;
 531                     vnet_buffer(b2)->mpls.pyld_proto = DPO_PROTO_IP6;
 532                     vnet_buffer(b3)->mpls.pyld_proto = DPO_PROTO_IP6;
 533                 }
 534                 else
 535                 {
 536                     /*
 537                      * nothing to change in the ethernet header
 538                      */
 539                     ttl0 = ttl1 = ttl2 = ttl3 = MPLS_LABEL_DEFAULT_TTL;
 540                     exp0 = exp1 = exp2 = exp3 = MPLS_LABEL_DEFAULT_EXP;
 541                 }
 542                 /*
 543                  * These are the non-MPLS payload imposition cases.
 544                  * Based on the LSP mode either, for uniform, copy down the TTL
 545                  * and EXP from the payload or, for pipe mode, slap on the value
 546                  * requested from config
 547                  */
 548                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 549                 {
 550                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 551                     hdr1 = mpls_label_paint_w_ttl_exp(b1, mld1, ttl1, exp1);
 552                     hdr2 = mpls_label_paint_w_ttl_exp(b2, mld2, ttl2, exp2);
 553                     hdr3 = mpls_label_paint_w_ttl_exp(b3, mld3, ttl3, exp3);
 554                 }
 555                 else
 556                 {
 557                     hdr0 = mpls_label_paint(b0, mld0);
 558                     hdr1 = mpls_label_paint(b1, mld1);
 559                     hdr2 = mpls_label_paint(b2, mld2);
 560                     hdr3 = mpls_label_paint(b3, mld3);
 561                 }
 562             }
 563             else
 564             {
 565                 /*
 566                  * else, the packet to be encapped is an MPLS packet
 567                  * there are two cases to consider:
 568                  *  1 - this is an MPLS label swap at an LSP midpoint.
 569                  *      recognisable because mpls.first = 1. In this case the
 570                  *      TTL must be set to the current value -1.
 571                  *  2 - The MPLS packet is recursing (or being injected into)
 572                  *      this LSP, in which case the pipe/uniform rules apply
 573                  *
 574                  */
 575                 if (PREDICT_TRUE(vnet_buffer(b0)->mpls.first))
 576                 {
 577                     /*
 578                      * The first label to be imposed on the packet. this is a
 579                      * label swap.in which case we stashed the TTL and EXP bits
 580                      * in the packet in the lookup node
 581                      */
 582                     ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
 583
 584                     ttl0 = vnet_buffer(b0)->mpls.ttl - 1;
 585                     exp0 = vnet_buffer(b0)->mpls.exp;
 586                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 587                 }
 588                 else
 589                 {
 590                     /*
 591                      * not the first label. implying we are recusring down a
 592                      * chain of output labels. Each layer is considered a new
 593                      * LSP - hence the TTL/EXP are pipe/uniform handled
 594                      */
 595                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 596                     {
 597                         hdr0 = vlib_buffer_get_current(b0);
 598                         ttl0 = ((u8*)hdr0)[3];
 599                         exp0 = ((u8*)hdr0)[2] & 0xe;
 600                         hdr0 = mpls_label_paint_w_ttl_mpls_exp(b0, mld0, ttl0, exp0);
 601                     }
 602                     else
 603                     {
 604                         hdr0 = mpls_label_paint(b0, mld0);
 605                     }
 606                 }
 607                 if (PREDICT_TRUE(vnet_buffer(b1)->mpls.first))
 608                 {
 609                     ASSERT(0 != vnet_buffer (b1)->mpls.ttl);
 610
 611                     ttl1 = vnet_buffer(b1)->mpls.ttl - 1;
 612                     exp1 = vnet_buffer(b1)->mpls.exp;
 613                     hdr1 = mpls_label_paint_w_ttl_exp(b1, mld1, ttl1, exp1);
 614                 }
 615                 else
 616                 {
 617                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 618                     {
 619                         hdr1 = vlib_buffer_get_current(b1);
 620                         ttl1 = ((u8*)hdr1)[3];
 621                         exp1 = ((u8*)hdr1)[2] & 0xe;
 622                         hdr1 = mpls_label_paint_w_ttl_mpls_exp(b1, mld1, ttl1, exp1);
 623                     }
 624                     else
 625                     {
 626                         hdr1 = mpls_label_paint(b1, mld1);
 627                     }
 628                 }
 629                 if (PREDICT_TRUE(vnet_buffer(b2)->mpls.first))
 630                 {
 631                     ASSERT(0 != vnet_buffer (b2)->mpls.ttl);
 632
 633                     ttl2 = vnet_buffer(b2)->mpls.ttl - 1;
 634                     exp2 = vnet_buffer(b2)->mpls.exp;
 635                     hdr2 = mpls_label_paint_w_ttl_exp(b2, mld2, ttl2, exp2);
 636                 }
 637                 else
 638                 {
 639                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 640                     {
 641                         hdr2 = vlib_buffer_get_current(b2);
 642                         ttl2 = ((u8*)hdr2)[3];
 643                         exp2 = ((u8*)hdr2)[2] & 0xe;
 644                         hdr2 = mpls_label_paint_w_ttl_mpls_exp(b2, mld2, ttl2, exp2);
 645                     }
 646                     else
 647                     {
 648                         hdr2 = mpls_label_paint(b2, mld2);
 649                     }
 650                 }
 651                 if (PREDICT_TRUE(vnet_buffer(b3)->mpls.first))
 652                 {
 653                     ASSERT(0 != vnet_buffer (b3)->mpls.ttl);
 654
 655                     ttl3 = vnet_buffer(b3)->mpls.ttl - 1;
 656                     exp3 = vnet_buffer(b3)->mpls.exp;
 657                     hdr3 = mpls_label_paint_w_ttl_exp(b3, mld3, ttl3, exp3);
 658                 }
 659                 else
 660                 {
 661                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 662                     {
 663                         hdr3 = vlib_buffer_get_current(b3);
 664                         ttl3 = ((u8*)hdr3)[3];
 665                         exp3 = ((u8*)hdr3)[2] & 0xe;
 666                         hdr3 = mpls_label_paint_w_ttl_mpls_exp(b3, mld3, ttl3, exp3);
 667                     }
 668                     else
 669                     {
 670                         hdr3 = mpls_label_paint(b3, mld3);
 671                     }
 672                 }
 673
 674                 vnet_buffer(b0)->mpls.first = 0;
 675                 vnet_buffer(b1)->mpls.first = 0;
 676                 vnet_buffer(b2)->mpls.first = 0;
 677                 vnet_buffer(b3)->mpls.first = 0;
 678             }
 679
 680             next0 = mld0->mld_dpo.dpoi_next_node;
 681             next1 = mld1->mld_dpo.dpoi_next_node;
 682             next2 = mld2->mld_dpo.dpoi_next_node;
 683             next3 = mld3->mld_dpo.dpoi_next_node;
 684
 685             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
 686             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = mld1->mld_dpo.dpoi_index;
 687             vnet_buffer(b2)->ip.adj_index[VLIB_TX] = mld2->mld_dpo.dpoi_index;
 688             vnet_buffer(b3)->ip.adj_index[VLIB_TX] = mld3->mld_dpo.dpoi_index;
 689
 690             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 691             {
 692                 mpls_label_imposition_trace_t *tr =
 693                     vlib_add_trace (vm, node, b0, sizeof (*tr));
 694                 tr->hdr = *hdr0;
 695                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 696                 {
 697                     tr->ttl = ttl0;
 698                     tr->exp = exp0;
 699                 }
 700                 else
 701                 {
 702                     tr->ttl = tr->exp = 0;
 703                 }
 704             }
 705             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 706             {
 707                 mpls_label_imposition_trace_t *tr =
 708                     vlib_add_trace (vm, node, b1, sizeof (*tr));
 709                 tr->hdr = *hdr1;
 710                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 711                 {
 712                     tr->ttl = ttl1;
 713                     tr->exp = exp1;
 714                 }
 715                 else
 716                 {
 717                     tr->ttl = tr->exp = 0;
 718                 }
 719             }
 720             if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
 721             {
 722                 mpls_label_imposition_trace_t *tr =
 723                     vlib_add_trace (vm, node, b2, sizeof (*tr));
 724                 tr->hdr = *hdr2;
 725                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 726                 {
 727                     tr->ttl = ttl2;
 728                     tr->exp = exp2;
 729                 }
 730                 else
 731                 {
 732                     tr->ttl = tr->exp = 0;
 733                 }
 734             }
 735             if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
 736             {
 737                 mpls_label_imposition_trace_t *tr =
 738                     vlib_add_trace (vm, node, b3, sizeof (*tr));
 739                 tr->hdr = *hdr3;
 740                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 741                 {
 742                     tr->ttl = ttl3;
 743                     tr->exp = exp3;
 744                 }
 745                 else
 746                 {
 747                     tr->ttl = tr->exp = 0;
 748                 }
 749             }
 750
 751             vlib_validate_buffer_enqueue_x4(vm, node, next_index, to_next,
 752                                             n_left_to_next,
 753                                             bi0, bi1, bi2, bi3,
 754                                             next0, next1, next2, next3);
 755         }
 756
 757         while (n_left_from > 0 && n_left_to_next > 0)
 758         {
 759             mpls_unicast_header_t *hdr0;
 760             mpls_label_dpo_t *mld0;
 761             vlib_buffer_t * b0;
 762             u32 bi0, mldi0;
 763             u8 ttl0, exp0;
 764             u32 next0;
 765
 766             bi0 = from[0];
 767             to_next[0] = bi0;
 768             from += 1;
 769             to_next += 1;
 770             n_left_from -= 1;
 771             n_left_to_next -= 1;
 772
 773             b0 = vlib_get_buffer (vm, bi0);
 774
 775             /* dst lookup was done by ip4 lookup */
 776             mldi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 777             mld0 = mpls_label_dpo_get(mldi0);
 778
 779             if (DPO_PROTO_MPLS != dproto)
 780             {
 781                 if (DPO_PROTO_IP4 == dproto)
 782                 {
 783                     /*
 784                      * decrement the TTL on ingress to the LSP
 785                      */
 786                     ip4_header_t * ip0 = vlib_buffer_get_current(b0);
 787                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 788                     {
 789                         u32 checksum0;
 790
 791                         checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
 792                         checksum0 += checksum0 >= 0xffff;
 793
 794                         ip0->checksum = checksum0;
 795                         ip0->ttl -= 1;
 796                     }
 797                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 798                     {
 799                         ttl0 = ip0->ttl;
 800                         exp0 = ip_dscp_to_mpls_exp(ip0->tos);
 801                     }
 802
 803                     /* save the payload proto information in mpls opaque */
 804                     vnet_buffer(b0)->mpls.pyld_proto = DPO_PROTO_IP4;
 805                 }
 806                 else if (DPO_PROTO_IP6 == dproto)
 807                 {
 808                     /*
 809                      * decrement the TTL on ingress to the LSP
 810                      */
 811                     ip6_header_t * ip0 = vlib_buffer_get_current(b0);
 812
 813                     if (!(MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR & flags))
 814                     {
 815                         ip0->hop_limit -= 1;
 816                     }
 817                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 818                     {
 819                         ttl0 = ip0->hop_limit;
 820                         exp0 = ip_dscp_to_mpls_exp(
 821                             ip6_traffic_class_network_order(ip0));
 822                     }
 823
 824                     /* save the payload proto information in mpls opaque */
 825                     vnet_buffer(b0)->mpls.pyld_proto = DPO_PROTO_IP6;
 826                 }
 827                 else
 828                 {
 829                     /*
 830                      * nothing to change in the ethernet header
 831                      */
 832                     ttl0 = MPLS_LABEL_DEFAULT_TTL;
 833                     exp0 = MPLS_LABEL_DEFAULT_EXP;
 834                 }
 835
 836                 /*
 837                  * These are the non-MPLS payload imposition cases.
 838                  * Based on the LSP mode either, for uniform, copy down the TTL
 839                  * from the payload or, for pipe mode, slap on the value
 840                  * requested from config
 841                  */
 842                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 843                 {
 844                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 845                 }
 846                 else
 847                 {
 848                     hdr0 = mpls_label_paint(b0, mld0);
 849                 }
 850             }
 851             else
 852             {
 853                 if (PREDICT_TRUE(vnet_buffer(b0)->mpls.first))
 854                 {
 855                     ASSERT(0 != vnet_buffer (b0)->mpls.ttl);
 856
 857                     ttl0 = vnet_buffer(b0)->mpls.ttl - 1;
 858                     exp0 = vnet_buffer(b0)->mpls.exp;
 859                     hdr0 = mpls_label_paint_w_ttl_exp(b0, mld0, ttl0, exp0);
 860                 }
 861                 else
 862                 {
 863                     if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 864                     {
 865                         hdr0 = vlib_buffer_get_current(b0);
 866                         ttl0 = ((u8*)hdr0)[3];
 867                         exp0 = ((u8*)hdr0)[2] & 0xe;
 868                         hdr0 = mpls_label_paint_w_ttl_mpls_exp(b0, mld0, ttl0, exp0);
 869                     }
 870                     else
 871                     {
 872                         hdr0 = mpls_label_paint(b0, mld0);
 873                     }
 874                 }
 875
 876                 vnet_buffer(b0)->mpls.first = 0;
 877             }
 878
 879             next0 = mld0->mld_dpo.dpoi_next_node;
 880             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mld0->mld_dpo.dpoi_index;
 881
 882             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 883             {
 884                 mpls_label_imposition_trace_t *tr =
 885                     vlib_add_trace (vm, node, b0, sizeof (*tr));
 886                 tr->hdr = *hdr0;
 887                 if (flags & MPLS_LABEL_DPO_FLAG_UNIFORM_MODE)
 888                 {
 889                     tr->ttl = ttl0;
 890                     tr->exp = exp0;
 891                 }
 892                 else
 893                 {
 894                     tr->ttl = tr->exp = 0;
 895                 }
 896            }
 897
 898             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 899                                             n_left_to_next, bi0, next0);
 900         }
 901         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 902     }
 903     return from_frame->n_vectors;
 904 }
 905
 906 static u8 *
 907 format_mpls_label_imposition_trace (u8 * s, va_list * args)
 908 {
 909     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 910     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 911     mpls_label_imposition_trace_t * t;
 912     mpls_unicast_header_t hdr;
 913     u32 indent;
 914
 915     t = va_arg (*args, mpls_label_imposition_trace_t *);
 916     indent = format_get_indent (s);
 917     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
 918
 919     s = format (s, "%Umpls-header:%U",
 920                 format_white_space, indent,
 921                 format_mpls_header, hdr);
 922     return (s);
 923 }
 924
 925 VLIB_NODE_FN (mpls_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 926                                  vlib_node_runtime_t * node,
 927                                  vlib_frame_t * frame)
 928 {
 929     return (mpls_label_imposition_inline(vm, node, frame,
 930                                          DPO_PROTO_MPLS,
 931                                          MPLS_LABEL_DPO_FLAG_NONE));
 932 }
 933
 934 VLIB_REGISTER_NODE (mpls_mpls_label_imposition_pipe_node) = {
 935     .name = "mpls-label-imposition-pipe",
 936     .vector_size = sizeof (u32),
 937
 938     .format_trace = format_mpls_label_imposition_trace,
 939     .n_next_nodes = 1,
 940     .next_nodes = {
 941         [0] = "mpls-drop",
 942     }
 943 };
 944
 945 VLIB_NODE_FN (ip4_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 946                                 vlib_node_runtime_t * node,
 947                                 vlib_frame_t * frame)
 948 {
 949     return (mpls_label_imposition_inline(vm, node, frame,
 950                                          DPO_PROTO_IP4,
 951                                          MPLS_LABEL_DPO_FLAG_NONE));
 952 }
 953
 954 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_pipe_node) = {
 955     .name = "ip4-mpls-label-imposition-pipe",
 956     .vector_size = sizeof (u32),
 957
 958     .format_trace = format_mpls_label_imposition_trace,
 959     .n_next_nodes = 1,
 960     .next_nodes = {
 961         [0] = "ip4-drop",
 962     }
 963 };
 964
 965 VLIB_NODE_FN (ip6_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 966                                 vlib_node_runtime_t * node,
 967                                 vlib_frame_t * frame)
 968 {
 969     return (mpls_label_imposition_inline(vm, node, frame,
 970                                          DPO_PROTO_IP6,
 971                                          MPLS_LABEL_DPO_FLAG_NONE));
 972 }
 973
 974 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_pipe_node) = {
 975     .name = "ip6-mpls-label-imposition-pipe",
 976     .vector_size = sizeof (u32),
 977
 978     .format_trace = format_mpls_label_imposition_trace,
 979     .n_next_nodes = 1,
 980     .next_nodes = {
 981         [0] = "ip6-drop",
 982     }
 983 };
 984
 985 VLIB_NODE_FN (ethernet_mpls_label_imposition_pipe_node) (vlib_main_t * vm,
 986                                      vlib_node_runtime_t * node,
 987                                      vlib_frame_t * frame)
 988 {
 989     return (mpls_label_imposition_inline(vm, node, frame,
 990                                          DPO_PROTO_ETHERNET,
 991                                          MPLS_LABEL_DPO_FLAG_NONE));
 992 }
 993
 994 VLIB_REGISTER_NODE (ethernet_mpls_label_imposition_pipe_node) = {
 995     .name = "ethernet-mpls-label-imposition-pipe",
 996     .vector_size = sizeof (u32),
 997
 998     .format_trace = format_mpls_label_imposition_trace,
 999     .n_next_nodes = 1,
1000     .next_nodes = {
1001         [0] = "error-drop",
1002     }
1003 };
1004
1005 VLIB_NODE_FN (mpls_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
1006                                     vlib_node_runtime_t * node,
1007                                     vlib_frame_t * frame)
1008 {
1009     return (mpls_label_imposition_inline(vm, node, frame,
1010                                          DPO_PROTO_MPLS,
1011                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
1012 }
1013
1014 VLIB_REGISTER_NODE (mpls_mpls_label_imposition_uniform_node) = {
1015     .name = "mpls-label-imposition-uniform",
1016     .vector_size = sizeof (u32),
1017
1018     .format_trace = format_mpls_label_imposition_trace,
1019     .n_next_nodes = 1,
1020     .next_nodes = {
1021         [0] = "mpls-drop",
1022     }
1023 };
1024
1025 VLIB_NODE_FN (ip4_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
1026                                    vlib_node_runtime_t * node,
1027                                    vlib_frame_t * frame)
1028 {
1029     return (mpls_label_imposition_inline(vm, node, frame,
1030                                          DPO_PROTO_IP4,
1031                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
1032 }
1033
1034 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_uniform_node) = {
1035     .name = "ip4-mpls-label-imposition-uniform",
1036     .vector_size = sizeof (u32),
1037
1038     .format_trace = format_mpls_label_imposition_trace,
1039     .n_next_nodes = 1,
1040     .next_nodes = {
1041         [0] = "ip4-drop",
1042     }
1043 };
1044
1045 VLIB_NODE_FN (ip6_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
1046                                    vlib_node_runtime_t * node,
1047                                    vlib_frame_t * frame)
1048 {
1049     return (mpls_label_imposition_inline(vm, node, frame,
1050                                          DPO_PROTO_IP6,
1051                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
1052 }
1053
1054 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_uniform_node) = {
1055     .name = "ip6-mpls-label-imposition-uniform",
1056     .vector_size = sizeof (u32),
1057
1058     .format_trace = format_mpls_label_imposition_trace,
1059     .n_next_nodes = 1,
1060     .next_nodes = {
1061         [0] = "ip6-drop",
1062     }
1063 };
1064
1065 VLIB_NODE_FN (ethernet_mpls_label_imposition_uniform_node) (vlib_main_t * vm,
1066                                         vlib_node_runtime_t * node,
1067                                         vlib_frame_t * frame)
1068 {
1069     return (mpls_label_imposition_inline(vm, node, frame,
1070                                          DPO_PROTO_ETHERNET,
1071                                          MPLS_LABEL_DPO_FLAG_UNIFORM_MODE));
1072 }
1073
1074 VLIB_REGISTER_NODE (ethernet_mpls_label_imposition_uniform_node) = {
1075     .name = "ethernet-mpls-label-imposition-uniform",
1076     .vector_size = sizeof (u32),
1077
1078     .format_trace = format_mpls_label_imposition_trace,
1079     .n_next_nodes = 1,
1080     .next_nodes = {
1081         [0] = "error-drop",
1082     }
1083 };
1084
1085
1086 VLIB_NODE_FN (ip4_mpls_label_imposition_pipe_no_ip_ttl_decr_node) (vlib_main_t * vm,
1087                                                vlib_node_runtime_t * node,
1088                                                vlib_frame_t * frame)
1089 {
1090     return (mpls_label_imposition_inline(vm, node, frame,
1091                                          DPO_PROTO_IP4,
1092                                          MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR));
1093 }
1094
1095 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_pipe_no_ip_ttl_decr_node) = {
1096     .name = "ip4-mpls-label-imposition-pipe-no-ip-ttl-decr",
1097     .vector_size = sizeof (u32),
1098
1099     .format_trace = format_mpls_label_imposition_trace,
1100     .n_next_nodes = 1,
1101     .next_nodes = {
1102         [0] = "ip4-drop",
1103     }
1104 };
1105
1106 VLIB_NODE_FN (ip6_mpls_label_imposition_pipe_no_ip_ttl_decr_node) (vlib_main_t * vm,
1107                                                vlib_node_runtime_t * node,
1108                                                vlib_frame_t * frame)
1109 {
1110     return (mpls_label_imposition_inline(vm, node, frame,
1111                                          DPO_PROTO_IP6,
1112                                          MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR));
1113 }
1114
1115 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_pipe_no_ip_ttl_decr_node) = {
1116     .name = "ip6-mpls-label-imposition-pipe-no-ip-ttl-decr",
1117     .vector_size = sizeof (u32),
1118
1119     .format_trace = format_mpls_label_imposition_trace,
1120     .n_next_nodes = 1,
1121     .next_nodes = {
1122         [0] = "ip6-drop",
1123     }
1124 };
1125
1126 VLIB_NODE_FN (ip4_mpls_label_imposition_uniform_no_ip_ttl_decr_node) (vlib_main_t * vm,
1127                                                   vlib_node_runtime_t * node,
1128                                                   vlib_frame_t * frame)
1129 {
1130     return (mpls_label_imposition_inline(vm, node, frame,
1131                                          DPO_PROTO_IP4,
1132                                          (MPLS_LABEL_DPO_FLAG_UNIFORM_MODE |
1133                                           MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR)));
1134 }
1135
1136 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_uniform_no_ip_ttl_decr_node) = {
1137     .name = "ip4-mpls-label-imposition-uniform-no-ip-ttl-decr",
1138     .vector_size = sizeof (u32),
1139
1140     .format_trace = format_mpls_label_imposition_trace,
1141     .n_next_nodes = 1,
1142     .next_nodes = {
1143         [0] = "ip4-drop",
1144     }
1145 };
1146
1147 VLIB_NODE_FN (ip6_mpls_label_imposition_uniform_no_ip_ttl_decr_node) (vlib_main_t * vm,
1148                                                   vlib_node_runtime_t * node,
1149                                                   vlib_frame_t * frame)
1150 {
1151     return (mpls_label_imposition_inline(vm, node, frame,
1152                                          DPO_PROTO_IP6,
1153                                          (MPLS_LABEL_DPO_FLAG_UNIFORM_MODE |
1154                                           MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR)));
1155 }
1156
1157 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_uniform_no_ip_ttl_decr_node) = {
1158     .name = "ip6-mpls-label-imposition-uniform-no-ip-ttl-decr",
1159     .vector_size = sizeof (u32),
1160
1161     .format_trace = format_mpls_label_imposition_trace,
1162     .n_next_nodes = 1,
1163     .next_nodes = {
1164         [0] = "ip6-drop",
1165     }
1166 };
1167
1168
1169 #ifndef CLIB_MARCH_VARIANT
1170 static void
1171 mpls_label_dpo_mem_show (void)
1172 {
1173     fib_show_memory_usage("MPLS label",
1174                           pool_elts(mpls_label_dpo_pool),
1175                           pool_len(mpls_label_dpo_pool),
1176                           sizeof(mpls_label_dpo_t));
1177 }
1178
1179 /**
1180  * Interpose a label DPO. used in the FIB unit tests
1181  */
1182 static void
1183 mpls_label_interpose (const dpo_id_t *original,
1184                       const dpo_id_t *parent,
1185                       dpo_id_t *clone)
1186 {
1187     mpls_label_dpo_t *mld, *mld_clone;
1188
1189     mld_clone = mpls_label_dpo_alloc();
1190     mld = mpls_label_dpo_get(original->dpoi_index);
1191
1192     mld_clone->mld_locks = 0;
1193     clib_memcpy_fast(&mld_clone->mld_hdr,
1194                 &mld->mld_hdr,
1195                 sizeof(mld_clone->mld_hdr));
1196     mld_clone->mld_payload_proto = mld->mld_payload_proto;
1197     mld_clone->mld_n_labels = mld->mld_n_labels;
1198     mld_clone->mld_n_hdr_bytes = mld->mld_n_hdr_bytes;
1199
1200     dpo_stack(mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NONE],
1201               mld_clone->mld_payload_proto,
1202               &mld_clone->mld_dpo,
1203               parent);
1204
1205     dpo_set(clone,
1206             mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NONE],
1207             mld_clone->mld_payload_proto,
1208             mpls_label_dpo_get_index(mld_clone));
1209 }
1210
1211 const static dpo_vft_t mld_vft = {
1212     .dv_lock = mpls_label_dpo_lock,
1213     .dv_unlock = mpls_label_dpo_unlock,
1214     .dv_format = format_mpls_label_dpo,
1215     .dv_mem_show = mpls_label_dpo_mem_show,
1216     .dv_mk_interpose = mpls_label_interpose,
1217 };
1218
1219 const static char* const mpls_label_imp_pipe_ip4_nodes[] =
1220 {
1221     "ip4-mpls-label-imposition-pipe",
1222     NULL,
1223 };
1224 const static char* const mpls_label_imp_pipe_ip6_nodes[] =
1225 {
1226     "ip6-mpls-label-imposition-pipe",
1227     NULL,
1228 };
1229 const static char* const mpls_label_imp_pipe_mpls_nodes[] =
1230 {
1231     "mpls-label-imposition-pipe",
1232     NULL,
1233 };
1234 const static char* const mpls_label_imp_pipe_ethernet_nodes[] =
1235 {
1236     "ethernet-mpls-label-imposition-pipe",
1237     NULL,
1238 };
1239
1240 const static char* const * const mpls_label_imp_pipe_nodes[DPO_PROTO_NUM] =
1241 {
1242     [DPO_PROTO_IP4]  = mpls_label_imp_pipe_ip4_nodes,
1243     [DPO_PROTO_IP6]  = mpls_label_imp_pipe_ip6_nodes,
1244     [DPO_PROTO_MPLS] = mpls_label_imp_pipe_mpls_nodes,
1245     [DPO_PROTO_ETHERNET] = mpls_label_imp_pipe_ethernet_nodes,
1246 };
1247
1248 const static char* const mpls_label_imp_uniform_ip4_nodes[] =
1249 {
1250     "ip4-mpls-label-imposition-uniform",
1251     NULL,
1252 };
1253 const static char* const mpls_label_imp_uniform_ip6_nodes[] =
1254 {
1255     "ip6-mpls-label-imposition-uniform",
1256     NULL,
1257 };
1258 const static char* const mpls_label_imp_uniform_mpls_nodes[] =
1259 {
1260     "mpls-label-imposition-uniform",
1261     NULL,
1262 };
1263 const static char* const mpls_label_imp_uniform_ethernet_nodes[] =
1264 {
1265     "ethernet-mpls-label-imposition-uniform",
1266     NULL,
1267 };
1268
1269 const static char* const * const mpls_label_imp_uniform_nodes[DPO_PROTO_NUM] =
1270 {
1271     [DPO_PROTO_IP4]  = mpls_label_imp_uniform_ip4_nodes,
1272     [DPO_PROTO_IP6]  = mpls_label_imp_uniform_ip6_nodes,
1273     [DPO_PROTO_MPLS] = mpls_label_imp_uniform_mpls_nodes,
1274     [DPO_PROTO_ETHERNET] = mpls_label_imp_uniform_ethernet_nodes,
1275 };
1276
1277 const static char* const mpls_label_imp_pipe_no_ip_tll_decr_ip4_nodes[] =
1278 {
1279     "ip4-mpls-label-imposition-pipe-no-ip-ttl-decr",
1280     NULL,
1281 };
1282 const static char* const mpls_label_imp_pipe_no_ip_tll_decr_ip6_nodes[] =
1283 {
1284     "ip6-mpls-label-imposition-pipe-no-ip-ttl-decr",
1285     NULL,
1286 };
1287
1288 const static char* const * const mpls_label_imp_pipe_no_ip_tll_decr_nodes[DPO_PROTO_NUM] =
1289 {
1290     [DPO_PROTO_IP4]  = mpls_label_imp_pipe_no_ip_tll_decr_ip4_nodes,
1291     [DPO_PROTO_IP6]  = mpls_label_imp_pipe_no_ip_tll_decr_ip6_nodes,
1292 };
1293
1294 const static char* const mpls_label_imp_uniform_no_ip_tll_decr_ip4_nodes[] =
1295 {
1296     "ip4-mpls-label-imposition-uniform-no-ip-ttl-decr",
1297     NULL,
1298 };
1299 const static char* const mpls_label_imp_uniform_no_ip_tll_decr_ip6_nodes[] =
1300 {
1301     "ip6-mpls-label-imposition-uniform-no-ip-ttl-decr",
1302     NULL,
1303 };
1304
1305 const static char* const * const mpls_label_imp_uniform_no_ip_tll_decr_nodes[DPO_PROTO_NUM] =
1306 {
1307     [DPO_PROTO_IP4]  = mpls_label_imp_uniform_no_ip_tll_decr_ip4_nodes,
1308     [DPO_PROTO_IP6]  = mpls_label_imp_uniform_no_ip_tll_decr_ip6_nodes,
1309 };
1310
1311 void
1312 mpls_label_dpo_module_init (void)
1313 {
1314     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NONE] =
1315         dpo_register_new_type(&mld_vft,
1316                               mpls_label_imp_pipe_nodes);
1317     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR] =
1318         dpo_register_new_type(&mld_vft,
1319                               mpls_label_imp_pipe_no_ip_tll_decr_nodes);
1320     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_UNIFORM_MODE] =
1321         dpo_register_new_type(&mld_vft,
1322                               mpls_label_imp_uniform_nodes);
1323     mpls_label_dpo_types[MPLS_LABEL_DPO_FLAG_UNIFORM_MODE |
1324                          MPLS_LABEL_DPO_FLAG_NO_IP_TTL_DECR] =
1325         dpo_register_new_type(&mld_vft,
1326                               mpls_label_imp_uniform_no_ip_tll_decr_nodes);
1327 }
1328
1329 dpo_type_t
1330 mpls_label_dpo_get_type (mpls_label_dpo_flags_t flags)
1331 {
1332     return (mpls_label_dpo_types[flags]);
1333 }
1334 #endif /* CLIB_MARCH_VARIANT */