src/vnet/map/ip4_map_t.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 #include "map.h"
  16
  17 #include "../ip/ip_frag.h"
  18
  19 #define IP4_MAP_T_DUAL_LOOP 1
  20
  21 typedef enum
  22 {
  23   IP4_MAPT_NEXT_MAPT_TCP_UDP,
  24   IP4_MAPT_NEXT_MAPT_ICMP,
  25   IP4_MAPT_NEXT_MAPT_FRAGMENTED,
  26   IP4_MAPT_NEXT_DROP,
  27   IP4_MAPT_N_NEXT
  28 } ip4_mapt_next_t;
  29
  30 typedef enum
  31 {
  32   IP4_MAPT_ICMP_NEXT_IP6_LOOKUP,
  33   IP4_MAPT_ICMP_NEXT_IP6_FRAG,
  34   IP4_MAPT_ICMP_NEXT_DROP,
  35   IP4_MAPT_ICMP_N_NEXT
  36 } ip4_mapt_icmp_next_t;
  37
  38 typedef enum
  39 {
  40   IP4_MAPT_TCP_UDP_NEXT_IP6_LOOKUP,
  41   IP4_MAPT_TCP_UDP_NEXT_IP6_FRAG,
  42   IP4_MAPT_TCP_UDP_NEXT_DROP,
  43   IP4_MAPT_TCP_UDP_N_NEXT
  44 } ip4_mapt_tcp_udp_next_t;
  45
  46 typedef enum
  47 {
  48   IP4_MAPT_FRAGMENTED_NEXT_IP6_LOOKUP,
  49   IP4_MAPT_FRAGMENTED_NEXT_IP6_FRAG,
  50   IP4_MAPT_FRAGMENTED_NEXT_DROP,
  51   IP4_MAPT_FRAGMENTED_N_NEXT
  52 } ip4_mapt_fragmented_next_t;
  53
  54 //This is used to pass information within the buffer data.
  55 //Buffer structure being too small to contain big structures like this.
  56 /* *INDENT-OFF* */
  57 typedef CLIB_PACKED (struct {
  58   ip6_address_t daddr;
  59   ip6_address_t saddr;
  60   //IPv6 header + Fragmentation header will be here
  61   //sizeof(ip6) + sizeof(ip_frag) - sizeof(ip4)
  62   u8 unused[28];
  63 }) ip4_mapt_pseudo_header_t;
  64 /* *INDENT-ON* */
  65
  66 #define frag_id_4to6(id) (id)
  67
  68 //TODO: Find the right place in memory for this.
  69 /* *INDENT-OFF* */
  70 static u8 icmp_to_icmp6_updater_pointer_table[] =
  71   { 0, 1, 4, 4, ~0,
  72     ~0, ~0, ~0, 7, 6,
  73     ~0, ~0, 8, 8, 8,
  74     8, 24, 24, 24, 24
  75   };
  76 /* *INDENT-ON* */
  77
  78
  79 static_always_inline int
  80 ip4_map_fragment_cache (ip4_header_t * ip4, u16 port)
  81 {
  82   u32 *ignore = NULL;
  83   map_ip4_reass_lock ();
  84   map_ip4_reass_t *r =
  85     map_ip4_reass_get (ip4->src_address.as_u32, ip4->dst_address.as_u32,
  86                        ip4->fragment_id,
  87                        (ip4->protocol ==
  88                         IP_PROTOCOL_ICMP) ? IP_PROTOCOL_ICMP6 : ip4->protocol,
  89                        &ignore);
  90   if (r)
  91     r->port = port;
  92
  93   map_ip4_reass_unlock ();
  94   return !r;
  95 }
  96
  97 static_always_inline i32
  98 ip4_map_fragment_get_port (ip4_header_t * ip4)
  99 {
 100   u32 *ignore = NULL;
 101   map_ip4_reass_lock ();
 102   map_ip4_reass_t *r =
 103     map_ip4_reass_get (ip4->src_address.as_u32, ip4->dst_address.as_u32,
 104                        ip4->fragment_id,
 105                        (ip4->protocol ==
 106                         IP_PROTOCOL_ICMP) ? IP_PROTOCOL_ICMP6 : ip4->protocol,
 107                        &ignore);
 108   i32 ret = r ? r->port : -1;
 109   map_ip4_reass_unlock ();
 110   return ret;
 111 }
 112
 113
 114 /* Statelessly translates an ICMP packet into ICMPv6.
 115  *
 116  * Warning: The checksum will need to be recomputed.
 117  *
 118  */
 119 static_always_inline int
 120 ip4_icmp_to_icmp6_in_place (icmp46_header_t * icmp, u32 icmp_len,
 121                             i32 * receiver_port, ip4_header_t ** inner_ip4)
 122 {
 123   *inner_ip4 = NULL;
 124   switch (icmp->type)
 125     {
 126     case ICMP4_echo_reply:
 127       *receiver_port = ((u16 *) icmp)[2];
 128       icmp->type = ICMP6_echo_reply;
 129       break;
 130     case ICMP4_echo_request:
 131       *receiver_port = ((u16 *) icmp)[2];
 132       icmp->type = ICMP6_echo_request;
 133       break;
 134     case ICMP4_destination_unreachable:
 135       *inner_ip4 = (ip4_header_t *) (((u8 *) icmp) + 8);
 136       *receiver_port = ip4_get_port (*inner_ip4, MAP_SENDER, icmp_len - 8);
 137
 138       switch (icmp->code)
 139         {
 140         case ICMP4_destination_unreachable_destination_unreachable_net: //0
 141         case ICMP4_destination_unreachable_destination_unreachable_host:        //1
 142           icmp->type = ICMP6_destination_unreachable;
 143           icmp->code = ICMP6_destination_unreachable_no_route_to_destination;
 144           break;
 145         case ICMP4_destination_unreachable_protocol_unreachable:        //2
 146           icmp->type = ICMP6_parameter_problem;
 147           icmp->code = ICMP6_parameter_problem_unrecognized_next_header;
 148           break;
 149         case ICMP4_destination_unreachable_port_unreachable:    //3
 150           icmp->type = ICMP6_destination_unreachable;
 151           icmp->code = ICMP6_destination_unreachable_port_unreachable;
 152           break;
 153         case ICMP4_destination_unreachable_fragmentation_needed_and_dont_fragment_set:  //4
 154           icmp->type =
 155             ICMP6_packet_too_big;
 156           icmp->code = 0;
 157           {
 158             u32 advertised_mtu = clib_net_to_host_u32 (*((u32 *) (icmp + 1)));
 159             if (advertised_mtu)
 160               advertised_mtu += 20;
 161             else
 162               advertised_mtu = 1000;    //FIXME ! (RFC 1191 - plateau value)
 163
 164             //FIXME: = minimum(advertised MTU+20, MTU_of_IPv6_nexthop, (MTU_of_IPv4_nexthop)+20)
 165             *((u32 *) (icmp + 1)) = clib_host_to_net_u32 (advertised_mtu);
 166           }
 167           break;
 168
 169         case ICMP4_destination_unreachable_source_route_failed: //5
 170         case ICMP4_destination_unreachable_destination_network_unknown: //6
 171         case ICMP4_destination_unreachable_destination_host_unknown:    //7
 172         case ICMP4_destination_unreachable_source_host_isolated:        //8
 173         case ICMP4_destination_unreachable_network_unreachable_for_type_of_service:     //11
 174         case ICMP4_destination_unreachable_host_unreachable_for_type_of_service:        //12
 175           icmp->type =
 176             ICMP6_destination_unreachable;
 177           icmp->code = ICMP6_destination_unreachable_no_route_to_destination;
 178           break;
 179         case ICMP4_destination_unreachable_network_administratively_prohibited: //9
 180         case ICMP4_destination_unreachable_host_administratively_prohibited:    //10
 181         case ICMP4_destination_unreachable_communication_administratively_prohibited:   //13
 182         case ICMP4_destination_unreachable_precedence_cutoff_in_effect: //15
 183           icmp->type = ICMP6_destination_unreachable;
 184           icmp->code =
 185             ICMP6_destination_unreachable_destination_administratively_prohibited;
 186           break;
 187         case ICMP4_destination_unreachable_host_precedence_violation:   //14
 188         default:
 189           return -1;
 190         }
 191       break;
 192
 193     case ICMP4_time_exceeded:   //11
 194       *inner_ip4 = (ip4_header_t *) (((u8 *) icmp) + 8);
 195       *receiver_port = ip4_get_port (*inner_ip4, MAP_SENDER, icmp_len - 8);
 196       icmp->type = ICMP6_time_exceeded;
 197       //icmp->code = icmp->code //unchanged
 198       break;
 199
 200     case ICMP4_parameter_problem:
 201       *inner_ip4 = (ip4_header_t *) (((u8 *) icmp) + 8);
 202       *receiver_port = ip4_get_port (*inner_ip4, MAP_SENDER, icmp_len - 8);
 203
 204       switch (icmp->code)
 205         {
 206         case ICMP4_parameter_problem_pointer_indicates_error:
 207         case ICMP4_parameter_problem_bad_length:
 208           icmp->type = ICMP6_parameter_problem;
 209           icmp->code = ICMP6_parameter_problem_erroneous_header_field;
 210           {
 211             u8 ptr =
 212               icmp_to_icmp6_updater_pointer_table[*((u8 *) (icmp + 1))];
 213             if (ptr == 0xff)
 214               return -1;
 215
 216             *((u32 *) (icmp + 1)) = clib_host_to_net_u32 (ptr);
 217           }
 218           break;
 219         default:
 220           //All other codes cause dropping the packet
 221           return -1;
 222         }
 223       break;
 224
 225     default:
 226       //All other types cause dropping the packet
 227       return -1;
 228       break;
 229     }
 230   return 0;
 231 }
 232
 233 static_always_inline void
 234 _ip4_map_t_icmp (map_domain_t * d, vlib_buffer_t * p, u8 * error)
 235 {
 236   ip4_header_t *ip4, *inner_ip4;
 237   ip6_header_t *ip6, *inner_ip6;
 238   u32 ip_len;
 239   icmp46_header_t *icmp;
 240   i32 recv_port;
 241   ip_csum_t csum;
 242   u16 *inner_L4_checksum = 0;
 243   ip6_frag_hdr_t *inner_frag;
 244   u32 inner_frag_id;
 245   u32 inner_frag_offset;
 246   u8 inner_frag_more;
 247
 248   ip4 = vlib_buffer_get_current (p);
 249   ip_len = clib_net_to_host_u16 (ip4->length);
 250   ASSERT (ip_len <= p->current_length);
 251
 252   icmp = (icmp46_header_t *) (ip4 + 1);
 253   if (ip4_icmp_to_icmp6_in_place (icmp, ip_len - sizeof (*ip4),
 254                                   &recv_port, &inner_ip4))
 255     {
 256       *error = MAP_ERROR_ICMP;
 257       return;
 258     }
 259
 260   if (recv_port < 0)
 261     {
 262       // In case of 1:1 mapping, we don't care about the port
 263       if (d->ea_bits_len == 0 && d->rules)
 264         {
 265           recv_port = 0;
 266         }
 267       else
 268         {
 269           *error = MAP_ERROR_ICMP;
 270           return;
 271         }
 272     }
 273
 274   if (inner_ip4)
 275     {
 276       //We have 2 headers to translate.
 277       //We need to make some room in the middle of the packet
 278
 279       if (PREDICT_FALSE (ip4_is_fragment (inner_ip4)))
 280         {
 281           //Here it starts getting really tricky
 282           //We will add a fragmentation header in the inner packet
 283
 284           if (!ip4_is_first_fragment (inner_ip4))
 285             {
 286               //For now we do not handle unless it is the first fragment
 287               //Ideally we should handle the case as we are in slow path already
 288               *error = MAP_ERROR_FRAGMENTED;
 289               return;
 290             }
 291
 292           vlib_buffer_advance (p,
 293                                -2 * (sizeof (*ip6) - sizeof (*ip4)) -
 294                                sizeof (*inner_frag));
 295           ip6 = vlib_buffer_get_current (p);
 296           clib_memcpy (u8_ptr_add (ip6, sizeof (*ip6) - sizeof (*ip4)), ip4,
 297                        20 + 8);
 298           ip4 =
 299             (ip4_header_t *) u8_ptr_add (ip6, sizeof (*ip6) - sizeof (*ip4));
 300           icmp = (icmp46_header_t *) (ip4 + 1);
 301
 302           inner_ip6 =
 303             (ip6_header_t *) u8_ptr_add (inner_ip4,
 304                                          sizeof (*ip4) - sizeof (*ip6) -
 305                                          sizeof (*inner_frag));
 306           inner_frag =
 307             (ip6_frag_hdr_t *) u8_ptr_add (inner_ip6, sizeof (*inner_ip6));
 308           ip6->payload_length =
 309             u16_net_add (ip4->length,
 310                          sizeof (*ip6) - 2 * sizeof (*ip4) +
 311                          sizeof (*inner_frag));
 312           inner_frag_id = frag_id_4to6 (inner_ip4->fragment_id);
 313           inner_frag_offset = ip4_get_fragment_offset (inner_ip4);
 314           inner_frag_more =
 315             ! !(inner_ip4->flags_and_fragment_offset &
 316                 clib_net_to_host_u16 (IP4_HEADER_FLAG_MORE_FRAGMENTS));
 317         }
 318       else
 319         {
 320           vlib_buffer_advance (p, -2 * (sizeof (*ip6) - sizeof (*ip4)));
 321           ip6 = vlib_buffer_get_current (p);
 322           clib_memcpy (u8_ptr_add (ip6, sizeof (*ip6) - sizeof (*ip4)), ip4,
 323                        20 + 8);
 324           ip4 =
 325             (ip4_header_t *) u8_ptr_add (ip6, sizeof (*ip6) - sizeof (*ip4));
 326           icmp = (icmp46_header_t *) u8_ptr_add (ip4, sizeof (*ip4));
 327           inner_ip6 =
 328             (ip6_header_t *) u8_ptr_add (inner_ip4,
 329                                          sizeof (*ip4) - sizeof (*ip6));
 330           ip6->payload_length =
 331             u16_net_add (ip4->length, sizeof (*ip6) - 2 * sizeof (*ip4));
 332           inner_frag = NULL;
 333         }
 334
 335       if (PREDICT_TRUE (inner_ip4->protocol == IP_PROTOCOL_TCP))
 336         {
 337           inner_L4_checksum = &((tcp_header_t *) (inner_ip4 + 1))->checksum;
 338           *inner_L4_checksum =
 339             ip_csum_fold (ip_csum_sub_even
 340                           (*inner_L4_checksum,
 341                            *((u64 *) (&inner_ip4->src_address))));
 342         }
 343       else if (PREDICT_TRUE (inner_ip4->protocol == IP_PROTOCOL_UDP))
 344         {
 345           inner_L4_checksum = &((udp_header_t *) (inner_ip4 + 1))->checksum;
 346           if (!*inner_L4_checksum)
 347             {
 348               //The inner packet was first translated, and therefore came from IPv6.
 349               //As the packet was an IPv6 packet, the UDP checksum can't be NULL
 350               *error = MAP_ERROR_ICMP;
 351               return;
 352             }
 353           *inner_L4_checksum =
 354             ip_csum_fold (ip_csum_sub_even
 355                           (*inner_L4_checksum,
 356                            *((u64 *) (&inner_ip4->src_address))));
 357         }
 358       else if (inner_ip4->protocol == IP_PROTOCOL_ICMP)
 359         {
 360           //We have an ICMP inside an ICMP
 361           //It needs to be translated, but not for error ICMP messages
 362           icmp46_header_t *inner_icmp = (icmp46_header_t *) (inner_ip4 + 1);
 363           csum = inner_icmp->checksum;
 364           //Only types ICMP4_echo_request and ICMP4_echo_reply are handled by ip4_icmp_to_icmp6_in_place
 365           csum = ip_csum_sub_even (csum, *((u16 *) inner_icmp));
 366           inner_icmp->type = (inner_icmp->type == ICMP4_echo_request) ?
 367             ICMP6_echo_request : ICMP6_echo_reply;
 368           csum = ip_csum_add_even (csum, *((u16 *) inner_icmp));
 369           csum =
 370             ip_csum_add_even (csum, clib_host_to_net_u16 (IP_PROTOCOL_ICMP6));
 371           csum =
 372             ip_csum_add_even (csum, inner_ip4->length - sizeof (*inner_ip4));
 373           inner_icmp->checksum = ip_csum_fold (csum);
 374           inner_L4_checksum = &inner_icmp->checksum;
 375           inner_ip4->protocol = IP_PROTOCOL_ICMP6;
 376         }
 377       else
 378         {
 379           /* To shut up Coverity */
 380           os_panic ();
 381         }
 382
 383       //FIXME: Security check with the port found in the inner packet
 384
 385       csum = *inner_L4_checksum;        //Initial checksum of the inner L4 header
 386       //FIXME: Shouldn't we remove ip addresses from there ?
 387
 388       inner_ip6->ip_version_traffic_class_and_flow_label =
 389         clib_host_to_net_u32 ((6 << 28) + (inner_ip4->tos << 20));
 390       inner_ip6->payload_length =
 391         u16_net_add (inner_ip4->length, -sizeof (*inner_ip4));
 392       inner_ip6->hop_limit = inner_ip4->ttl;
 393       inner_ip6->protocol = inner_ip4->protocol;
 394
 395       //Note that the source address is within the domain
 396       //while the destination address is the one outside the domain
 397       ip4_map_t_embedded_address (d, &inner_ip6->dst_address,
 398                                   &inner_ip4->dst_address);
 399       inner_ip6->src_address.as_u64[0] =
 400         map_get_pfx_net (d, inner_ip4->src_address.as_u32, recv_port);
 401       inner_ip6->src_address.as_u64[1] =
 402         map_get_sfx_net (d, inner_ip4->src_address.as_u32, recv_port);
 403
 404       if (PREDICT_FALSE (inner_frag != NULL))
 405         {
 406           inner_frag->next_hdr = inner_ip6->protocol;
 407           inner_frag->identification = inner_frag_id;
 408           inner_frag->rsv = 0;
 409           inner_frag->fragment_offset_and_more =
 410             ip6_frag_hdr_offset_and_more (inner_frag_offset, inner_frag_more);
 411           inner_ip6->protocol = IP_PROTOCOL_IPV6_FRAGMENTATION;
 412           inner_ip6->payload_length =
 413             clib_host_to_net_u16 (clib_net_to_host_u16
 414                                   (inner_ip6->payload_length) +
 415                                   sizeof (*inner_frag));
 416         }
 417
 418       csum = ip_csum_add_even (csum, inner_ip6->src_address.as_u64[0]);
 419       csum = ip_csum_add_even (csum, inner_ip6->src_address.as_u64[1]);
 420       csum = ip_csum_add_even (csum, inner_ip6->dst_address.as_u64[0]);
 421       csum = ip_csum_add_even (csum, inner_ip6->dst_address.as_u64[1]);
 422       *inner_L4_checksum = ip_csum_fold (csum);
 423
 424     }
 425   else
 426     {
 427       vlib_buffer_advance (p, sizeof (*ip4) - sizeof (*ip6));
 428       ip6 = vlib_buffer_get_current (p);
 429       ip6->payload_length =
 430         clib_host_to_net_u16 (clib_net_to_host_u16 (ip4->length) -
 431                               sizeof (*ip4));
 432     }
 433
 434   //Translate outer IPv6
 435   ip6->ip_version_traffic_class_and_flow_label =
 436     clib_host_to_net_u32 ((6 << 28) + (ip4->tos << 20));
 437
 438   ip6->hop_limit = ip4->ttl;
 439   ip6->protocol = IP_PROTOCOL_ICMP6;
 440
 441   ip4_map_t_embedded_address (d, &ip6->src_address, &ip4->src_address);
 442   ip6->dst_address.as_u64[0] =
 443     map_get_pfx_net (d, ip4->dst_address.as_u32, recv_port);
 444   ip6->dst_address.as_u64[1] =
 445     map_get_sfx_net (d, ip4->dst_address.as_u32, recv_port);
 446
 447   //Truncate when the packet exceeds the minimal IPv6 MTU
 448   if (p->current_length > 1280)
 449     {
 450       ip6->payload_length = clib_host_to_net_u16 (1280 - sizeof (*ip6));
 451       p->current_length = 1280; //Looks too simple to be correct...
 452     }
 453
 454   //TODO: We could do an easy diff-checksum for echo requests/replies
 455   //Recompute ICMP checksum
 456   icmp->checksum = 0;
 457   csum = ip_csum_with_carry (0, ip6->payload_length);
 458   csum = ip_csum_with_carry (csum, clib_host_to_net_u16 (ip6->protocol));
 459   csum = ip_csum_with_carry (csum, ip6->src_address.as_u64[0]);
 460   csum = ip_csum_with_carry (csum, ip6->src_address.as_u64[1]);
 461   csum = ip_csum_with_carry (csum, ip6->dst_address.as_u64[0]);
 462   csum = ip_csum_with_carry (csum, ip6->dst_address.as_u64[1]);
 463   csum =
 464     ip_incremental_checksum (csum, icmp,
 465                              clib_net_to_host_u16 (ip6->payload_length));
 466   icmp->checksum = ~ip_csum_fold (csum);
 467 }
 468
 469 static uword
 470 ip4_map_t_icmp (vlib_main_t * vm,
 471                 vlib_node_runtime_t * node, vlib_frame_t * frame)
 472 {
 473   u32 n_left_from, *from, next_index, *to_next, n_left_to_next;
 474   vlib_node_runtime_t *error_node =
 475     vlib_node_get_runtime (vm, ip4_map_t_icmp_node.index);
 476   from = vlib_frame_vector_args (frame);
 477   n_left_from = frame->n_vectors;
 478   next_index = node->cached_next_index;
 479   vlib_combined_counter_main_t *cm = map_main.domain_counters;
 480   u32 cpu_index = os_get_cpu_number ();
 481
 482   while (n_left_from > 0)
 483     {
 484       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 485
 486       while (n_left_from > 0 && n_left_to_next > 0)
 487         {
 488           u32 pi0;
 489           vlib_buffer_t *p0;
 490           ip4_mapt_icmp_next_t next0;
 491           u8 error0;
 492           map_domain_t *d0;
 493           u16 len0;
 494
 495           next0 = IP4_MAPT_ICMP_NEXT_IP6_LOOKUP;
 496           pi0 = to_next[0] = from[0];
 497           from += 1;
 498           n_left_from -= 1;
 499           to_next += 1;
 500           n_left_to_next -= 1;
 501           error0 = MAP_ERROR_NONE;
 502
 503           p0 = vlib_get_buffer (vm, pi0);
 504           vlib_buffer_advance (p0, sizeof (ip4_mapt_pseudo_header_t));  //The pseudo-header is not used
 505           len0 =
 506             clib_net_to_host_u16 (((ip4_header_t *)
 507                                    vlib_buffer_get_current (p0))->length);
 508           d0 =
 509             pool_elt_at_index (map_main.domains,
 510                                vnet_buffer (p0)->map_t.map_domain_index);
 511           _ip4_map_t_icmp (d0, p0, &error0);
 512
 513           if (vnet_buffer (p0)->map_t.mtu < p0->current_length)
 514             {
 515               vnet_buffer (p0)->ip_frag.header_offset = 0;
 516               vnet_buffer (p0)->ip_frag.mtu = vnet_buffer (p0)->map_t.mtu;
 517               vnet_buffer (p0)->ip_frag.next_index = IP6_FRAG_NEXT_IP6_LOOKUP;
 518               next0 = IP4_MAPT_ICMP_NEXT_IP6_FRAG;
 519             }
 520           if (PREDICT_TRUE (error0 == MAP_ERROR_NONE))
 521             {
 522               vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX,
 523                                                cpu_index,
 524                                                vnet_buffer (p0)->map_t.
 525                                                map_domain_index, 1, len0);
 526             }
 527           else
 528             {
 529               next0 = IP4_MAPT_ICMP_NEXT_DROP;
 530             }
 531           p0->error = error_node->errors[error0];
 532           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 533                                            to_next, n_left_to_next, pi0,
 534                                            next0);
 535         }
 536       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 537     }
 538   return frame->n_vectors;
 539 }
 540
 541 static uword
 542 ip4_map_t_fragmented (vlib_main_t * vm,
 543                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 544 {
 545   u32 n_left_from, *from, next_index, *to_next, n_left_to_next;
 546   from = vlib_frame_vector_args (frame);
 547   n_left_from = frame->n_vectors;
 548   next_index = node->cached_next_index;
 549
 550   while (n_left_from > 0)
 551     {
 552       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 553
 554       while (n_left_from > 0 && n_left_to_next > 0)
 555         {
 556           u32 pi0;
 557           vlib_buffer_t *p0;
 558           ip4_header_t *ip40;
 559           ip6_header_t *ip60;
 560           ip6_frag_hdr_t *frag0;
 561           ip4_mapt_pseudo_header_t *pheader0;
 562           ip4_mapt_fragmented_next_t next0;
 563
 564           next0 = IP4_MAPT_FRAGMENTED_NEXT_IP6_LOOKUP;
 565           pi0 = to_next[0] = from[0];
 566           from += 1;
 567           n_left_from -= 1;
 568           to_next += 1;
 569           n_left_to_next -= 1;
 570
 571           p0 = vlib_get_buffer (vm, pi0);
 572
 573           //Accessing pseudo header
 574           pheader0 = vlib_buffer_get_current (p0);
 575           vlib_buffer_advance (p0, sizeof (*pheader0));
 576
 577           //Accessing ip4 header
 578           ip40 = vlib_buffer_get_current (p0);
 579           frag0 =
 580             (ip6_frag_hdr_t *) u8_ptr_add (ip40,
 581                                            sizeof (*ip40) - sizeof (*frag0));
 582           ip60 =
 583             (ip6_header_t *) u8_ptr_add (ip40,
 584                                          sizeof (*ip40) - sizeof (*frag0) -
 585                                          sizeof (*ip60));
 586           vlib_buffer_advance (p0,
 587                                sizeof (*ip40) - sizeof (*ip60) -
 588                                sizeof (*frag0));
 589
 590           //We know that the protocol was one of ICMP, TCP or UDP
 591           //because the first fragment was found and cached
 592           frag0->next_hdr =
 593             (ip40->protocol ==
 594              IP_PROTOCOL_ICMP) ? IP_PROTOCOL_ICMP6 : ip40->protocol;
 595           frag0->identification = frag_id_4to6 (ip40->fragment_id);
 596           frag0->rsv = 0;
 597           frag0->fragment_offset_and_more =
 598             ip6_frag_hdr_offset_and_more (ip4_get_fragment_offset (ip40),
 599                                           clib_net_to_host_u16
 600                                           (ip40->flags_and_fragment_offset) &
 601                                           IP4_HEADER_FLAG_MORE_FRAGMENTS);
 602
 603           ip60->ip_version_traffic_class_and_flow_label =
 604             clib_host_to_net_u32 ((6 << 28) + (ip40->tos << 20));
 605           ip60->payload_length =
 606             clib_host_to_net_u16 (clib_net_to_host_u16 (ip40->length) -
 607                                   sizeof (*ip40) + sizeof (*frag0));
 608           ip60->hop_limit = ip40->ttl;
 609           ip60->protocol = IP_PROTOCOL_IPV6_FRAGMENTATION;
 610           ip60->dst_address.as_u64[0] = pheader0->daddr.as_u64[0];
 611           ip60->dst_address.as_u64[1] = pheader0->daddr.as_u64[1];
 612           ip60->src_address.as_u64[0] = pheader0->saddr.as_u64[0];
 613           ip60->src_address.as_u64[1] = pheader0->saddr.as_u64[1];
 614
 615           if (vnet_buffer (p0)->map_t.mtu < p0->current_length)
 616             {
 617               vnet_buffer (p0)->ip_frag.header_offset = 0;
 618               vnet_buffer (p0)->ip_frag.mtu = vnet_buffer (p0)->map_t.mtu;
 619               vnet_buffer (p0)->ip_frag.next_index = IP6_FRAG_NEXT_IP6_LOOKUP;
 620               next0 = IP4_MAPT_FRAGMENTED_NEXT_IP6_FRAG;
 621             }
 622
 623           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 624                                            to_next, n_left_to_next, pi0,
 625                                            next0);
 626         }
 627       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 628     }
 629   return frame->n_vectors;
 630 }
 631
 632 static uword
 633 ip4_map_t_tcp_udp (vlib_main_t * vm,
 634                    vlib_node_runtime_t * node, vlib_frame_t * frame)
 635 {
 636   u32 n_left_from, *from, next_index, *to_next, n_left_to_next;
 637   from = vlib_frame_vector_args (frame);
 638   n_left_from = frame->n_vectors;
 639   next_index = node->cached_next_index;
 640
 641   while (n_left_from > 0)
 642     {
 643       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 644
 645 #ifdef IP4_MAP_T_DUAL_LOOP
 646       while (n_left_from >= 4 && n_left_to_next >= 2)
 647         {
 648           u32 pi0, pi1;
 649           vlib_buffer_t *p0, *p1;
 650           ip4_header_t *ip40, *ip41;
 651           ip6_header_t *ip60, *ip61;
 652           ip_csum_t csum0, csum1;
 653           u16 *checksum0, *checksum1;
 654           ip6_frag_hdr_t *frag0, *frag1;
 655           u32 frag_id0, frag_id1;
 656           ip4_mapt_pseudo_header_t *pheader0, *pheader1;
 657           ip4_mapt_tcp_udp_next_t next0, next1;
 658
 659           pi0 = to_next[0] = from[0];
 660           pi1 = to_next[1] = from[1];
 661           from += 2;
 662           n_left_from -= 2;
 663           to_next += 2;
 664           n_left_to_next -= 2;
 665
 666           next0 = IP4_MAPT_TCP_UDP_NEXT_IP6_LOOKUP;
 667           next1 = IP4_MAPT_TCP_UDP_NEXT_IP6_LOOKUP;
 668           p0 = vlib_get_buffer (vm, pi0);
 669           p1 = vlib_get_buffer (vm, pi1);
 670
 671           //Accessing pseudo header
 672           pheader0 = vlib_buffer_get_current (p0);
 673           pheader1 = vlib_buffer_get_current (p1);
 674           vlib_buffer_advance (p0, sizeof (*pheader0));
 675           vlib_buffer_advance (p1, sizeof (*pheader1));
 676
 677           //Accessing ip4 header
 678           ip40 = vlib_buffer_get_current (p0);
 679           ip41 = vlib_buffer_get_current (p1);
 680           checksum0 =
 681             (u16 *) u8_ptr_add (ip40,
 682                                 vnet_buffer (p0)->map_t.checksum_offset);
 683           checksum1 =
 684             (u16 *) u8_ptr_add (ip41,
 685                                 vnet_buffer (p1)->map_t.checksum_offset);
 686
 687           //UDP checksum is optional over IPv4 but mandatory for IPv6
 688           //We do not check udp->length sanity but use our safe computed value instead
 689           if (PREDICT_FALSE
 690               (!*checksum0 && ip40->protocol == IP_PROTOCOL_UDP))
 691             {
 692               u16 udp_len =
 693                 clib_host_to_net_u16 (ip40->length) - sizeof (*ip40);
 694               udp_header_t *udp =
 695                 (udp_header_t *) u8_ptr_add (ip40, sizeof (*ip40));
 696               ip_csum_t csum;
 697               csum = ip_incremental_checksum (0, udp, udp_len);
 698               csum =
 699                 ip_csum_with_carry (csum, clib_host_to_net_u16 (udp_len));
 700               csum =
 701                 ip_csum_with_carry (csum,
 702                                     clib_host_to_net_u16 (IP_PROTOCOL_UDP));
 703               csum =
 704                 ip_csum_with_carry (csum, *((u64 *) (&ip40->src_address)));
 705               *checksum0 = ~ip_csum_fold (csum);
 706             }
 707           if (PREDICT_FALSE
 708               (!*checksum1 && ip41->protocol == IP_PROTOCOL_UDP))
 709             {
 710               u16 udp_len =
 711                 clib_host_to_net_u16 (ip41->length) - sizeof (*ip40);
 712               udp_header_t *udp =
 713                 (udp_header_t *) u8_ptr_add (ip41, sizeof (*ip40));
 714               ip_csum_t csum;
 715               csum = ip_incremental_checksum (0, udp, udp_len);
 716               csum =
 717                 ip_csum_with_carry (csum, clib_host_to_net_u16 (udp_len));
 718               csum =
 719                 ip_csum_with_carry (csum,
 720                                     clib_host_to_net_u16 (IP_PROTOCOL_UDP));
 721               csum =
 722                 ip_csum_with_carry (csum, *((u64 *) (&ip41->src_address)));
 723               *checksum1 = ~ip_csum_fold (csum);
 724             }
 725
 726           csum0 = ip_csum_sub_even (*checksum0, ip40->src_address.as_u32);
 727           csum1 = ip_csum_sub_even (*checksum1, ip41->src_address.as_u32);
 728           csum0 = ip_csum_sub_even (csum0, ip40->dst_address.as_u32);
 729           csum1 = ip_csum_sub_even (csum1, ip41->dst_address.as_u32);
 730
 731           // Deal with fragmented packets
 732           if (PREDICT_FALSE (ip40->flags_and_fragment_offset &
 733                              clib_host_to_net_u16
 734                              (IP4_HEADER_FLAG_MORE_FRAGMENTS)))
 735             {
 736               ip60 =
 737                 (ip6_header_t *) u8_ptr_add (ip40,
 738                                              sizeof (*ip40) - sizeof (*ip60) -
 739                                              sizeof (*frag0));
 740               frag0 =
 741                 (ip6_frag_hdr_t *) u8_ptr_add (ip40,
 742                                                sizeof (*ip40) -
 743                                                sizeof (*frag0));
 744               frag_id0 = frag_id_4to6 (ip40->fragment_id);
 745               vlib_buffer_advance (p0,
 746                                    sizeof (*ip40) - sizeof (*ip60) -
 747                                    sizeof (*frag0));
 748             }
 749           else
 750             {
 751               ip60 =
 752                 (ip6_header_t *) (((u8 *) ip40) + sizeof (*ip40) -
 753                                   sizeof (*ip60));
 754               vlib_buffer_advance (p0, sizeof (*ip40) - sizeof (*ip60));
 755               frag0 = NULL;
 756             }
 757
 758           if (PREDICT_FALSE (ip41->flags_and_fragment_offset &
 759                              clib_host_to_net_u16
 760                              (IP4_HEADER_FLAG_MORE_FRAGMENTS)))
 761             {
 762               ip61 =
 763                 (ip6_header_t *) u8_ptr_add (ip41,
 764                                              sizeof (*ip40) - sizeof (*ip60) -
 765                                              sizeof (*frag0));
 766               frag1 =
 767                 (ip6_frag_hdr_t *) u8_ptr_add (ip41,
 768                                                sizeof (*ip40) -
 769                                                sizeof (*frag0));
 770               frag_id1 = frag_id_4to6 (ip41->fragment_id);
 771               vlib_buffer_advance (p1,
 772                                    sizeof (*ip40) - sizeof (*ip60) -
 773                                    sizeof (*frag0));
 774             }
 775           else
 776             {
 777               ip61 =
 778                 (ip6_header_t *) (((u8 *) ip41) + sizeof (*ip40) -
 779                                   sizeof (*ip60));
 780               vlib_buffer_advance (p1, sizeof (*ip40) - sizeof (*ip60));
 781               frag1 = NULL;
 782             }
 783
 784           ip60->ip_version_traffic_class_and_flow_label =
 785             clib_host_to_net_u32 ((6 << 28) + (ip40->tos << 20));
 786           ip61->ip_version_traffic_class_and_flow_label =
 787             clib_host_to_net_u32 ((6 << 28) + (ip41->tos << 20));
 788           ip60->payload_length = u16_net_add (ip40->length, -sizeof (*ip40));
 789           ip61->payload_length = u16_net_add (ip41->length, -sizeof (*ip40));
 790           ip60->hop_limit = ip40->ttl;
 791           ip61->hop_limit = ip41->ttl;
 792           ip60->protocol = ip40->protocol;
 793           ip61->protocol = ip41->protocol;
 794
 795           if (PREDICT_FALSE (frag0 != NULL))
 796             {
 797               frag0->next_hdr = ip60->protocol;
 798               frag0->identification = frag_id0;
 799               frag0->rsv = 0;
 800               frag0->fragment_offset_and_more =
 801                 ip6_frag_hdr_offset_and_more (0, 1);
 802               ip60->protocol = IP_PROTOCOL_IPV6_FRAGMENTATION;
 803               ip60->payload_length =
 804                 u16_net_add (ip60->payload_length, sizeof (*frag0));
 805             }
 806
 807           if (PREDICT_FALSE (frag1 != NULL))
 808             {
 809               frag1->next_hdr = ip61->protocol;
 810               frag1->identification = frag_id1;
 811               frag1->rsv = 0;
 812               frag1->fragment_offset_and_more =
 813                 ip6_frag_hdr_offset_and_more (0, 1);
 814               ip61->protocol = IP_PROTOCOL_IPV6_FRAGMENTATION;
 815               ip61->payload_length =
 816                 u16_net_add (ip61->payload_length, sizeof (*frag0));
 817             }
 818
 819           //Finally copying the address
 820           ip60->dst_address.as_u64[0] = pheader0->daddr.as_u64[0];
 821           ip61->dst_address.as_u64[0] = pheader1->daddr.as_u64[0];
 822           ip60->dst_address.as_u64[1] = pheader0->daddr.as_u64[1];
 823           ip61->dst_address.as_u64[1] = pheader1->daddr.as_u64[1];
 824           ip60->src_address.as_u64[0] = pheader0->saddr.as_u64[0];
 825           ip61->src_address.as_u64[0] = pheader1->saddr.as_u64[0];
 826           ip60->src_address.as_u64[1] = pheader0->saddr.as_u64[1];
 827           ip61->src_address.as_u64[1] = pheader1->saddr.as_u64[1];
 828
 829           csum0 = ip_csum_add_even (csum0, ip60->src_address.as_u64[0]);
 830           csum1 = ip_csum_add_even (csum1, ip61->src_address.as_u64[0]);
 831           csum0 = ip_csum_add_even (csum0, ip60->src_address.as_u64[1]);
 832           csum1 = ip_csum_add_even (csum1, ip61->src_address.as_u64[1]);
 833           csum0 = ip_csum_add_even (csum0, ip60->dst_address.as_u64[0]);
 834           csum1 = ip_csum_add_even (csum1, ip61->dst_address.as_u64[0]);
 835           csum0 = ip_csum_add_even (csum0, ip60->dst_address.as_u64[1]);
 836           csum1 = ip_csum_add_even (csum1, ip61->dst_address.as_u64[1]);
 837           *checksum0 = ip_csum_fold (csum0);
 838           *checksum1 = ip_csum_fold (csum1);
 839
 840           if (vnet_buffer (p0)->map_t.mtu < p0->current_length)
 841             {
 842               vnet_buffer (p0)->ip_frag.header_offset = 0;
 843               vnet_buffer (p0)->ip_frag.mtu = vnet_buffer (p0)->map_t.mtu;
 844               vnet_buffer (p0)->ip_frag.next_index = IP6_FRAG_NEXT_IP6_LOOKUP;
 845               next0 = IP4_MAPT_TCP_UDP_NEXT_IP6_FRAG;
 846             }
 847
 848           if (vnet_buffer (p1)->map_t.mtu < p1->current_length)
 849             {
 850               vnet_buffer (p1)->ip_frag.header_offset = 0;
 851               vnet_buffer (p1)->ip_frag.mtu = vnet_buffer (p1)->map_t.mtu;
 852               vnet_buffer (p1)->ip_frag.next_index = IP6_FRAG_NEXT_IP6_LOOKUP;
 853               next1 = IP4_MAPT_TCP_UDP_NEXT_IP6_FRAG;
 854             }
 855
 856           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 857                                            to_next, n_left_to_next, pi0, pi1,
 858                                            next0, next1);
 859         }
 860 #endif
 861
 862       while (n_left_from > 0 && n_left_to_next > 0)
 863         {
 864           u32 pi0;
 865           vlib_buffer_t *p0;
 866           ip4_header_t *ip40;
 867           ip6_header_t *ip60;
 868           ip_csum_t csum0;
 869           u16 *checksum0;
 870           ip6_frag_hdr_t *frag0;
 871           u32 frag_id0;
 872           ip4_mapt_pseudo_header_t *pheader0;
 873           ip4_mapt_tcp_udp_next_t next0;
 874
 875           pi0 = to_next[0] = from[0];
 876           from += 1;
 877           n_left_from -= 1;
 878           to_next += 1;
 879           n_left_to_next -= 1;
 880
 881           next0 = IP4_MAPT_TCP_UDP_NEXT_IP6_LOOKUP;
 882           p0 = vlib_get_buffer (vm, pi0);
 883
 884           //Accessing pseudo header
 885           pheader0 = vlib_buffer_get_current (p0);
 886           vlib_buffer_advance (p0, sizeof (*pheader0));
 887
 888           //Accessing ip4 header
 889           ip40 = vlib_buffer_get_current (p0);
 890           checksum0 =
 891             (u16 *) u8_ptr_add (ip40,
 892                                 vnet_buffer (p0)->map_t.checksum_offset);
 893
 894           //UDP checksum is optional over IPv4 but mandatory for IPv6
 895           //We do not check udp->length sanity but use our safe computed value instead
 896           if (PREDICT_FALSE
 897               (!*checksum0 && ip40->protocol == IP_PROTOCOL_UDP))
 898             {
 899               u16 udp_len =
 900                 clib_host_to_net_u16 (ip40->length) - sizeof (*ip40);
 901               udp_header_t *udp =
 902                 (udp_header_t *) u8_ptr_add (ip40, sizeof (*ip40));
 903               ip_csum_t csum;
 904               csum = ip_incremental_checksum (0, udp, udp_len);
 905               csum =
 906                 ip_csum_with_carry (csum, clib_host_to_net_u16 (udp_len));
 907               csum =
 908                 ip_csum_with_carry (csum,
 909                                     clib_host_to_net_u16 (IP_PROTOCOL_UDP));
 910               csum =
 911                 ip_csum_with_carry (csum, *((u64 *) (&ip40->src_address)));
 912               *checksum0 = ~ip_csum_fold (csum);
 913             }
 914
 915           csum0 = ip_csum_sub_even (*checksum0, ip40->src_address.as_u32);
 916           csum0 = ip_csum_sub_even (csum0, ip40->dst_address.as_u32);
 917
 918           // Deal with fragmented packets
 919           if (PREDICT_FALSE (ip40->flags_and_fragment_offset &
 920                              clib_host_to_net_u16
 921                              (IP4_HEADER_FLAG_MORE_FRAGMENTS)))
 922             {
 923               ip60 =
 924                 (ip6_header_t *) u8_ptr_add (ip40,
 925                                              sizeof (*ip40) - sizeof (*ip60) -
 926                                              sizeof (*frag0));
 927               frag0 =
 928                 (ip6_frag_hdr_t *) u8_ptr_add (ip40,
 929                                                sizeof (*ip40) -
 930                                                sizeof (*frag0));
 931               frag_id0 = frag_id_4to6 (ip40->fragment_id);
 932               vlib_buffer_advance (p0,
 933                                    sizeof (*ip40) - sizeof (*ip60) -
 934                                    sizeof (*frag0));
 935             }
 936           else
 937             {
 938               ip60 =
 939                 (ip6_header_t *) (((u8 *) ip40) + sizeof (*ip40) -
 940                                   sizeof (*ip60));
 941               vlib_buffer_advance (p0, sizeof (*ip40) - sizeof (*ip60));
 942               frag0 = NULL;
 943             }
 944
 945           ip60->ip_version_traffic_class_and_flow_label =
 946             clib_host_to_net_u32 ((6 << 28) + (ip40->tos << 20));
 947           ip60->payload_length = u16_net_add (ip40->length, -sizeof (*ip40));
 948           ip60->hop_limit = ip40->ttl;
 949           ip60->protocol = ip40->protocol;
 950
 951           if (PREDICT_FALSE (frag0 != NULL))
 952             {
 953               frag0->next_hdr = ip60->protocol;
 954               frag0->identification = frag_id0;
 955               frag0->rsv = 0;
 956               frag0->fragment_offset_and_more =
 957                 ip6_frag_hdr_offset_and_more (0, 1);
 958               ip60->protocol = IP_PROTOCOL_IPV6_FRAGMENTATION;
 959               ip60->payload_length =
 960                 u16_net_add (ip60->payload_length, sizeof (*frag0));
 961             }
 962
 963           //Finally copying the address
 964           ip60->dst_address.as_u64[0] = pheader0->daddr.as_u64[0];
 965           ip60->dst_address.as_u64[1] = pheader0->daddr.as_u64[1];
 966           ip60->src_address.as_u64[0] = pheader0->saddr.as_u64[0];
 967           ip60->src_address.as_u64[1] = pheader0->saddr.as_u64[1];
 968
 969           csum0 = ip_csum_add_even (csum0, ip60->src_address.as_u64[0]);
 970           csum0 = ip_csum_add_even (csum0, ip60->src_address.as_u64[1]);
 971           csum0 = ip_csum_add_even (csum0, ip60->dst_address.as_u64[0]);
 972           csum0 = ip_csum_add_even (csum0, ip60->dst_address.as_u64[1]);
 973           *checksum0 = ip_csum_fold (csum0);
 974
 975           if (vnet_buffer (p0)->map_t.mtu < p0->current_length)
 976             {
 977               //Send to fragmentation node if necessary
 978               vnet_buffer (p0)->ip_frag.header_offset = 0;
 979               vnet_buffer (p0)->ip_frag.mtu = vnet_buffer (p0)->map_t.mtu;
 980               vnet_buffer (p0)->ip_frag.next_index = IP6_FRAG_NEXT_IP6_LOOKUP;
 981               next0 = IP4_MAPT_TCP_UDP_NEXT_IP6_FRAG;
 982             }
 983
 984           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 985                                            to_next, n_left_to_next, pi0,
 986                                            next0);
 987         }
 988       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 989     }
 990
 991   return frame->n_vectors;
 992 }
 993
 994 static_always_inline void
 995 ip4_map_t_classify (vlib_buffer_t * p0, map_domain_t * d0,
 996                     ip4_header_t * ip40, u16 ip4_len0, i32 * dst_port0,
 997                     u8 * error0, ip4_mapt_next_t * next0)
 998 {
 999   if (PREDICT_FALSE (ip4_get_fragment_offset (ip40)))
1000     {
1001       *next0 = IP4_MAPT_NEXT_MAPT_FRAGMENTED;
1002       if (d0->ea_bits_len == 0 && d0->rules)
1003         {
1004           *dst_port0 = 0;
1005         }
1006       else
1007         {
1008           *dst_port0 = ip4_map_fragment_get_port (ip40);
1009           *error0 = (*dst_port0 == -1) ? MAP_ERROR_FRAGMENT_MEMORY : *error0;
1010         }
1011     }
1012   else if (PREDICT_TRUE (ip40->protocol == IP_PROTOCOL_TCP))
1013     {
1014       vnet_buffer (p0)->map_t.checksum_offset = 36;
1015       *next0 = IP4_MAPT_NEXT_MAPT_TCP_UDP;
1016       *error0 = ip4_len0 < 40 ? MAP_ERROR_MALFORMED : *error0;
1017       *dst_port0 = (i32) * ((u16 *) u8_ptr_add (ip40, sizeof (*ip40) + 2));
1018     }
1019   else if (PREDICT_TRUE (ip40->protocol == IP_PROTOCOL_UDP))
1020     {
1021       vnet_buffer (p0)->map_t.checksum_offset = 26;
1022       *next0 = IP4_MAPT_NEXT_MAPT_TCP_UDP;
1023       *error0 = ip4_len0 < 28 ? MAP_ERROR_MALFORMED : *error0;
1024       *dst_port0 = (i32) * ((u16 *) u8_ptr_add (ip40, sizeof (*ip40) + 2));
1025     }
1026   else if (ip40->protocol == IP_PROTOCOL_ICMP)
1027     {
1028       *next0 = IP4_MAPT_NEXT_MAPT_ICMP;
1029       if (d0->ea_bits_len == 0 && d0->rules)
1030         *dst_port0 = 0;
1031       else if (((icmp46_header_t *) u8_ptr_add (ip40, sizeof (*ip40)))->code
1032                == ICMP4_echo_reply
1033                || ((icmp46_header_t *)
1034                    u8_ptr_add (ip40,
1035                                sizeof (*ip40)))->code == ICMP4_echo_request)
1036         *dst_port0 = (i32) * ((u16 *) u8_ptr_add (ip40, sizeof (*ip40) + 6));
1037     }
1038   else
1039     {
1040       *error0 = MAP_ERROR_BAD_PROTOCOL;
1041     }
1042 }
1043
1044 static uword
1045 ip4_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1046 {
1047   u32 n_left_from, *from, next_index, *to_next, n_left_to_next;
1048   vlib_node_runtime_t *error_node =
1049     vlib_node_get_runtime (vm, ip4_map_t_node.index);
1050   from = vlib_frame_vector_args (frame);
1051   n_left_from = frame->n_vectors;
1052   next_index = node->cached_next_index;
1053   vlib_combined_counter_main_t *cm = map_main.domain_counters;
1054   u32 cpu_index = os_get_cpu_number ();
1055
1056   while (n_left_from > 0)
1057     {
1058       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1059
1060 #ifdef IP4_MAP_T_DUAL_LOOP
1061       while (n_left_from >= 4 && n_left_to_next >= 2)
1062         {
1063           u32 pi0, pi1;
1064           vlib_buffer_t *p0, *p1;
1065           ip4_header_t *ip40, *ip41;
1066           map_domain_t *d0, *d1;
1067           ip4_mapt_next_t next0 = 0, next1 = 0;
1068           u16 ip4_len0, ip4_len1;
1069           u8 error0, error1;
1070           i32 dst_port0, dst_port1;
1071           ip4_mapt_pseudo_header_t *pheader0, *pheader1;
1072
1073           pi0 = to_next[0] = from[0];
1074           pi1 = to_next[1] = from[1];
1075           from += 2;
1076           n_left_from -= 2;
1077           to_next += 2;
1078           n_left_to_next -= 2;
1079           error0 = MAP_ERROR_NONE;
1080           error1 = MAP_ERROR_NONE;
1081
1082           p0 = vlib_get_buffer (vm, pi0);
1083           p1 = vlib_get_buffer (vm, pi1);
1084           ip40 = vlib_buffer_get_current (p0);
1085           ip41 = vlib_buffer_get_current (p1);
1086           ip4_len0 = clib_host_to_net_u16 (ip40->length);
1087           ip4_len1 = clib_host_to_net_u16 (ip41->length);
1088
1089           if (PREDICT_FALSE (p0->current_length < ip4_len0 ||
1090                              ip40->ip_version_and_header_length != 0x45))
1091             {
1092               error0 = MAP_ERROR_UNKNOWN;
1093               next0 = IP4_MAPT_NEXT_DROP;
1094             }
1095
1096           if (PREDICT_FALSE (p1->current_length < ip4_len1 ||
1097                              ip41->ip_version_and_header_length != 0x45))
1098             {
1099               error1 = MAP_ERROR_UNKNOWN;
1100               next1 = IP4_MAPT_NEXT_DROP;
1101             }
1102
1103           d0 = ip4_map_get_domain (vnet_buffer (p0)->ip.adj_index[VLIB_TX],
1104                                    &vnet_buffer (p0)->map_t.map_domain_index);
1105           d1 = ip4_map_get_domain (vnet_buffer (p1)->ip.adj_index[VLIB_TX],
1106                                    &vnet_buffer (p1)->map_t.map_domain_index);
1107
1108           vnet_buffer (p0)->map_t.mtu = d0->mtu ? d0->mtu : ~0;
1109           vnet_buffer (p1)->map_t.mtu = d1->mtu ? d1->mtu : ~0;
1110
1111           dst_port0 = -1;
1112           dst_port1 = -1;
1113
1114           ip4_map_t_classify (p0, d0, ip40, ip4_len0, &dst_port0, &error0,
1115                               &next0);
1116           ip4_map_t_classify (p1, d1, ip41, ip4_len1, &dst_port1, &error1,
1117                               &next1);
1118
1119           //Add MAP-T pseudo header in front of the packet
1120           vlib_buffer_advance (p0, -sizeof (*pheader0));
1121           vlib_buffer_advance (p1, -sizeof (*pheader1));
1122           pheader0 = vlib_buffer_get_current (p0);
1123           pheader1 = vlib_buffer_get_current (p1);
1124
1125           //Save addresses within the packet
1126           ip4_map_t_embedded_address (d0, &pheader0->saddr,
1127                                       &ip40->src_address);
1128           ip4_map_t_embedded_address (d1, &pheader1->saddr,
1129                                       &ip41->src_address);
1130           pheader0->daddr.as_u64[0] =
1131             map_get_pfx_net (d0, ip40->dst_address.as_u32, (u16) dst_port0);
1132           pheader0->daddr.as_u64[1] =
1133             map_get_sfx_net (d0, ip40->dst_address.as_u32, (u16) dst_port0);
1134           pheader1->daddr.as_u64[0] =
1135             map_get_pfx_net (d1, ip41->dst_address.as_u32, (u16) dst_port1);
1136           pheader1->daddr.as_u64[1] =
1137             map_get_sfx_net (d1, ip41->dst_address.as_u32, (u16) dst_port1);
1138
1139           if (PREDICT_FALSE
1140               (ip4_is_first_fragment (ip40) && (dst_port0 != -1)
1141                && (d0->ea_bits_len != 0 || !d0->rules)
1142                && ip4_map_fragment_cache (ip40, dst_port0)))
1143             {
1144               error0 = MAP_ERROR_FRAGMENT_MEMORY;
1145             }
1146
1147           if (PREDICT_FALSE
1148               (ip4_is_first_fragment (ip41) && (dst_port1 != -1)
1149                && (d1->ea_bits_len != 0 || !d1->rules)
1150                && ip4_map_fragment_cache (ip41, dst_port1)))
1151             {
1152               error1 = MAP_ERROR_FRAGMENT_MEMORY;
1153             }
1154
1155           if (PREDICT_TRUE
1156               (error0 == MAP_ERROR_NONE && next0 != IP4_MAPT_NEXT_MAPT_ICMP))
1157             {
1158               vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX,
1159                                                cpu_index,
1160                                                vnet_buffer (p0)->map_t.
1161                                                map_domain_index, 1,
1162                                                clib_net_to_host_u16 (ip40->
1163                                                                      length));
1164             }
1165
1166           if (PREDICT_TRUE
1167               (error1 == MAP_ERROR_NONE && next1 != IP4_MAPT_NEXT_MAPT_ICMP))
1168             {
1169               vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX,
1170                                                cpu_index,
1171                                                vnet_buffer (p1)->map_t.
1172                                                map_domain_index, 1,
1173                                                clib_net_to_host_u16 (ip41->
1174                                                                      length));
1175             }
1176
1177           next0 = (error0 != MAP_ERROR_NONE) ? IP4_MAPT_NEXT_DROP : next0;
1178           next1 = (error1 != MAP_ERROR_NONE) ? IP4_MAPT_NEXT_DROP : next1;
1179           p0->error = error_node->errors[error0];
1180           p1->error = error_node->errors[error1];
1181           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1182                                            n_left_to_next, pi0, pi1, next0,
1183                                            next1);
1184         }
1185 #endif
1186
1187       while (n_left_from > 0 && n_left_to_next > 0)
1188         {
1189           u32 pi0;
1190           vlib_buffer_t *p0;
1191           ip4_header_t *ip40;
1192           map_domain_t *d0;
1193           ip4_mapt_next_t next0;
1194           u16 ip4_len0;
1195           u8 error0;
1196           i32 dst_port0;
1197           ip4_mapt_pseudo_header_t *pheader0;
1198
1199           pi0 = to_next[0] = from[0];
1200           from += 1;
1201           n_left_from -= 1;
1202           to_next += 1;
1203           n_left_to_next -= 1;
1204           error0 = MAP_ERROR_NONE;
1205
1206           p0 = vlib_get_buffer (vm, pi0);
1207           ip40 = vlib_buffer_get_current (p0);
1208           ip4_len0 = clib_host_to_net_u16 (ip40->length);
1209           if (PREDICT_FALSE (p0->current_length < ip4_len0 ||
1210                              ip40->ip_version_and_header_length != 0x45))
1211             {
1212               error0 = MAP_ERROR_UNKNOWN;
1213               next0 = IP4_MAPT_NEXT_DROP;
1214             }
1215
1216           d0 = ip4_map_get_domain (vnet_buffer (p0)->ip.adj_index[VLIB_TX],
1217                                    &vnet_buffer (p0)->map_t.map_domain_index);
1218
1219           vnet_buffer (p0)->map_t.mtu = d0->mtu ? d0->mtu : ~0;
1220
1221           dst_port0 = -1;
1222           ip4_map_t_classify (p0, d0, ip40, ip4_len0, &dst_port0, &error0,
1223                               &next0);
1224
1225           //Add MAP-T pseudo header in front of the packet
1226           vlib_buffer_advance (p0, -sizeof (*pheader0));
1227           pheader0 = vlib_buffer_get_current (p0);
1228
1229           //Save addresses within the packet
1230           ip4_map_t_embedded_address (d0, &pheader0->saddr,
1231                                       &ip40->src_address);
1232           pheader0->daddr.as_u64[0] =
1233             map_get_pfx_net (d0, ip40->dst_address.as_u32, (u16) dst_port0);
1234           pheader0->daddr.as_u64[1] =
1235             map_get_sfx_net (d0, ip40->dst_address.as_u32, (u16) dst_port0);
1236
1237           //It is important to cache at this stage because the result might be necessary
1238           //for packets within the same vector.
1239           //Actually, this approach even provides some limited out-of-order fragments support
1240           if (PREDICT_FALSE
1241               (ip4_is_first_fragment (ip40) && (dst_port0 != -1)
1242                && (d0->ea_bits_len != 0 || !d0->rules)
1243                && ip4_map_fragment_cache (ip40, dst_port0)))
1244             {
1245               error0 = MAP_ERROR_UNKNOWN;
1246             }
1247
1248           if (PREDICT_TRUE
1249               (error0 == MAP_ERROR_NONE && next0 != IP4_MAPT_NEXT_MAPT_ICMP))
1250             {
1251               vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX,
1252                                                cpu_index,
1253                                                vnet_buffer (p0)->map_t.
1254                                                map_domain_index, 1,
1255                                                clib_net_to_host_u16 (ip40->
1256                                                                      length));
1257             }
1258
1259           next0 = (error0 != MAP_ERROR_NONE) ? IP4_MAPT_NEXT_DROP : next0;
1260           p0->error = error_node->errors[error0];
1261           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1262                                            to_next, n_left_to_next, pi0,
1263                                            next0);
1264         }
1265       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1266     }
1267   return frame->n_vectors;
1268 }
1269
1270 static char *map_t_error_strings[] = {
1271 #define _(sym,string) string,
1272   foreach_map_error
1273 #undef _
1274 };
1275
1276 /* *INDENT-OFF* */
1277 VLIB_REGISTER_NODE(ip4_map_t_fragmented_node) = {
1278   .function = ip4_map_t_fragmented,
1279   .name = "ip4-map-t-fragmented",
1280   .vector_size = sizeof(u32),
1281   .format_trace = format_map_trace,
1282   .type = VLIB_NODE_TYPE_INTERNAL,
1283
1284   .n_errors = MAP_N_ERROR,
1285   .error_strings = map_t_error_strings,
1286
1287   .n_next_nodes = IP4_MAPT_FRAGMENTED_N_NEXT,
1288   .next_nodes = {
1289       [IP4_MAPT_FRAGMENTED_NEXT_IP6_LOOKUP] = "ip6-lookup",
1290       [IP4_MAPT_FRAGMENTED_NEXT_IP6_FRAG] = IP6_FRAG_NODE_NAME,
1291       [IP4_MAPT_FRAGMENTED_NEXT_DROP] = "error-drop",
1292   },
1293 };
1294 /* *INDENT-ON* */
1295
1296 /* *INDENT-OFF* */
1297 VLIB_REGISTER_NODE(ip4_map_t_icmp_node) = {
1298   .function = ip4_map_t_icmp,
1299   .name = "ip4-map-t-icmp",
1300   .vector_size = sizeof(u32),
1301   .format_trace = format_map_trace,
1302   .type = VLIB_NODE_TYPE_INTERNAL,
1303
1304   .n_errors = MAP_N_ERROR,
1305   .error_strings = map_t_error_strings,
1306
1307   .n_next_nodes = IP4_MAPT_ICMP_N_NEXT,
1308   .next_nodes = {
1309       [IP4_MAPT_ICMP_NEXT_IP6_LOOKUP] = "ip6-lookup",
1310       [IP4_MAPT_ICMP_NEXT_IP6_FRAG] = IP6_FRAG_NODE_NAME,
1311       [IP4_MAPT_ICMP_NEXT_DROP] = "error-drop",
1312   },
1313 };
1314 /* *INDENT-ON* */
1315
1316 /* *INDENT-OFF* */
1317 VLIB_REGISTER_NODE(ip4_map_t_tcp_udp_node) = {
1318   .function = ip4_map_t_tcp_udp,
1319   .name = "ip4-map-t-tcp-udp",
1320   .vector_size = sizeof(u32),
1321   .format_trace = format_map_trace,
1322   .type = VLIB_NODE_TYPE_INTERNAL,
1323
1324   .n_errors = MAP_N_ERROR,
1325   .error_strings = map_t_error_strings,
1326
1327   .n_next_nodes = IP4_MAPT_TCP_UDP_N_NEXT,
1328   .next_nodes = {
1329       [IP4_MAPT_TCP_UDP_NEXT_IP6_LOOKUP] = "ip6-lookup",
1330       [IP4_MAPT_TCP_UDP_NEXT_IP6_FRAG] = IP6_FRAG_NODE_NAME,
1331       [IP4_MAPT_TCP_UDP_NEXT_DROP] = "error-drop",
1332   },
1333 };
1334 /* *INDENT-ON* */
1335
1336 /* *INDENT-OFF* */
1337 VLIB_REGISTER_NODE(ip4_map_t_node) = {
1338   .function = ip4_map_t,
1339   .name = "ip4-map-t",
1340   .vector_size = sizeof(u32),
1341   .format_trace = format_map_trace,
1342   .type = VLIB_NODE_TYPE_INTERNAL,
1343
1344   .n_errors = MAP_N_ERROR,
1345   .error_strings = map_t_error_strings,
1346
1347   .n_next_nodes = IP4_MAPT_N_NEXT,
1348   .next_nodes = {
1349       [IP4_MAPT_NEXT_MAPT_TCP_UDP] = "ip4-map-t-tcp-udp",
1350       [IP4_MAPT_NEXT_MAPT_ICMP] = "ip4-map-t-icmp",
1351       [IP4_MAPT_NEXT_MAPT_FRAGMENTED] = "ip4-map-t-fragmented",
1352       [IP4_MAPT_NEXT_DROP] = "error-drop",
1353   },
1354 };
1355 /* *INDENT-ON* */
1356
1357 /*
1358  * fd.io coding-style-patch-verification: ON
1359  *
1360  * Local Variables:
1361  * eval: (c-set-style "gnu")
1362  * End:
1363  */