src/vnet/ipsec/ipsec_input.c

   1 /*
   2  * decap.c : IPSec tunnel decapsulation
   3  *
   4  * Copyright (c) 2015 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/api_errno.h>
  20 #include <vnet/ip/ip.h>
  21 #include <vnet/feature/feature.h>
  22 #include <vnet/ipsec/ipsec_spd_fp_lookup.h>
  23
  24 #include <vnet/ipsec/ipsec.h>
  25 #include <vnet/ipsec/esp.h>
  26 #include <vnet/ipsec/ah.h>
  27 #include <vnet/ipsec/ipsec_io.h>
  28
  29 #define foreach_ipsec_input_error                       \
  30 _(RX_PKTS, "IPSec pkts received")                       \
  31 _(RX_POLICY_MATCH, "IPSec policy match")                \
  32 _(RX_POLICY_NO_MATCH, "IPSec policy not matched")       \
  33 _(RX_POLICY_BYPASS, "IPSec policy bypass")              \
  34 _(RX_POLICY_DISCARD, "IPSec policy discard")
  35
  36 typedef enum
  37 {
  38 #define _(sym,str) IPSEC_INPUT_ERROR_##sym,
  39   foreach_ipsec_input_error
  40 #undef _
  41     IPSEC_INPUT_N_ERROR,
  42 } ipsec_input_error_t;
  43
  44 static char *ipsec_input_error_strings[] = {
  45 #define _(sym,string) string,
  46   foreach_ipsec_input_error
  47 #undef _
  48 };
  49
  50 typedef struct
  51 {
  52   ip_protocol_t proto;
  53   u32 spd;
  54   u32 policy_index;
  55   u32 policy_type;
  56   u32 sa_id;
  57   u32 spi;
  58   u32 seq;
  59 } ipsec_input_trace_t;
  60
  61 /* packet trace format function */
  62 static u8 *
  63 format_ipsec_input_trace (u8 * s, va_list * args)
  64 {
  65   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  66   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  67   ipsec_input_trace_t *t = va_arg (*args, ipsec_input_trace_t *);
  68
  69   s =
  70     format (s, "%U: sa_id %u type: %u spd %u policy %d spi %u (0x%08x) seq %u",
  71             format_ip_protocol, t->proto, t->sa_id, t->policy_type, t->spd,
  72             t->policy_index, t->spi, t->spi, t->seq);
  73
  74   return s;
  75 }
  76
  77 always_inline void
  78 ipsec4_input_spd_add_flow_cache_entry (ipsec_main_t *im, u32 sa, u32 da,
  79                                        ipsec_spd_policy_type_t policy_type,
  80                                        u32 pol_id)
  81 {
  82   u64 hash;
  83   u8 is_overwrite = 0, is_stale_overwrite = 0;
  84   /* Store in network byte order to avoid conversion on lookup */
  85   ipsec4_inbound_spd_tuple_t ip4_tuple = {
  86     .ip4_src_addr = (ip4_address_t) clib_host_to_net_u32 (sa),
  87     .ip4_dest_addr = (ip4_address_t) clib_host_to_net_u32 (da),
  88     .policy_type = policy_type
  89   };
  90
  91   ip4_tuple.kv_16_8.value =
  92     (((u64) pol_id) << 32) | ((u64) im->input_epoch_count);
  93
  94   hash = ipsec4_hash_16_8 (&ip4_tuple.kv_16_8);
  95   hash &= (im->ipsec4_in_spd_hash_num_buckets - 1);
  96
  97   ipsec_spinlock_lock (&im->ipsec4_in_spd_hash_tbl[hash].bucket_lock);
  98   /* Check if we are overwriting an existing entry so we know
  99     whether to increment the flow cache counter. Since flow
 100     cache counter is reset on any policy add/remove, but
 101     hash table values are not, we need to check if the entry
 102     we are overwriting is stale or not. If it's a stale entry
 103     overwrite, we still want to increment flow cache counter */
 104   is_overwrite = (im->ipsec4_in_spd_hash_tbl[hash].value != 0);
 105   /* Check if we are overwriting a stale entry by comparing
 106      with current epoch count */
 107   if (PREDICT_FALSE (is_overwrite))
 108     is_stale_overwrite =
 109       (im->input_epoch_count !=
 110        ((u32) (im->ipsec4_in_spd_hash_tbl[hash].value & 0xFFFFFFFF)));
 111   clib_memcpy_fast (&im->ipsec4_in_spd_hash_tbl[hash], &ip4_tuple.kv_16_8,
 112                     sizeof (ip4_tuple.kv_16_8));
 113   ipsec_spinlock_unlock (&im->ipsec4_in_spd_hash_tbl[hash].bucket_lock);
 114
 115   /* Increment the counter to track active flow cache entries
 116     when entering a fresh entry or overwriting a stale one */
 117   if (!is_overwrite || is_stale_overwrite)
 118     clib_atomic_fetch_add_relax (&im->ipsec4_in_spd_flow_cache_entries, 1);
 119
 120   return;
 121 }
 122
 123 always_inline ipsec_policy_t *
 124 ipsec4_input_spd_find_flow_cache_entry (ipsec_main_t *im, u32 sa, u32 da,
 125                                         ipsec_spd_policy_type_t policy_type)
 126 {
 127   ipsec_policy_t *p = NULL;
 128   ipsec4_hash_kv_16_8_t kv_result;
 129   u64 hash;
 130   ipsec4_inbound_spd_tuple_t ip4_tuple = { .ip4_src_addr = (ip4_address_t) sa,
 131                                            .ip4_dest_addr = (ip4_address_t) da,
 132                                            .policy_type = policy_type };
 133
 134   hash = ipsec4_hash_16_8 (&ip4_tuple.kv_16_8);
 135   hash &= (im->ipsec4_in_spd_hash_num_buckets - 1);
 136
 137   ipsec_spinlock_lock (&im->ipsec4_in_spd_hash_tbl[hash].bucket_lock);
 138   kv_result = im->ipsec4_in_spd_hash_tbl[hash];
 139   ipsec_spinlock_unlock (&im->ipsec4_in_spd_hash_tbl[hash].bucket_lock);
 140
 141   if (ipsec4_hash_key_compare_16_8 ((u64 *) &ip4_tuple.kv_16_8,
 142                                     (u64 *) &kv_result))
 143     {
 144       if (im->input_epoch_count == ((u32) (kv_result.value & 0xFFFFFFFF)))
 145         {
 146           /* Get the policy based on the index */
 147           p =
 148             pool_elt_at_index (im->policies, ((u32) (kv_result.value >> 32)));
 149         }
 150     }
 151
 152   return p;
 153 }
 154
 155 always_inline void
 156 ipsec_fp_in_5tuple_from_ip4_range (ipsec_fp_5tuple_t *tuple, u32 la, u32 ra,
 157                                    u32 spi, u8 action)
 158 {
 159   clib_memset (tuple->l3_zero_pad, 0, sizeof (tuple->l3_zero_pad));
 160   tuple->laddr.as_u32 = la;
 161   tuple->raddr.as_u32 = ra;
 162   tuple->spi = spi;
 163   tuple->action = action;
 164   tuple->is_ipv6 = 0;
 165 }
 166
 167 always_inline void
 168 ipsec_fp_in_5tuple_from_ip6_range (ipsec_fp_5tuple_t *tuple, ip6_address_t *la,
 169                                    ip6_address_t *ra, u32 spi, u8 action)
 170
 171 {
 172   clib_memcpy (&tuple->ip6_laddr, la, sizeof (ip6_address_t));
 173   clib_memcpy (&tuple->ip6_raddr, ra, sizeof (ip6_address_t));
 174
 175   tuple->spi = spi;
 176   tuple->action = action;
 177   tuple->is_ipv6 = 1;
 178 }
 179
 180 always_inline ipsec_policy_t *
 181 ipsec_input_policy_match (ipsec_spd_t *spd, u32 sa, u32 da,
 182                           ipsec_spd_policy_type_t policy_type)
 183 {
 184   ipsec_main_t *im = &ipsec_main;
 185   ipsec_policy_t *p;
 186   u32 *i;
 187
 188   vec_foreach (i, spd->policies[policy_type])
 189   {
 190     p = pool_elt_at_index (im->policies, *i);
 191
 192     if (da < clib_net_to_host_u32 (p->laddr.start.ip4.as_u32))
 193       continue;
 194
 195     if (da > clib_net_to_host_u32 (p->laddr.stop.ip4.as_u32))
 196       continue;
 197
 198     if (sa < clib_net_to_host_u32 (p->raddr.start.ip4.as_u32))
 199       continue;
 200
 201     if (sa > clib_net_to_host_u32 (p->raddr.stop.ip4.as_u32))
 202       continue;
 203
 204     if (im->input_flow_cache_flag)
 205       {
 206         /* Add an Entry in Flow cache */
 207         ipsec4_input_spd_add_flow_cache_entry (im, sa, da, policy_type, *i);
 208       }
 209     return p;
 210   }
 211   return 0;
 212 }
 213
 214 always_inline ipsec_policy_t *
 215 ipsec_input_protect_policy_match (ipsec_spd_t *spd, u32 sa, u32 da, u32 spi)
 216 {
 217   ipsec_main_t *im = &ipsec_main;
 218   ipsec_policy_t *p;
 219   ipsec_sa_t *s;
 220   u32 *i;
 221
 222   vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP4_INBOUND_PROTECT])
 223   {
 224     p = pool_elt_at_index (im->policies, *i);
 225     s = ipsec_sa_get (p->sa_index);
 226
 227     if (spi != s->spi)
 228       continue;
 229
 230     if (ipsec_sa_is_set_IS_TUNNEL (s))
 231       {
 232         if (da != clib_net_to_host_u32 (s->tunnel.t_dst.ip.ip4.as_u32))
 233           continue;
 234
 235         if (sa != clib_net_to_host_u32 (s->tunnel.t_src.ip.ip4.as_u32))
 236           continue;
 237
 238         goto return_policy;
 239       }
 240
 241     if (da < clib_net_to_host_u32 (p->laddr.start.ip4.as_u32))
 242       continue;
 243
 244     if (da > clib_net_to_host_u32 (p->laddr.stop.ip4.as_u32))
 245       continue;
 246
 247     if (sa < clib_net_to_host_u32 (p->raddr.start.ip4.as_u32))
 248       continue;
 249
 250     if (sa > clib_net_to_host_u32 (p->raddr.stop.ip4.as_u32))
 251       continue;
 252
 253   return_policy:
 254     if (im->input_flow_cache_flag)
 255       {
 256         /* Add an Entry in Flow cache */
 257         ipsec4_input_spd_add_flow_cache_entry (
 258           im, sa, da, IPSEC_SPD_POLICY_IP4_INBOUND_PROTECT, *i);
 259       }
 260
 261     return p;
 262   }
 263   return 0;
 264 }
 265
 266 always_inline uword
 267 ip6_addr_match_range (ip6_address_t * a, ip6_address_t * la,
 268                       ip6_address_t * ua)
 269 {
 270   if ((memcmp (a->as_u64, la->as_u64, 2 * sizeof (u64)) >= 0) &&
 271       (memcmp (a->as_u64, ua->as_u64, 2 * sizeof (u64)) <= 0))
 272     return 1;
 273   return 0;
 274 }
 275
 276 always_inline ipsec_policy_t *
 277 ipsec6_input_protect_policy_match (ipsec_spd_t * spd,
 278                                    ip6_address_t * sa,
 279                                    ip6_address_t * da, u32 spi)
 280 {
 281   ipsec_main_t *im = &ipsec_main;
 282   ipsec_policy_t *p;
 283   ipsec_sa_t *s;
 284   u32 *i;
 285
 286   vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP6_INBOUND_PROTECT])
 287   {
 288     p = pool_elt_at_index (im->policies, *i);
 289     s = ipsec_sa_get (p->sa_index);
 290
 291     if (spi != s->spi)
 292       continue;
 293
 294     if (ipsec_sa_is_set_IS_TUNNEL (s))
 295       {
 296         if (!ip6_address_is_equal (sa, &s->tunnel.t_src.ip.ip6))
 297           continue;
 298
 299         if (!ip6_address_is_equal (da, &s->tunnel.t_dst.ip.ip6))
 300           continue;
 301
 302         return p;
 303       }
 304
 305     if (!ip6_addr_match_range (sa, &p->raddr.start.ip6, &p->raddr.stop.ip6))
 306       continue;
 307
 308     if (!ip6_addr_match_range (da, &p->laddr.start.ip6, &p->laddr.stop.ip6))
 309       continue;
 310
 311     return p;
 312   }
 313   return 0;
 314 }
 315
 316 extern vlib_node_registration_t ipsec4_input_node;
 317
 318 VLIB_NODE_FN (ipsec4_input_node) (vlib_main_t * vm,
 319                                   vlib_node_runtime_t * node,
 320                                   vlib_frame_t * frame)
 321 {
 322   u32 n_left_from, *from, thread_index;
 323   ipsec_main_t *im = &ipsec_main;
 324   u64 ipsec_unprocessed = 0, ipsec_matched = 0;
 325   u64 ipsec_dropped = 0, ipsec_bypassed = 0;
 326   vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
 327   vlib_buffer_t **b = bufs;
 328   u16 nexts[VLIB_FRAME_SIZE], *next;
 329
 330   from = vlib_frame_vector_args (frame);
 331   n_left_from = frame->n_vectors;
 332   next = nexts;
 333   vlib_get_buffers (vm, from, bufs, n_left_from);
 334   thread_index = vm->thread_index;
 335
 336
 337   while (n_left_from > 0)
 338     {
 339       u32 next32, pi0;
 340       ip4_header_t *ip0;
 341       esp_header_t *esp0 = NULL;
 342       ah_header_t *ah0;
 343       ip4_ipsec_config_t *c0;
 344       ipsec_spd_t *spd0;
 345       ipsec_policy_t *p0 = NULL;
 346       u8 has_space0;
 347       bool search_flow_cache = false;
 348       ipsec_policy_t *policies[1];
 349       ipsec_fp_5tuple_t tuples[1];
 350       bool ip_v6 = true;
 351
 352       if (n_left_from > 2)
 353         {
 354           vlib_prefetch_buffer_data (b[1], LOAD);
 355         }
 356
 357       b[0]->flags |= VNET_BUFFER_F_IS_IP4;
 358       b[0]->flags &= ~VNET_BUFFER_F_IS_IP6;
 359       c0 = vnet_feature_next_with_data (&next32, b[0], sizeof (c0[0]));
 360       next[0] = (u16) next32;
 361
 362       spd0 = pool_elt_at_index (im->spds, c0->spd_index);
 363
 364       ip0 = vlib_buffer_get_current (b[0]);
 365
 366       if (PREDICT_TRUE
 367           (ip0->protocol == IP_PROTOCOL_IPSEC_ESP
 368            || ip0->protocol == IP_PROTOCOL_UDP))
 369         {
 370
 371           esp0 = (esp_header_t *) ((u8 *) ip0 + ip4_header_bytes (ip0));
 372           if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_UDP))
 373             {
 374               /* FIXME Skip, if not a UDP encapsulated packet */
 375               esp0 = (esp_header_t *) ((u8 *) esp0 + sizeof (udp_header_t));
 376             }
 377
 378           // if flow cache is enabled, first search through flow cache for a
 379           // policy match for either protect, bypass or discard rules, in that
 380           // order. if no match is found search_flow_cache is set to false (1)
 381           // and we revert back to linear search
 382           search_flow_cache = im->input_flow_cache_flag;
 383
 384         esp_or_udp:
 385           if (im->fp_spd_ipv4_in_is_enabled &&
 386               PREDICT_TRUE (INDEX_INVALID !=
 387                             spd0->fp_spd.ip4_in_lookup_hash_idx))
 388             {
 389               ipsec_fp_in_5tuple_from_ip4_range (
 390                 &tuples[0], ip0->src_address.as_u32, ip0->dst_address.as_u32,
 391                 clib_net_to_host_u32 (esp0->spi),
 392                 IPSEC_SPD_POLICY_IP4_INBOUND_PROTECT);
 393               ipsec_fp_in_policy_match_n (&spd0->fp_spd, !ip_v6, tuples,
 394                                           policies, 1);
 395               p0 = policies[0];
 396             }
 397           else if (search_flow_cache) // attempt to match policy in flow cache
 398             {
 399               p0 = ipsec4_input_spd_find_flow_cache_entry (
 400                 im, ip0->src_address.as_u32, ip0->dst_address.as_u32,
 401                 IPSEC_SPD_POLICY_IP4_INBOUND_PROTECT);
 402             }
 403
 404           else // linear search if flow cache is not enabled,
 405                // or flow cache search just failed
 406             {
 407               p0 = ipsec_input_protect_policy_match (
 408                 spd0, clib_net_to_host_u32 (ip0->src_address.as_u32),
 409                 clib_net_to_host_u32 (ip0->dst_address.as_u32),
 410                 clib_net_to_host_u32 (esp0->spi));
 411             }
 412
 413           has_space0 =
 414             vlib_buffer_has_space (b[0],
 415                                    (clib_address_t) (esp0 + 1) -
 416                                    (clib_address_t) ip0);
 417
 418           if (PREDICT_TRUE ((p0 != NULL) & (has_space0)))
 419             {
 420               ipsec_matched += 1;
 421
 422               pi0 = p0 - im->policies;
 423               vlib_increment_combined_counter
 424                 (&ipsec_spd_policy_counters,
 425                  thread_index, pi0, 1, clib_net_to_host_u16 (ip0->length));
 426
 427               vnet_buffer (b[0])->ipsec.sad_index = p0->sa_index;
 428               next[0] = im->esp4_decrypt_next_index;
 429               vlib_buffer_advance (b[0], ((u8 *) esp0 - (u8 *) ip0));
 430               goto trace0;
 431             }
 432           else
 433             {
 434               p0 = 0;
 435               pi0 = ~0;
 436             };
 437
 438           if (im->fp_spd_ipv4_in_is_enabled &&
 439               PREDICT_TRUE (INDEX_INVALID !=
 440                             spd0->fp_spd.ip4_in_lookup_hash_idx))
 441             {
 442               tuples->action = IPSEC_SPD_POLICY_IP4_INBOUND_BYPASS;
 443               ipsec_fp_in_policy_match_n (&spd0->fp_spd, !ip_v6, tuples,
 444                                           policies, 1);
 445               p0 = policies[0];
 446             }
 447           else if (search_flow_cache)
 448             {
 449               p0 = ipsec4_input_spd_find_flow_cache_entry (
 450                 im, ip0->src_address.as_u32, ip0->dst_address.as_u32,
 451                 IPSEC_SPD_POLICY_IP4_INBOUND_BYPASS);
 452             }
 453
 454           else
 455             {
 456               p0 = ipsec_input_policy_match (
 457                 spd0, clib_net_to_host_u32 (ip0->src_address.as_u32),
 458                 clib_net_to_host_u32 (ip0->dst_address.as_u32),
 459                 IPSEC_SPD_POLICY_IP4_INBOUND_BYPASS);
 460             }
 461
 462           if (PREDICT_TRUE ((p0 != NULL)))
 463             {
 464               ipsec_bypassed += 1;
 465
 466               pi0 = p0 - im->policies;
 467               vlib_increment_combined_counter (
 468                 &ipsec_spd_policy_counters, thread_index, pi0, 1,
 469                 clib_net_to_host_u16 (ip0->length));
 470
 471               goto trace0;
 472             }
 473           else
 474             {
 475               p0 = 0;
 476               pi0 = ~0;
 477             };
 478
 479           if (im->fp_spd_ipv4_in_is_enabled &&
 480               PREDICT_TRUE (INDEX_INVALID !=
 481                             spd0->fp_spd.ip4_in_lookup_hash_idx))
 482             {
 483               tuples->action = IPSEC_SPD_POLICY_IP4_INBOUND_DISCARD;
 484               ipsec_fp_in_policy_match_n (&spd0->fp_spd, !ip_v6, tuples,
 485                                           policies, 1);
 486               p0 = policies[0];
 487             }
 488           else
 489
 490             if (search_flow_cache)
 491             {
 492               p0 = ipsec4_input_spd_find_flow_cache_entry (
 493                 im, ip0->src_address.as_u32, ip0->dst_address.as_u32,
 494                 IPSEC_SPD_POLICY_IP4_INBOUND_DISCARD);
 495             }
 496
 497           else
 498             {
 499               p0 = ipsec_input_policy_match (
 500                 spd0, clib_net_to_host_u32 (ip0->src_address.as_u32),
 501                 clib_net_to_host_u32 (ip0->dst_address.as_u32),
 502                 IPSEC_SPD_POLICY_IP4_INBOUND_DISCARD);
 503             }
 504
 505           if (PREDICT_TRUE ((p0 != NULL)))
 506             {
 507               ipsec_dropped += 1;
 508
 509               pi0 = p0 - im->policies;
 510               vlib_increment_combined_counter (
 511                 &ipsec_spd_policy_counters, thread_index, pi0, 1,
 512                 clib_net_to_host_u16 (ip0->length));
 513
 514               next[0] = IPSEC_INPUT_NEXT_DROP;
 515               goto trace0;
 516             }
 517           else
 518             {
 519               p0 = 0;
 520               pi0 = ~0;
 521             };
 522
 523           // flow cache search failed, try again with linear search
 524           if (search_flow_cache && p0 == NULL)
 525             {
 526               search_flow_cache = false;
 527               goto esp_or_udp;
 528             }
 529
 530           /* Drop by default if no match on PROTECT, BYPASS or DISCARD */
 531           ipsec_unprocessed += 1;
 532           next[0] = IPSEC_INPUT_NEXT_DROP;
 533
 534         trace0:
 535           if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
 536               PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
 537             {
 538               ipsec_input_trace_t *tr =
 539                 vlib_add_trace (vm, node, b[0], sizeof (*tr));
 540
 541               tr->proto = ip0->protocol;
 542               tr->sa_id = p0 ? p0->sa_id : ~0;
 543               tr->spi = has_space0 ? clib_net_to_host_u32 (esp0->spi) : ~0;
 544               tr->seq = has_space0 ? clib_net_to_host_u32 (esp0->seq) : ~0;
 545               tr->spd = spd0->id;
 546               tr->policy_index = pi0;
 547             }
 548         }
 549       else if (ip0->protocol == IP_PROTOCOL_IPSEC_AH)
 550         {
 551           ah0 = (ah_header_t *) ((u8 *) ip0 + ip4_header_bytes (ip0));
 552
 553           // if flow cache is enabled, first search through flow cache for a
 554           // policy match and revert back to linear search on failure
 555           search_flow_cache = im->input_flow_cache_flag;
 556
 557         ah:
 558           if (search_flow_cache)
 559             {
 560               p0 = ipsec4_input_spd_find_flow_cache_entry (
 561                 im, ip0->src_address.as_u32, ip0->dst_address.as_u32,
 562                 IPSEC_SPD_POLICY_IP4_INBOUND_PROTECT);
 563             }
 564
 565           else
 566             {
 567               p0 = ipsec_input_protect_policy_match (
 568                 spd0, clib_net_to_host_u32 (ip0->src_address.as_u32),
 569                 clib_net_to_host_u32 (ip0->dst_address.as_u32),
 570                 clib_net_to_host_u32 (ah0->spi));
 571             }
 572
 573           has_space0 =
 574             vlib_buffer_has_space (b[0],
 575                                    (clib_address_t) (ah0 + 1) -
 576                                    (clib_address_t) ip0);
 577
 578           if (PREDICT_TRUE ((p0 != NULL) & (has_space0)))
 579             {
 580               ipsec_matched += 1;
 581
 582               pi0 = p0 - im->policies;
 583               vlib_increment_combined_counter
 584                 (&ipsec_spd_policy_counters,
 585                  thread_index, pi0, 1, clib_net_to_host_u16 (ip0->length));
 586
 587               vnet_buffer (b[0])->ipsec.sad_index = p0->sa_index;
 588               next[0] = im->ah4_decrypt_next_index;
 589               goto trace1;
 590             }
 591           else
 592             {
 593               p0 = 0;
 594               pi0 = ~0;
 595             }
 596
 597           if (search_flow_cache)
 598             {
 599               p0 = ipsec4_input_spd_find_flow_cache_entry (
 600                 im, ip0->src_address.as_u32, ip0->dst_address.as_u32,
 601                 IPSEC_SPD_POLICY_IP4_INBOUND_BYPASS);
 602             }
 603
 604           else
 605             {
 606               p0 = ipsec_input_policy_match (
 607                 spd0, clib_net_to_host_u32 (ip0->src_address.as_u32),
 608                 clib_net_to_host_u32 (ip0->dst_address.as_u32),
 609                 IPSEC_SPD_POLICY_IP4_INBOUND_BYPASS);
 610             }
 611
 612           if (PREDICT_TRUE ((p0 != NULL)))
 613             {
 614               ipsec_bypassed += 1;
 615
 616               pi0 = p0 - im->policies;
 617               vlib_increment_combined_counter (
 618                 &ipsec_spd_policy_counters, thread_index, pi0, 1,
 619                 clib_net_to_host_u16 (ip0->length));
 620
 621               goto trace1;
 622             }
 623           else
 624             {
 625               p0 = 0;
 626               pi0 = ~0;
 627             };
 628
 629           if (search_flow_cache)
 630             {
 631               p0 = ipsec4_input_spd_find_flow_cache_entry (
 632                 im, ip0->src_address.as_u32, ip0->dst_address.as_u32,
 633                 IPSEC_SPD_POLICY_IP4_INBOUND_DISCARD);
 634             }
 635
 636           else
 637             {
 638               p0 = ipsec_input_policy_match (
 639                 spd0, clib_net_to_host_u32 (ip0->src_address.as_u32),
 640                 clib_net_to_host_u32 (ip0->dst_address.as_u32),
 641                 IPSEC_SPD_POLICY_IP4_INBOUND_DISCARD);
 642             }
 643
 644           if (PREDICT_TRUE ((p0 != NULL)))
 645             {
 646               ipsec_dropped += 1;
 647
 648               pi0 = p0 - im->policies;
 649               vlib_increment_combined_counter (
 650                 &ipsec_spd_policy_counters, thread_index, pi0, 1,
 651                 clib_net_to_host_u16 (ip0->length));
 652
 653               next[0] = IPSEC_INPUT_NEXT_DROP;
 654               goto trace1;
 655             }
 656           else
 657             {
 658               p0 = 0;
 659               pi0 = ~0;
 660             };
 661
 662           // flow cache search failed, retry with linear search
 663           if (search_flow_cache && p0 == NULL)
 664             {
 665               search_flow_cache = false;
 666               goto ah;
 667             }
 668
 669           /* Drop by default if no match on PROTECT, BYPASS or DISCARD */
 670           ipsec_unprocessed += 1;
 671           next[0] = IPSEC_INPUT_NEXT_DROP;
 672
 673         trace1:
 674           if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
 675               PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
 676             {
 677               ipsec_input_trace_t *tr =
 678                 vlib_add_trace (vm, node, b[0], sizeof (*tr));
 679
 680               tr->proto = ip0->protocol;
 681               tr->sa_id = p0 ? p0->sa_id : ~0;
 682               tr->spi = has_space0 ? clib_net_to_host_u32 (ah0->spi) : ~0;
 683               tr->seq = has_space0 ? clib_net_to_host_u32 (ah0->seq_no) : ~0;
 684               tr->spd = spd0->id;
 685               tr->policy_index = pi0;
 686             }
 687         }
 688       else
 689         {
 690           ipsec_unprocessed += 1;
 691         }
 692       n_left_from -= 1;
 693       b += 1;
 694       next += 1;
 695     }
 696
 697   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
 698
 699   vlib_node_increment_counter (vm, ipsec4_input_node.index,
 700                                IPSEC_INPUT_ERROR_RX_PKTS, frame->n_vectors);
 701
 702   vlib_node_increment_counter (vm, ipsec4_input_node.index,
 703                                IPSEC_INPUT_ERROR_RX_POLICY_MATCH,
 704                                ipsec_matched);
 705
 706   vlib_node_increment_counter (vm, ipsec4_input_node.index,
 707                                IPSEC_INPUT_ERROR_RX_POLICY_NO_MATCH,
 708                                ipsec_unprocessed);
 709
 710   vlib_node_increment_counter (vm, ipsec4_input_node.index,
 711                                IPSEC_INPUT_ERROR_RX_POLICY_DISCARD,
 712                                ipsec_dropped);
 713
 714   vlib_node_increment_counter (vm, ipsec4_input_node.index,
 715                                IPSEC_INPUT_ERROR_RX_POLICY_BYPASS,
 716                                ipsec_bypassed);
 717
 718   return frame->n_vectors;
 719 }
 720
 721
 722 /* *INDENT-OFF* */
 723 VLIB_REGISTER_NODE (ipsec4_input_node) = {
 724   .name = "ipsec4-input-feature",
 725   .vector_size = sizeof (u32),
 726   .format_trace = format_ipsec_input_trace,
 727   .type = VLIB_NODE_TYPE_INTERNAL,
 728   .n_errors = ARRAY_LEN(ipsec_input_error_strings),
 729   .error_strings = ipsec_input_error_strings,
 730   .n_next_nodes = IPSEC_INPUT_N_NEXT,
 731   .next_nodes = {
 732 #define _(s,n) [IPSEC_INPUT_NEXT_##s] = n,
 733     foreach_ipsec_input_next
 734 #undef _
 735   },
 736 };
 737 /* *INDENT-ON* */
 738
 739 extern vlib_node_registration_t ipsec6_input_node;
 740
 741
 742 VLIB_NODE_FN (ipsec6_input_node) (vlib_main_t * vm,
 743                                   vlib_node_runtime_t * node,
 744                                   vlib_frame_t * from_frame)
 745 {
 746   u32 n_left_from, *from, next_index, *to_next, thread_index;
 747   ipsec_main_t *im = &ipsec_main;
 748   u32 ipsec_unprocessed = 0;
 749   u32 ipsec_matched = 0;
 750   ipsec_policy_t *policies[1];
 751   ipsec_fp_5tuple_t tuples[1];
 752   bool ip_v6 = true;
 753
 754   from = vlib_frame_vector_args (from_frame);
 755   n_left_from = from_frame->n_vectors;
 756   thread_index = vm->thread_index;
 757
 758   next_index = node->cached_next_index;
 759
 760   while (n_left_from > 0)
 761     {
 762       u32 n_left_to_next;
 763
 764       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 765
 766       while (n_left_from > 0 && n_left_to_next > 0)
 767         {
 768           u32 bi0, next0, pi0 = ~0;
 769           vlib_buffer_t *b0;
 770           ip6_header_t *ip0;
 771           esp_header_t *esp0;
 772           ip4_ipsec_config_t *c0;
 773           ipsec_spd_t *spd0;
 774           ipsec_policy_t *p0 = 0;
 775           ah_header_t *ah0;
 776           u32 header_size = sizeof (ip0[0]);
 777
 778           bi0 = to_next[0] = from[0];
 779           from += 1;
 780           n_left_from -= 1;
 781           to_next += 1;
 782           n_left_to_next -= 1;
 783
 784           b0 = vlib_get_buffer (vm, bi0);
 785           b0->flags |= VNET_BUFFER_F_IS_IP6;
 786           b0->flags &= ~VNET_BUFFER_F_IS_IP4;
 787           c0 = vnet_feature_next_with_data (&next0, b0, sizeof (c0[0]));
 788
 789           spd0 = pool_elt_at_index (im->spds, c0->spd_index);
 790
 791           ip0 = vlib_buffer_get_current (b0);
 792           esp0 = (esp_header_t *) ((u8 *) ip0 + header_size);
 793           ah0 = (ah_header_t *) ((u8 *) ip0 + header_size);
 794
 795           if (PREDICT_TRUE (ip0->protocol == IP_PROTOCOL_IPSEC_ESP))
 796             {
 797 #if 0
 798               clib_warning
 799                 ("packet received from %U to %U spi %u size %u spd_id %u",
 800                  format_ip6_address, &ip0->src_address, format_ip6_address,
 801                  &ip0->dst_address, clib_net_to_host_u32 (esp0->spi),
 802                  clib_net_to_host_u16 (ip0->payload_length) + header_size,
 803                  spd0->id);
 804 #endif
 805               if (im->fp_spd_ipv6_in_is_enabled &&
 806                   PREDICT_TRUE (INDEX_INVALID !=
 807                                 spd0->fp_spd.ip6_in_lookup_hash_idx))
 808                 {
 809                   ipsec_fp_in_5tuple_from_ip6_range (
 810                     &tuples[0], &ip0->src_address, &ip0->dst_address,
 811                     clib_net_to_host_u32 (esp0->spi),
 812                     IPSEC_SPD_POLICY_IP6_INBOUND_PROTECT);
 813                   ipsec_fp_in_policy_match_n (&spd0->fp_spd, ip_v6, tuples,
 814                                               policies, 1);
 815                   p0 = policies[0];
 816                 }
 817               else
 818                 p0 = ipsec6_input_protect_policy_match (
 819                   spd0, &ip0->src_address, &ip0->dst_address,
 820                   clib_net_to_host_u32 (esp0->spi));
 821
 822               if (PREDICT_TRUE (p0 != 0))
 823                 {
 824                   ipsec_matched += 1;
 825
 826                   pi0 = p0 - im->policies;
 827                   vlib_increment_combined_counter
 828                     (&ipsec_spd_policy_counters,
 829                      thread_index, pi0, 1,
 830                      clib_net_to_host_u16 (ip0->payload_length) +
 831                      header_size);
 832
 833                   vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
 834                   next0 = im->esp6_decrypt_next_index;
 835                   vlib_buffer_advance (b0, header_size);
 836                   /* TODO Add policy matching for bypass and discard policy
 837                    * type */
 838                   goto trace0;
 839                 }
 840               else
 841                 {
 842                   pi0 = ~0;
 843                   ipsec_unprocessed += 1;
 844                   next0 = IPSEC_INPUT_NEXT_DROP;
 845                 }
 846             }
 847           else if (ip0->protocol == IP_PROTOCOL_IPSEC_AH)
 848             {
 849               p0 = ipsec6_input_protect_policy_match (spd0,
 850                                                       &ip0->src_address,
 851                                                       &ip0->dst_address,
 852                                                       clib_net_to_host_u32
 853                                                       (ah0->spi));
 854
 855               if (PREDICT_TRUE (p0 != 0))
 856                 {
 857                   ipsec_matched += 1;
 858                   pi0 = p0 - im->policies;
 859                   vlib_increment_combined_counter
 860                     (&ipsec_spd_policy_counters,
 861                      thread_index, pi0, 1,
 862                      clib_net_to_host_u16 (ip0->payload_length) +
 863                      header_size);
 864
 865                   vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
 866                   next0 = im->ah6_decrypt_next_index;
 867                   goto trace0;
 868                 }
 869               else
 870                 {
 871                   pi0 = ~0;
 872                   ipsec_unprocessed += 1;
 873                   next0 = IPSEC_INPUT_NEXT_DROP;
 874                 }
 875             }
 876           else
 877             {
 878               ipsec_unprocessed += 1;
 879             }
 880
 881         trace0:
 882           if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
 883               PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
 884             {
 885               ipsec_input_trace_t *tr =
 886                 vlib_add_trace (vm, node, b0, sizeof (*tr));
 887
 888               if (p0)
 889                 {
 890                   tr->sa_id = p0->sa_id;
 891                   tr->policy_type = p0->type;
 892                 }
 893
 894               tr->proto = ip0->protocol;
 895               tr->spi = clib_net_to_host_u32 (esp0->spi);
 896               tr->seq = clib_net_to_host_u32 (esp0->seq);
 897               tr->spd = spd0->id;
 898               tr->policy_index = pi0;
 899             }
 900
 901           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 902                                            n_left_to_next, bi0, next0);
 903         }
 904       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 905     }
 906
 907   vlib_node_increment_counter (vm, ipsec6_input_node.index,
 908                                IPSEC_INPUT_ERROR_RX_PKTS,
 909                                from_frame->n_vectors - ipsec_unprocessed);
 910
 911   vlib_node_increment_counter (vm, ipsec6_input_node.index,
 912                                IPSEC_INPUT_ERROR_RX_POLICY_MATCH,
 913                                ipsec_matched);
 914
 915   return from_frame->n_vectors;
 916 }
 917
 918
 919 /* *INDENT-OFF* */
 920 VLIB_REGISTER_NODE (ipsec6_input_node) = {
 921   .name = "ipsec6-input-feature",
 922   .vector_size = sizeof (u32),
 923   .format_trace = format_ipsec_input_trace,
 924   .type = VLIB_NODE_TYPE_INTERNAL,
 925   .n_errors = ARRAY_LEN(ipsec_input_error_strings),
 926   .error_strings = ipsec_input_error_strings,
 927   .n_next_nodes = IPSEC_INPUT_N_NEXT,
 928   .next_nodes = {
 929 #define _(s,n) [IPSEC_INPUT_NEXT_##s] = n,
 930     foreach_ipsec_input_next
 931 #undef _
 932   },
 933 };
 934 /* *INDENT-ON* */
 935
 936 /*
 937  * fd.io coding-style-patch-verification: ON
 938  *
 939  * Local Variables:
 940  * eval: (c-set-style "gnu")
 941  * End:
 942  */