vnet/vnet/ipsec/esp_decrypt.c

   1 /*
   2  * esp_decrypt.c : IPSec ESP decrypt node
   3  *
   4  * Copyright (c) 2015 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/api_errno.h>
  20 #include <vnet/ip/ip.h>
  21
  22 #include <vnet/ipsec/ipsec.h>
  23 #include <vnet/ipsec/esp.h>
  24
  25 #define ESP_WINDOW_SIZE 64
  26
  27 #define foreach_esp_decrypt_next                \
  28 _(DROP, "error-drop")                           \
  29 _(IP4_INPUT, "ip4-input")                       \
  30 _(IP6_INPUT, "ip6-input")                       \
  31 _(IPSEC_GRE_INPUT, "ipsec-gre-input")
  32
  33 #define _(v, s) ESP_DECRYPT_NEXT_##v,
  34 typedef enum
  35 {
  36   foreach_esp_decrypt_next
  37 #undef _
  38     ESP_DECRYPT_N_NEXT,
  39 } esp_decrypt_next_t;
  40
  41
  42 #define foreach_esp_decrypt_error                   \
  43  _(RX_PKTS, "ESP pkts received")                    \
  44  _(NO_BUFFER, "No buffer (packed dropped)")         \
  45  _(DECRYPTION_FAILED, "ESP decryption failed")      \
  46  _(INTEG_ERROR, "Integrity check failed")           \
  47  _(REPLAY, "SA replayed packet")                    \
  48  _(NOT_IP, "Not IP packet (dropped)")
  49
  50
  51 typedef enum
  52 {
  53 #define _(sym,str) ESP_DECRYPT_ERROR_##sym,
  54   foreach_esp_decrypt_error
  55 #undef _
  56     ESP_DECRYPT_N_ERROR,
  57 } esp_decrypt_error_t;
  58
  59 static char *esp_decrypt_error_strings[] = {
  60 #define _(sym,string) string,
  61   foreach_esp_decrypt_error
  62 #undef _
  63 };
  64
  65 typedef struct
  66 {
  67   ipsec_crypto_alg_t crypto_alg;
  68   ipsec_integ_alg_t integ_alg;
  69 } esp_decrypt_trace_t;
  70
  71 /* packet trace format function */
  72 static u8 *
  73 format_esp_decrypt_trace (u8 * s, va_list * args)
  74 {
  75   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  76   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  77   esp_decrypt_trace_t *t = va_arg (*args, esp_decrypt_trace_t *);
  78
  79   s = format (s, "esp: crypto %U integrity %U",
  80               format_ipsec_crypto_alg, t->crypto_alg,
  81               format_ipsec_integ_alg, t->integ_alg);
  82   return s;
  83 }
  84
  85 always_inline void
  86 esp_decrypt_aes_cbc (ipsec_crypto_alg_t alg,
  87                      u8 * in, u8 * out, size_t in_len, u8 * key, u8 * iv)
  88 {
  89   esp_main_t *em = &esp_main;
  90   u32 cpu_index = os_get_cpu_number ();
  91   EVP_CIPHER_CTX *ctx = &(em->per_thread_data[cpu_index].decrypt_ctx);
  92   const EVP_CIPHER *cipher = NULL;
  93   int out_len;
  94
  95   ASSERT (alg < IPSEC_CRYPTO_N_ALG);
  96
  97   if (PREDICT_FALSE (em->esp_crypto_algs[alg].type == 0))
  98     return;
  99
 100   if (PREDICT_FALSE (alg != em->per_thread_data[cpu_index].last_decrypt_alg))
 101     {
 102       cipher = em->esp_crypto_algs[alg].type;
 103       em->per_thread_data[cpu_index].last_decrypt_alg = alg;
 104     }
 105
 106   EVP_DecryptInit_ex (ctx, cipher, NULL, key, iv);
 107
 108   EVP_DecryptUpdate (ctx, out, &out_len, in, in_len);
 109   EVP_DecryptFinal_ex (ctx, out + out_len, &out_len);
 110 }
 111
 112 always_inline int
 113 esp_replay_check (ipsec_sa_t * sa, u32 seq)
 114 {
 115   u32 diff;
 116
 117   if (PREDICT_TRUE (seq > sa->last_seq))
 118     return 0;
 119
 120   diff = sa->last_seq - seq;
 121
 122   if (ESP_WINDOW_SIZE > diff)
 123     return (sa->replay_window & (1ULL << diff)) ? 1 : 0;
 124   else
 125     return 1;
 126
 127   return 0;
 128 }
 129
 130 always_inline int
 131 esp_replay_check_esn (ipsec_sa_t * sa, u32 seq)
 132 {
 133   u32 tl = sa->last_seq;
 134   u32 th = sa->last_seq_hi;
 135   u32 diff = tl - seq;
 136
 137   if (PREDICT_TRUE (tl >= (ESP_WINDOW_SIZE - 1)))
 138     {
 139       if (seq >= (tl - ESP_WINDOW_SIZE + 1))
 140         {
 141           sa->seq_hi = th;
 142           if (seq <= tl)
 143             return (sa->replay_window & (1ULL << diff)) ? 1 : 0;
 144           else
 145             return 0;
 146         }
 147       else
 148         {
 149           sa->seq_hi = th + 1;
 150           return 0;
 151         }
 152     }
 153   else
 154     {
 155       if (seq >= (tl - ESP_WINDOW_SIZE + 1))
 156         {
 157           sa->seq_hi = th - 1;
 158           return (sa->replay_window & (1ULL << diff)) ? 1 : 0;
 159         }
 160       else
 161         {
 162           sa->seq_hi = th;
 163           if (seq <= tl)
 164             return (sa->replay_window & (1ULL << diff)) ? 1 : 0;
 165           else
 166             return 0;
 167         }
 168     }
 169
 170   return 0;
 171 }
 172
 173 always_inline void
 174 esp_replay_advance (ipsec_sa_t * sa, u32 seq)
 175 {
 176   u32 pos;
 177
 178   if (seq > sa->last_seq)
 179     {
 180       pos = seq - sa->last_seq;
 181       if (pos < ESP_WINDOW_SIZE)
 182         sa->replay_window = ((sa->replay_window) << pos) | 1;
 183       else
 184         sa->replay_window = 1;
 185       sa->last_seq = seq;
 186     }
 187   else
 188     {
 189       pos = sa->last_seq - seq;
 190       sa->replay_window |= (1ULL << pos);
 191     }
 192 }
 193
 194 always_inline void
 195 esp_replay_advance_esn (ipsec_sa_t * sa, u32 seq)
 196 {
 197   int wrap = sa->seq_hi - sa->last_seq_hi;
 198   u32 pos;
 199
 200   if (wrap == 0 && seq > sa->last_seq)
 201     {
 202       pos = seq - sa->last_seq;
 203       if (pos < ESP_WINDOW_SIZE)
 204         sa->replay_window = ((sa->replay_window) << pos) | 1;
 205       else
 206         sa->replay_window = 1;
 207       sa->last_seq = seq;
 208     }
 209   else if (wrap > 0)
 210     {
 211       pos = ~seq + sa->last_seq + 1;
 212       if (pos < ESP_WINDOW_SIZE)
 213         sa->replay_window = ((sa->replay_window) << pos) | 1;
 214       else
 215         sa->replay_window = 1;
 216       sa->last_seq = seq;
 217       sa->last_seq_hi = sa->seq_hi;
 218     }
 219   else if (wrap < 0)
 220     {
 221       pos = ~seq + sa->last_seq + 1;
 222       sa->replay_window |= (1ULL << pos);
 223     }
 224   else
 225     {
 226       pos = sa->last_seq - seq;
 227       sa->replay_window |= (1ULL << pos);
 228     }
 229 }
 230
 231 static uword
 232 esp_decrypt_node_fn (vlib_main_t * vm,
 233                      vlib_node_runtime_t * node, vlib_frame_t * from_frame)
 234 {
 235   u32 n_left_from, *from, next_index, *to_next;
 236   ipsec_main_t *im = &ipsec_main;
 237   esp_main_t *em = &esp_main;
 238   u32 *recycle = 0;
 239   from = vlib_frame_vector_args (from_frame);
 240   n_left_from = from_frame->n_vectors;
 241   u32 cpu_index = os_get_cpu_number ();
 242
 243   ipsec_alloc_empty_buffers (vm, im);
 244
 245   u32 *empty_buffers = im->empty_buffers[cpu_index];
 246
 247   if (PREDICT_FALSE (vec_len (empty_buffers) < n_left_from))
 248     {
 249       vlib_node_increment_counter (vm, esp_decrypt_node.index,
 250                                    ESP_DECRYPT_ERROR_NO_BUFFER, n_left_from);
 251       goto free_buffers_and_exit;
 252     }
 253
 254   next_index = node->cached_next_index;
 255
 256   while (n_left_from > 0)
 257     {
 258       u32 n_left_to_next;
 259
 260       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 261
 262       while (n_left_from > 0 && n_left_to_next > 0)
 263         {
 264           u32 i_bi0, o_bi0 = (u32) ~ 0, next0;
 265           vlib_buffer_t *i_b0;
 266           vlib_buffer_t *o_b0 = 0;
 267           esp_header_t *esp0;
 268           ipsec_sa_t *sa0;
 269           u32 sa_index0 = ~0;
 270           u32 seq;
 271           ip4_header_t *ih4 = 0, *oh4 = 0;
 272           ip6_header_t *ih6 = 0, *oh6 = 0;
 273           u8 tunnel_mode = 1;
 274           u8 transport_ip6 = 0;
 275
 276
 277           i_bi0 = from[0];
 278           from += 1;
 279           n_left_from -= 1;
 280           n_left_to_next -= 1;
 281
 282           next0 = ESP_DECRYPT_NEXT_DROP;
 283
 284           i_b0 = vlib_get_buffer (vm, i_bi0);
 285           esp0 = vlib_buffer_get_current (i_b0);
 286
 287           sa_index0 = vnet_buffer (i_b0)->output_features.ipsec_sad_index;
 288           sa0 = pool_elt_at_index (im->sad, sa_index0);
 289
 290           seq = clib_host_to_net_u32 (esp0->seq);
 291
 292           /* anti-replay check */
 293           if (sa0->use_anti_replay)
 294             {
 295               int rv = 0;
 296
 297               if (PREDICT_TRUE (sa0->use_esn))
 298                 rv = esp_replay_check_esn (sa0, seq);
 299               else
 300                 rv = esp_replay_check (sa0, seq);
 301
 302               if (PREDICT_FALSE (rv))
 303                 {
 304                   clib_warning ("anti-replay SPI %u seq %u", sa0->spi, seq);
 305                   vlib_node_increment_counter (vm, esp_decrypt_node.index,
 306                                                ESP_DECRYPT_ERROR_REPLAY, 1);
 307                   o_bi0 = i_bi0;
 308                   to_next[0] = o_bi0;
 309                   to_next += 1;
 310                   goto trace;
 311                 }
 312             }
 313
 314           if (PREDICT_TRUE (sa0->integ_alg != IPSEC_INTEG_ALG_NONE))
 315             {
 316               u8 sig[64];
 317               int icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size;
 318               memset (sig, 0, sizeof (sig));
 319               u8 *icv =
 320                 vlib_buffer_get_current (i_b0) + i_b0->current_length -
 321                 icv_size;
 322               i_b0->current_length -= icv_size;
 323
 324               hmac_calc (sa0->integ_alg, sa0->integ_key, sa0->integ_key_len,
 325                          (u8 *) esp0, i_b0->current_length, sig, sa0->use_esn,
 326                          sa0->seq_hi);
 327
 328               if (PREDICT_FALSE (memcmp (icv, sig, icv_size)))
 329                 {
 330                   vlib_node_increment_counter (vm, esp_decrypt_node.index,
 331                                                ESP_DECRYPT_ERROR_INTEG_ERROR,
 332                                                1);
 333                   o_bi0 = i_bi0;
 334                   to_next[0] = o_bi0;
 335                   to_next += 1;
 336                   goto trace;
 337                 }
 338             }
 339
 340           if (PREDICT_TRUE (sa0->use_anti_replay))
 341             {
 342               if (PREDICT_TRUE (sa0->use_esn))
 343                 esp_replay_advance_esn (sa0, seq);
 344               else
 345                 esp_replay_advance (sa0, seq);
 346             }
 347
 348           /* grab free buffer */
 349           uword last_empty_buffer = vec_len (empty_buffers) - 1;
 350           o_bi0 = empty_buffers[last_empty_buffer];
 351           to_next[0] = o_bi0;
 352           to_next += 1;
 353           o_b0 = vlib_get_buffer (vm, o_bi0);
 354           vlib_prefetch_buffer_with_index (vm,
 355                                            empty_buffers[last_empty_buffer -
 356                                                          1], STORE);
 357           _vec_len (empty_buffers) = last_empty_buffer;
 358
 359           /* add old buffer to the recycle list */
 360           vec_add1 (recycle, i_bi0);
 361
 362           if (sa0->crypto_alg >= IPSEC_CRYPTO_ALG_AES_CBC_128 &&
 363               sa0->crypto_alg <= IPSEC_CRYPTO_ALG_AES_CBC_256)
 364             {
 365               const int BLOCK_SIZE = 16;
 366               const int IV_SIZE = 16;
 367               esp_footer_t *f0;
 368               u8 ip_hdr_size = 0;
 369
 370               int blocks =
 371                 (i_b0->current_length - sizeof (esp_header_t) -
 372                  IV_SIZE) / BLOCK_SIZE;
 373
 374               o_b0->current_data = sizeof (ethernet_header_t);
 375
 376               /* transport mode */
 377               if (PREDICT_FALSE (!sa0->is_tunnel && !sa0->is_tunnel_ip6))
 378                 {
 379                   tunnel_mode = 0;
 380                   ih4 =
 381                     (ip4_header_t *) (i_b0->data +
 382                                       sizeof (ethernet_header_t));
 383                   if (PREDICT_TRUE
 384                       ((ih4->ip_version_and_header_length & 0xF0) != 0x40))
 385                     {
 386                       if (PREDICT_TRUE
 387                           ((ih4->ip_version_and_header_length & 0xF0) ==
 388                            0x60))
 389                         {
 390                           transport_ip6 = 1;
 391                           ip_hdr_size = sizeof (ip6_header_t);
 392                           ih6 =
 393                             (ip6_header_t *) (i_b0->data +
 394                                               sizeof (ethernet_header_t));
 395                           oh6 = vlib_buffer_get_current (o_b0);
 396                         }
 397                       else
 398                         {
 399                           vlib_node_increment_counter (vm,
 400                                                        esp_decrypt_node.index,
 401                                                        ESP_DECRYPT_ERROR_NOT_IP,
 402                                                        1);
 403                           o_b0 = 0;
 404                           goto trace;
 405                         }
 406                     }
 407                   else
 408                     {
 409                       oh4 = vlib_buffer_get_current (o_b0);
 410                       ip_hdr_size = sizeof (ip4_header_t);
 411                     }
 412                 }
 413
 414               esp_decrypt_aes_cbc (sa0->crypto_alg,
 415                                    esp0->data + IV_SIZE,
 416                                    (u8 *) vlib_buffer_get_current (o_b0) +
 417                                    ip_hdr_size, BLOCK_SIZE * blocks,
 418                                    sa0->crypto_key, esp0->data);
 419
 420               o_b0->current_length = (blocks * 16) - 2 + ip_hdr_size;
 421               o_b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
 422               f0 =
 423                 (esp_footer_t *) ((u8 *) vlib_buffer_get_current (o_b0) +
 424                                   o_b0->current_length);
 425               o_b0->current_length -= f0->pad_length;
 426
 427               /* tunnel mode */
 428               if (PREDICT_TRUE (tunnel_mode))
 429                 {
 430                   if (PREDICT_TRUE (f0->next_header == IP_PROTOCOL_IP_IN_IP))
 431                     {
 432                       next0 = ESP_DECRYPT_NEXT_IP4_INPUT;
 433                       oh4 = vlib_buffer_get_current (o_b0);
 434                     }
 435                   else if (f0->next_header == IP_PROTOCOL_IPV6)
 436                     next0 = ESP_DECRYPT_NEXT_IP6_INPUT;
 437                   else
 438                     {
 439                       clib_warning ("next header: 0x%x", f0->next_header);
 440                       vlib_node_increment_counter (vm, esp_decrypt_node.index,
 441                                                    ESP_DECRYPT_ERROR_DECRYPTION_FAILED,
 442                                                    1);
 443                       o_b0 = 0;
 444                       goto trace;
 445                     }
 446                 }
 447               /* transport mode */
 448               else
 449                 {
 450                   if (PREDICT_FALSE (transport_ip6))
 451                     {
 452                       next0 = ESP_DECRYPT_NEXT_IP6_INPUT;
 453                       oh6->ip_version_traffic_class_and_flow_label =
 454                         ih6->ip_version_traffic_class_and_flow_label;
 455                       oh6->protocol = f0->next_header;
 456                       oh6->hop_limit = ih6->hop_limit;
 457                       oh6->src_address.as_u64[0] = ih6->src_address.as_u64[0];
 458                       oh6->src_address.as_u64[1] = ih6->src_address.as_u64[1];
 459                       oh6->dst_address.as_u64[0] = ih6->dst_address.as_u64[0];
 460                       oh6->dst_address.as_u64[1] = ih6->dst_address.as_u64[1];
 461                       oh6->payload_length =
 462                         clib_host_to_net_u16 (vlib_buffer_length_in_chain
 463                                               (vm,
 464                                                o_b0) - sizeof (ip6_header_t));
 465                     }
 466                   else
 467                     {
 468                       next0 = ESP_DECRYPT_NEXT_IP4_INPUT;
 469                       oh4->ip_version_and_header_length = 0x45;
 470                       oh4->tos = ih4->tos;
 471                       oh4->fragment_id = 0;
 472                       oh4->flags_and_fragment_offset = 0;
 473                       oh4->ttl = ih4->ttl;
 474                       oh4->protocol = f0->next_header;
 475                       oh4->src_address.as_u32 = ih4->src_address.as_u32;
 476                       oh4->dst_address.as_u32 = ih4->dst_address.as_u32;
 477                       oh4->length =
 478                         clib_host_to_net_u16 (vlib_buffer_length_in_chain
 479                                               (vm, o_b0));
 480                       oh4->checksum = ip4_header_checksum (oh4);
 481                     }
 482                 }
 483
 484               /* for IPSec-GRE tunnel next node is ipsec-gre-input */
 485               if (PREDICT_FALSE
 486                   ((vnet_buffer (i_b0)->output_features.ipsec_flags) &
 487                    IPSEC_FLAG_IPSEC_GRE_TUNNEL))
 488                 next0 = ESP_DECRYPT_NEXT_IPSEC_GRE_INPUT;
 489
 490               vnet_buffer (o_b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 491             }
 492
 493         trace:
 494           if (PREDICT_FALSE (i_b0->flags & VLIB_BUFFER_IS_TRACED))
 495             {
 496               if (o_b0)
 497                 {
 498                   o_b0->flags |= VLIB_BUFFER_IS_TRACED;
 499                   o_b0->trace_index = i_b0->trace_index;
 500                   esp_decrypt_trace_t *tr =
 501                     vlib_add_trace (vm, node, o_b0, sizeof (*tr));
 502                   tr->crypto_alg = sa0->crypto_alg;
 503                   tr->integ_alg = sa0->integ_alg;
 504                 }
 505             }
 506
 507           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 508                                            n_left_to_next, o_bi0, next0);
 509         }
 510       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 511     }
 512   vlib_node_increment_counter (vm, esp_decrypt_node.index,
 513                                ESP_DECRYPT_ERROR_RX_PKTS,
 514                                from_frame->n_vectors);
 515
 516 free_buffers_and_exit:
 517   if (recycle)
 518     vlib_buffer_free (vm, recycle, vec_len (recycle));
 519   vec_free (recycle);
 520   return from_frame->n_vectors;
 521 }
 522
 523
 524 /* *INDENT-OFF* */
 525 VLIB_REGISTER_NODE (esp_decrypt_node) = {
 526   .function = esp_decrypt_node_fn,
 527   .name = "esp-decrypt",
 528   .vector_size = sizeof (u32),
 529   .format_trace = format_esp_decrypt_trace,
 530   .type = VLIB_NODE_TYPE_INTERNAL,
 531
 532   .n_errors = ARRAY_LEN(esp_decrypt_error_strings),
 533   .error_strings = esp_decrypt_error_strings,
 534
 535   .n_next_nodes = ESP_DECRYPT_N_NEXT,
 536   .next_nodes = {
 537 #define _(s,n) [ESP_DECRYPT_NEXT_##s] = n,
 538     foreach_esp_decrypt_next
 539 #undef _
 540   },
 541 };
 542 /* *INDENT-ON* */
 543
 544 VLIB_NODE_FUNCTION_MULTIARCH (esp_decrypt_node, esp_decrypt_node_fn)
 545 /*
 546  * fd.io coding-style-patch-verification: ON
 547  *
 548  * Local Variables:
 549  * eval: (c-set-style "gnu")
 550  * End:
 551  */