vnet/vnet/ipsec/esp_decrypt.c

   1 /*
   2  * esp_decrypt.c : IPSec ESP decrypt node
   3  *
   4  * Copyright (c) 2015 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/api_errno.h>
  20 #include <vnet/ip/ip.h>
  21
  22 #include <vnet/ipsec/ipsec.h>
  23 #include <vnet/ipsec/esp.h>
  24
  25 #define ESP_WINDOW_SIZE 64
  26
  27 #define foreach_esp_decrypt_next                \
  28 _(DROP, "error-drop")                           \
  29 _(IP4_INPUT, "ip4-input")                       \
  30 _(IP6_INPUT, "ip6-input")
  31
  32 #define _(v, s) ESP_DECRYPT_NEXT_##v,
  33 typedef enum {
  34   foreach_esp_decrypt_next
  35 #undef _
  36   ESP_DECRYPT_N_NEXT,
  37 } esp_decrypt_next_t;
  38
  39
  40 #define foreach_esp_decrypt_error                   \
  41  _(RX_PKTS, "ESP pkts received")                    \
  42  _(NO_BUFFER, "No buffer (packed dropped)")         \
  43  _(DECRYPTION_FAILED, "ESP decryption failed")      \
  44  _(INTEG_ERROR, "Integrity check failed")           \
  45  _(REPLAY, "SA replayed packet")                    \
  46  _(NOT_IP, "Not IP packet (dropped)")
  47
  48
  49 typedef enum {
  50 #define _(sym,str) ESP_DECRYPT_ERROR_##sym,
  51   foreach_esp_decrypt_error
  52 #undef _
  53   ESP_DECRYPT_N_ERROR,
  54 } esp_decrypt_error_t;
  55
  56 static char * esp_decrypt_error_strings[] = {
  57 #define _(sym,string) string,
  58   foreach_esp_decrypt_error
  59 #undef _
  60 };
  61
  62 typedef struct {
  63   ipsec_crypto_alg_t crypto_alg;
  64   ipsec_integ_alg_t integ_alg;
  65 } esp_decrypt_trace_t;
  66
  67 /* packet trace format function */
  68 static u8 * format_esp_decrypt_trace (u8 * s, va_list * args)
  69 {
  70   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  71   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  72   esp_decrypt_trace_t * t = va_arg (*args, esp_decrypt_trace_t *);
  73
  74   s = format (s, "esp: crypto %U integrity %U",
  75               format_ipsec_crypto_alg, t->crypto_alg,
  76               format_ipsec_integ_alg, t->integ_alg);
  77   return s;
  78 }
  79
  80 always_inline void
  81 esp_decrypt_aes_cbc(ipsec_crypto_alg_t alg,
  82                     u8 * in,
  83                     u8 * out,
  84                     size_t in_len,
  85                     u8 * key,
  86                     u8 * iv)
  87 {
  88   esp_main_t * em = &esp_main;
  89   u32 cpu_index = os_get_cpu_number();
  90   EVP_CIPHER_CTX * ctx = &(em->per_thread_data[cpu_index].decrypt_ctx);
  91   const EVP_CIPHER * cipher = NULL;
  92   int out_len;
  93
  94   ASSERT(alg < IPSEC_CRYPTO_N_ALG);
  95
  96   if (PREDICT_FALSE(em->esp_crypto_algs[alg].type == 0))
  97     return;
  98
  99   if (PREDICT_FALSE(alg != em->per_thread_data[cpu_index].last_decrypt_alg)) {
 100     cipher = em->esp_crypto_algs[alg].type;
 101     em->per_thread_data[cpu_index].last_decrypt_alg = alg;
 102   }
 103
 104   EVP_DecryptInit_ex(ctx, cipher, NULL, key, iv);
 105
 106   EVP_DecryptUpdate(ctx, out, &out_len, in, in_len);
 107   EVP_DecryptFinal_ex(ctx, out + out_len, &out_len);
 108 }
 109
 110 always_inline int
 111 esp_replay_check (ipsec_sa_t * sa, u32 seq)
 112 {
 113   u32 diff;
 114
 115   if (PREDICT_TRUE(seq > sa->last_seq))
 116     return 0;
 117
 118   diff = sa->last_seq - seq;
 119
 120   if (ESP_WINDOW_SIZE > diff)
 121     return (sa->replay_window & (1ULL << diff)) ? 1 : 0;
 122   else
 123     return 1;
 124
 125   return 0;
 126 }
 127
 128 always_inline int
 129 esp_replay_check_esn (ipsec_sa_t * sa, u32 seq)
 130 {
 131   u32 tl = sa->last_seq;
 132   u32 th = sa->last_seq_hi;
 133   u32 diff = tl - seq;
 134
 135   if (PREDICT_TRUE(tl >= (ESP_WINDOW_SIZE - 1)))
 136     {
 137       if (seq >= (tl - ESP_WINDOW_SIZE + 1))
 138         {
 139           sa->seq_hi = th;
 140           if (seq <= tl)
 141             return (sa->replay_window & (1ULL << diff)) ? 1 : 0;
 142           else
 143             return 0;
 144         }
 145       else
 146         {
 147           sa->seq_hi = th + 1;
 148           return 0;
 149         }
 150     }
 151   else
 152     {
 153       if (seq >= (tl - ESP_WINDOW_SIZE + 1))
 154         {
 155           sa->seq_hi = th - 1;
 156           return (sa->replay_window & (1ULL << diff)) ? 1 : 0;
 157         }
 158       else
 159         {
 160           sa->seq_hi = th;
 161           if (seq <= tl)
 162             return (sa->replay_window & (1ULL << diff)) ? 1 : 0;
 163           else
 164             return 0;
 165         }
 166     }
 167
 168   return 0;
 169 }
 170
 171 always_inline void
 172 esp_replay_advance (ipsec_sa_t * sa, u32 seq)
 173 {
 174   u32 pos;
 175
 176   if (seq > sa->last_seq)
 177     {
 178       pos = seq - sa->last_seq;
 179       if (pos < ESP_WINDOW_SIZE)
 180         sa->replay_window = ((sa->replay_window) << pos) | 1;
 181       else
 182         sa->replay_window = 1;
 183       sa->last_seq = seq;
 184     }
 185   else
 186     {
 187       pos = sa->last_seq - seq;
 188       sa->replay_window |= (1ULL << pos);
 189     }
 190 }
 191
 192 always_inline void
 193 esp_replay_advance_esn (ipsec_sa_t * sa, u32 seq)
 194 {
 195   int wrap = sa->seq_hi - sa->last_seq_hi;
 196   u32 pos;
 197
 198   if (wrap == 0 && seq > sa->last_seq)
 199     {
 200       pos = seq - sa->last_seq;
 201       if (pos < ESP_WINDOW_SIZE)
 202         sa->replay_window = ((sa->replay_window) << pos) | 1;
 203       else
 204         sa->replay_window = 1;
 205       sa->last_seq = seq;
 206     }
 207   else if (wrap > 0)
 208     {
 209       pos = ~seq + sa->last_seq + 1;
 210       if (pos < ESP_WINDOW_SIZE)
 211         sa->replay_window = ((sa->replay_window) << pos) | 1;
 212       else
 213         sa->replay_window = 1;
 214       sa->last_seq = seq;
 215       sa->last_seq_hi = sa->seq_hi;
 216     }
 217   else if (wrap < 0)
 218     {
 219       pos = ~seq + sa->last_seq + 1;
 220       sa->replay_window |= (1ULL << pos);
 221     }
 222   else
 223     {
 224       pos = sa->last_seq - seq;
 225       sa->replay_window |= (1ULL << pos);
 226     }
 227 }
 228
 229 static uword
 230 esp_decrypt_node_fn (vlib_main_t * vm,
 231                      vlib_node_runtime_t * node,
 232                      vlib_frame_t * from_frame)
 233 {
 234   u32 n_left_from, *from, next_index, *to_next;
 235   ipsec_main_t *im = &ipsec_main;
 236   esp_main_t *em = &esp_main;
 237   u32 * recycle = 0;
 238   from = vlib_frame_vector_args (from_frame);
 239   n_left_from = from_frame->n_vectors;
 240   u32 cpu_index = os_get_cpu_number();
 241
 242   ipsec_alloc_empty_buffers(vm, im);
 243
 244   u32 * empty_buffers = im->empty_buffers[cpu_index];
 245
 246   if (PREDICT_FALSE(vec_len (empty_buffers) < n_left_from)){
 247     vlib_node_increment_counter (vm, esp_decrypt_node.index,
 248                                  ESP_DECRYPT_ERROR_NO_BUFFER, n_left_from);
 249     goto free_buffers_and_exit;
 250   }
 251
 252   next_index = node->cached_next_index;
 253
 254   while (n_left_from > 0)
 255     {
 256       u32 n_left_to_next;
 257
 258       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 259
 260       while (n_left_from > 0 && n_left_to_next > 0)
 261         {
 262           u32 i_bi0, o_bi0 = (u32) ~0, next0;
 263           vlib_buffer_t * i_b0;
 264           vlib_buffer_t * o_b0 = 0;
 265           esp_header_t * esp0;
 266           ipsec_sa_t * sa0;
 267           u32 sa_index0 = ~0;
 268           u32 seq;
 269           ip4_header_t *ih4 = 0, *oh4 = 0;
 270           ip6_header_t *ih6 = 0, *oh6 = 0;
 271           u8 tunnel_mode = 1;
 272           u8 transport_ip6 = 0;
 273
 274
 275           i_bi0 = from[0];
 276           from += 1;
 277           n_left_from -= 1;
 278           n_left_to_next -= 1;
 279
 280           next0 = ESP_DECRYPT_NEXT_DROP;
 281
 282           i_b0 = vlib_get_buffer (vm, i_bi0);
 283           esp0 = vlib_buffer_get_current (i_b0);
 284
 285           sa_index0 = vnet_buffer(i_b0)->output_features.ipsec_sad_index;
 286           sa0 = pool_elt_at_index (im->sad, sa_index0);
 287
 288           seq = clib_host_to_net_u32(esp0->seq);
 289
 290           /* anti-replay check */
 291           if (sa0->use_anti_replay)
 292             {
 293               int rv = 0;
 294
 295               if (PREDICT_TRUE(sa0->use_esn))
 296                 rv = esp_replay_check_esn(sa0, seq);
 297               else
 298                 rv = esp_replay_check(sa0, seq);
 299
 300               if (PREDICT_FALSE(rv))
 301                 {
 302                   clib_warning("anti-replay SPI %u seq %u", sa0->spi, seq);
 303                   vlib_node_increment_counter (vm, esp_decrypt_node.index,
 304                                                ESP_DECRYPT_ERROR_REPLAY, 1);
 305                   o_bi0 = i_bi0;
 306                   goto trace;
 307                 }
 308             }
 309
 310           if (PREDICT_TRUE(sa0->integ_alg != IPSEC_INTEG_ALG_NONE))
 311             {
 312               u8 sig[64];
 313               int icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size;
 314               memset(sig, 0, sizeof(sig));
 315               u8 * icv = vlib_buffer_get_current (i_b0) + i_b0->current_length - icv_size;
 316               i_b0->current_length -= icv_size;
 317
 318               hmac_calc(sa0->integ_alg, sa0->integ_key, sa0->integ_key_len,
 319                         (u8 *) esp0, i_b0->current_length, sig, sa0->use_esn,
 320                         sa0->seq_hi);
 321
 322               if (PREDICT_FALSE(memcmp(icv, sig, icv_size)))
 323                 {
 324                   vlib_node_increment_counter (vm, esp_decrypt_node.index,
 325                                                ESP_DECRYPT_ERROR_INTEG_ERROR, 1);
 326                   o_bi0 = i_bi0;
 327                   goto trace;
 328                 }
 329             }
 330
 331           if (PREDICT_TRUE(sa0->use_anti_replay))
 332             {
 333               if (PREDICT_TRUE(sa0->use_esn))
 334                 esp_replay_advance_esn(sa0, seq);
 335               else
 336                 esp_replay_advance(sa0, seq);
 337              }
 338
 339           /* grab free buffer */
 340           uword last_empty_buffer = vec_len (empty_buffers) - 1;
 341           o_bi0 = empty_buffers[last_empty_buffer];
 342           o_b0 = vlib_get_buffer (vm, o_bi0);
 343           vlib_prefetch_buffer_with_index (vm, empty_buffers[last_empty_buffer-1], STORE);
 344           _vec_len (empty_buffers) = last_empty_buffer;
 345
 346           /* add old buffer to the recycle list */
 347           vec_add1(recycle, i_bi0);
 348
 349           if (sa0->crypto_alg >= IPSEC_CRYPTO_ALG_AES_CBC_128 &&
 350               sa0->crypto_alg <= IPSEC_CRYPTO_ALG_AES_CBC_256) {
 351             const int BLOCK_SIZE = 16;
 352             const int IV_SIZE = 16;
 353             esp_footer_t * f0;
 354             u8 ip_hdr_size = 0;
 355
 356             int blocks = (i_b0->current_length - sizeof (esp_header_t) - IV_SIZE) / BLOCK_SIZE;
 357
 358             o_b0->current_data = sizeof(ethernet_header_t);
 359
 360             /* transport mode */
 361             if (PREDICT_FALSE(!sa0->is_tunnel && !sa0->is_tunnel_ip6))
 362               {
 363                 tunnel_mode = 0;
 364                 ih4 = (ip4_header_t *) (i_b0->data + sizeof(ethernet_header_t));
 365                 if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0 ) != 0x40))
 366                   {
 367                     if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0 ) == 0x60))
 368                       {
 369                         transport_ip6 = 1;
 370                         ip_hdr_size = sizeof(ip6_header_t);
 371                         ih6 = (ip6_header_t *) (i_b0->data + sizeof(ethernet_header_t));
 372                         oh6 = vlib_buffer_get_current (o_b0);
 373                       }
 374                     else
 375                       {
 376                         vlib_node_increment_counter (vm, esp_decrypt_node.index,
 377                                                      ESP_DECRYPT_ERROR_NOT_IP,
 378                                                      1);
 379                         o_b0 = 0;
 380                         goto trace;
 381                       }
 382                   }
 383                 else
 384                   {
 385                     oh4 = vlib_buffer_get_current (o_b0);
 386                     ip_hdr_size = sizeof(ip4_header_t);
 387                   }
 388               }
 389
 390             esp_decrypt_aes_cbc(sa0->crypto_alg,
 391                                 esp0->data + IV_SIZE,
 392                                 (u8 *) vlib_buffer_get_current (o_b0) + ip_hdr_size,
 393                                 BLOCK_SIZE * blocks,
 394                                 sa0->crypto_key,
 395                                 esp0->data);
 396
 397             o_b0->current_length = (blocks * 16) - 2 + ip_hdr_size;
 398             o_b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
 399             f0 = (esp_footer_t *) ((u8 *) vlib_buffer_get_current (o_b0) + o_b0->current_length);
 400             o_b0->current_length -= f0->pad_length;
 401
 402             /* tunnel mode */
 403             if (PREDICT_TRUE(tunnel_mode))
 404               {
 405                 if (PREDICT_TRUE(f0->next_header == IP_PROTOCOL_IP_IN_IP))
 406                   next0 = ESP_DECRYPT_NEXT_IP4_INPUT;
 407                 else if (f0->next_header == IP_PROTOCOL_IPV6)
 408                   next0 = ESP_DECRYPT_NEXT_IP6_INPUT;
 409                 else
 410                   {
 411                     clib_warning("next header: 0x%x", f0->next_header);
 412                     vlib_node_increment_counter (vm, esp_decrypt_node.index,
 413                                                  ESP_DECRYPT_ERROR_DECRYPTION_FAILED,
 414                                                  1);
 415                     o_b0 = 0;
 416                     goto trace;
 417                   }
 418               }
 419             /* transport mode */
 420             else
 421               {
 422                 if (PREDICT_FALSE(transport_ip6))
 423                   {
 424                     next0 = ESP_DECRYPT_NEXT_IP6_INPUT;
 425                     oh6->ip_version_traffic_class_and_flow_label =
 426                         ih6->ip_version_traffic_class_and_flow_label;
 427                     oh6->protocol = f0->next_header;
 428                     oh6->hop_limit = ih6->hop_limit;
 429                     oh6->src_address.as_u64[0] = ih6->src_address.as_u64[0];
 430                     oh6->src_address.as_u64[1] = ih6->src_address.as_u64[1];
 431                     oh6->dst_address.as_u64[0] = ih6->dst_address.as_u64[0];
 432                     oh6->dst_address.as_u64[1] = ih6->dst_address.as_u64[1];
 433                     oh6->payload_length = clib_host_to_net_u16 (
 434                         vlib_buffer_length_in_chain (vm, o_b0) - sizeof(ip6_header_t));
 435                   }
 436                 else
 437                   {
 438                     next0 = ESP_DECRYPT_NEXT_IP4_INPUT;
 439                     oh4->ip_version_and_header_length = 0x45;
 440                     oh4->tos = ih4->tos;
 441                     oh4->fragment_id = 0;
 442                     oh4->flags_and_fragment_offset = 0;
 443                     oh4->ttl = ih4->ttl;
 444                     oh4->protocol = f0->next_header;
 445                     oh4->src_address.as_u32 = ih4->src_address.as_u32;
 446                     oh4->dst_address.as_u32 = ih4->dst_address.as_u32;
 447                     oh4->length = clib_host_to_net_u16 (
 448                         vlib_buffer_length_in_chain (vm, o_b0));
 449                     oh4->checksum = ip4_header_checksum (oh4);
 450                   }
 451               }
 452
 453             to_next[0] = o_bi0;
 454             to_next += 1;
 455
 456             vnet_buffer (o_b0)->sw_if_index[VLIB_TX] = (u32)~0;
 457           }
 458
 459 trace:
 460           if (PREDICT_FALSE(i_b0->flags & VLIB_BUFFER_IS_TRACED)) {
 461             if (o_b0) {
 462               o_b0->flags |= VLIB_BUFFER_IS_TRACED;
 463               o_b0->trace_index = i_b0->trace_index;
 464             }
 465             esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, o_b0, sizeof (*tr));
 466             tr->crypto_alg = sa0->crypto_alg;
 467             tr->integ_alg = sa0->integ_alg;
 468           }
 469
 470           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 471                                            n_left_to_next, o_bi0, next0);
 472         }
 473       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 474     }
 475   vlib_node_increment_counter (vm, esp_decrypt_node.index,
 476                                ESP_DECRYPT_ERROR_RX_PKTS,
 477                                from_frame->n_vectors);
 478
 479 free_buffers_and_exit:
 480   vlib_buffer_free (vm, recycle, vec_len(recycle));
 481   vec_free(recycle);
 482   return from_frame->n_vectors;
 483 }
 484
 485
 486 VLIB_REGISTER_NODE (esp_decrypt_node) = {
 487   .function = esp_decrypt_node_fn,
 488   .name = "esp-decrypt",
 489   .vector_size = sizeof (u32),
 490   .format_trace = format_esp_decrypt_trace,
 491   .type = VLIB_NODE_TYPE_INTERNAL,
 492
 493   .n_errors = ARRAY_LEN(esp_decrypt_error_strings),
 494   .error_strings = esp_decrypt_error_strings,
 495
 496   .n_next_nodes = ESP_DECRYPT_N_NEXT,
 497   .next_nodes = {
 498 #define _(s,n) [ESP_DECRYPT_NEXT_##s] = n,
 499     foreach_esp_decrypt_next
 500 #undef _
 501   },
 502 };
 503
 504 VLIB_NODE_FUNCTION_MULTIARCH (esp_decrypt_node, esp_decrypt_node_fn)
 505