src/vnet/devices/af_packet/node.c

   1 /*
   2  *------------------------------------------------------------------
   3  * af_packet.c - linux kernel packet interface
   4  *
   5  * Copyright (c) 2016 Cisco and/or its affiliates.
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at:
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *------------------------------------------------------------------
  18  */
  19
  20 #include <linux/if_packet.h>
  21
  22 #include <vlib/vlib.h>
  23 #include <vlib/unix/unix.h>
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/ethernet/ethernet.h>
  26 #include <vnet/interface/rx_queue_funcs.h>
  27 #include <vnet/feature/feature.h>
  28 #include <vnet/ethernet/packet.h>
  29
  30 #include <vnet/devices/af_packet/af_packet.h>
  31
  32 #define foreach_af_packet_input_error \
  33   _(PARTIAL_PKT, "partial packet")
  34
  35 typedef enum
  36 {
  37 #define _(f,s) AF_PACKET_INPUT_ERROR_##f,
  38   foreach_af_packet_input_error
  39 #undef _
  40     AF_PACKET_INPUT_N_ERROR,
  41 } af_packet_input_error_t;
  42
  43 static char *af_packet_input_error_strings[] = {
  44 #define _(n,s) s,
  45   foreach_af_packet_input_error
  46 #undef _
  47 };
  48
  49 typedef struct
  50 {
  51   u32 next_index;
  52   u32 hw_if_index;
  53   int block;
  54   u32 num_pkts;
  55   void *block_start;
  56   block_desc_t bd;
  57   tpacket3_hdr_t tph;
  58 } af_packet_input_trace_t;
  59
  60 static u8 *
  61 format_af_packet_input_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   af_packet_input_trace_t *t = va_arg (*args, af_packet_input_trace_t *);
  66   u32 indent = format_get_indent (s);
  67
  68   s = format (s, "af_packet: hw_if_index %d next-index %d",
  69               t->hw_if_index, t->next_index);
  70
  71   s = format (s,
  72               "\n%Ublock %u:\n%Uaddress %p version %u seq_num %lu"
  73               " num_pkts %u",
  74               format_white_space, indent + 2, t->block, format_white_space,
  75               indent + 4, t->block_start, t->bd.version, t->bd.hdr.bh1.seq_num,
  76               t->num_pkts);
  77   s =
  78     format (s,
  79             "\n%Utpacket3_hdr:\n%Ustatus 0x%x len %u snaplen %u mac %u net %u"
  80             "\n%Usec 0x%x nsec 0x%x vlan %U"
  81 #ifdef TP_STATUS_VLAN_TPID_VALID
  82             " vlan_tpid %u"
  83 #endif
  84             ,
  85             format_white_space, indent + 2, format_white_space, indent + 4,
  86             t->tph.tp_status, t->tph.tp_len, t->tph.tp_snaplen, t->tph.tp_mac,
  87             t->tph.tp_net, format_white_space, indent + 4, t->tph.tp_sec,
  88             t->tph.tp_nsec, format_ethernet_vlan_tci, t->tph.hv1.tp_vlan_tci
  89 #ifdef TP_STATUS_VLAN_TPID_VALID
  90             ,
  91             t->tph.hv1.tp_vlan_tpid
  92 #endif
  93     );
  94   return s;
  95 }
  96
  97 always_inline void
  98 buffer_add_to_chain (vlib_main_t * vm, u32 bi, u32 first_bi, u32 prev_bi)
  99 {
 100   vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 101   vlib_buffer_t *first_b = vlib_get_buffer (vm, first_bi);
 102   vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_bi);
 103
 104   /* update first buffer */
 105   first_b->total_length_not_including_first_buffer += b->current_length;
 106
 107   /* update previous buffer */
 108   prev_b->next_buffer = bi;
 109   prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
 110
 111   /* update current buffer */
 112   b->next_buffer = 0;
 113 }
 114
 115 static_always_inline void
 116 fill_gso_buffer_flags (vlib_buffer_t *b, u32 gso_size, u8 l4_hdr_sz)
 117 {
 118   b->flags |= VNET_BUFFER_F_GSO;
 119   vnet_buffer2 (b)->gso_size = gso_size;
 120   vnet_buffer2 (b)->gso_l4_hdr_sz = l4_hdr_sz;
 121 }
 122
 123 static_always_inline void
 124 mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
 125 {
 126   ethernet_header_t *eth = vlib_buffer_get_current (b);
 127   vnet_buffer_oflags_t oflags = 0;
 128   if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP4)
 129     {
 130       ip4_header_t *ip4 =
 131         (vlib_buffer_get_current (b) + sizeof (ethernet_header_t));
 132       b->flags |= VNET_BUFFER_F_IS_IP4;
 133       if (ip4->protocol == IP_PROTOCOL_TCP)
 134         {
 135           oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
 136           tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b) +
 137                                                 sizeof (ethernet_header_t) +
 138                                                 ip4_header_bytes (ip4));
 139           *l4_hdr_sz = tcp_header_bytes (tcp);
 140         }
 141       else if (ip4->protocol == IP_PROTOCOL_UDP)
 142         {
 143           oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
 144           udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b) +
 145                                                 sizeof (ethernet_header_t) +
 146                                                 ip4_header_bytes (ip4));
 147           *l4_hdr_sz = sizeof (*udp);
 148         }
 149       vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
 150       vnet_buffer (b)->l4_hdr_offset =
 151         sizeof (ethernet_header_t) + ip4_header_bytes (ip4);
 152       if (oflags)
 153         vnet_buffer_offload_flags_set (b, oflags);
 154     }
 155   else if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP6)
 156     {
 157       ip6_header_t *ip6 =
 158         (vlib_buffer_get_current (b) + sizeof (ethernet_header_t));
 159       b->flags |= VNET_BUFFER_F_IS_IP6;
 160       u16 ip6_hdr_len = sizeof (ip6_header_t);
 161       if (ip6_ext_hdr (ip6->protocol))
 162         {
 163           ip6_ext_header_t *p = (void *) (ip6 + 1);
 164           ip6_hdr_len += ip6_ext_header_len (p);
 165           while (ip6_ext_hdr (p->next_hdr))
 166             {
 167               ip6_hdr_len += ip6_ext_header_len (p);
 168               p = ip6_ext_next_header (p);
 169             }
 170         }
 171       if (ip6->protocol == IP_PROTOCOL_TCP)
 172         {
 173           oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
 174           tcp_header_t *tcp =
 175             (tcp_header_t *) (vlib_buffer_get_current (b) +
 176                               sizeof (ethernet_header_t) + ip6_hdr_len);
 177           *l4_hdr_sz = tcp_header_bytes (tcp);
 178         }
 179       else if (ip6->protocol == IP_PROTOCOL_UDP)
 180         {
 181           oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
 182           udp_header_t *udp =
 183             (udp_header_t *) (vlib_buffer_get_current (b) +
 184                               sizeof (ethernet_header_t) + ip6_hdr_len);
 185           *l4_hdr_sz = sizeof (*udp);
 186         }
 187       vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
 188       vnet_buffer (b)->l4_hdr_offset =
 189         sizeof (ethernet_header_t) + ip6_hdr_len;
 190       if (oflags)
 191         vnet_buffer_offload_flags_set (b, oflags);
 192     }
 193 }
 194
 195 always_inline uword
 196 af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 197                            vlib_frame_t * frame, af_packet_if_t * apif)
 198 {
 199   af_packet_main_t *apm = &af_packet_main;
 200   tpacket3_hdr_t *tph;
 201   u32 next_index;
 202   u32 n_free_bufs;
 203   u32 n_rx_packets = 0;
 204   u32 n_rx_bytes = 0;
 205   u32 *to_next = 0;
 206   u32 block = apif->next_rx_block;
 207   u32 block_nr = apif->rx_req->tp_block_nr;
 208   u8 *block_start = 0;
 209   uword n_trace = vlib_get_trace_count (vm, node);
 210   u32 thread_index = vm->thread_index;
 211   u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
 212   u32 min_bufs = apif->rx_req->tp_frame_size / n_buffer_bytes;
 213   u32 eth_header_size = 0;
 214   u32 num_pkts = 0;
 215   u32 rx_frame_offset = 0;
 216   block_desc_t *bd = 0;
 217   vlib_buffer_t bt;
 218
 219   if (apif->mode == AF_PACKET_IF_MODE_IP)
 220     {
 221       next_index = VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
 222     }
 223   else
 224     {
 225       eth_header_size = sizeof (ethernet_header_t);
 226       next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
 227       if (PREDICT_FALSE (apif->per_interface_next_index != ~0))
 228         next_index = apif->per_interface_next_index;
 229
 230       /* redirect if feature path enabled */
 231       vnet_feature_start_device_input_x1 (apif->sw_if_index, &next_index, &bt);
 232     }
 233
 234   while ((((block_desc_t *) (block_start = apif->rx_ring[block]))
 235             ->hdr.bh1.block_status &
 236           TP_STATUS_USER) != 0)
 237     {
 238       u32 n_required = 0;
 239
 240       if (PREDICT_FALSE (num_pkts == 0))
 241         {
 242           bd = (block_desc_t *) block_start;
 243           num_pkts = bd->hdr.bh1.num_pkts;
 244           rx_frame_offset = sizeof (block_desc_t);
 245         }
 246
 247       n_required = clib_max (num_pkts, VLIB_FRAME_SIZE);
 248       n_free_bufs = vec_len (apm->rx_buffers[thread_index]);
 249       if (PREDICT_FALSE (n_free_bufs < n_required))
 250         {
 251           vec_validate (apm->rx_buffers[thread_index],
 252                         n_required + n_free_bufs - 1);
 253           n_free_bufs += vlib_buffer_alloc (
 254             vm, &apm->rx_buffers[thread_index][n_free_bufs], n_required);
 255           _vec_len (apm->rx_buffers[thread_index]) = n_free_bufs;
 256         }
 257
 258       while (num_pkts && (n_free_bufs > min_bufs))
 259         {
 260           vlib_buffer_t *b0 = 0, *first_b0 = 0;
 261           u32 next0 = next_index;
 262           u32 n_left_to_next;
 263           vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 264           while (num_pkts && n_left_to_next && (n_free_bufs > min_bufs))
 265             {
 266               tph = (tpacket3_hdr_t *) (block_start + rx_frame_offset);
 267
 268               if (num_pkts > 1)
 269                 CLIB_PREFETCH (block_start + rx_frame_offset +
 270                                  tph->tp_next_offset,
 271                                2 * CLIB_CACHE_LINE_BYTES, LOAD);
 272               u32 data_len = tph->tp_snaplen;
 273               u32 offset = 0;
 274               u32 bi0 = 0, first_bi0 = 0, prev_bi0;
 275               u8 l4_hdr_sz = 0;
 276
 277               while (data_len)
 278                 {
 279                   /* grab free buffer */
 280                   u32 last_empty_buffer =
 281                     vec_len (apm->rx_buffers[thread_index]) - 1;
 282                   prev_bi0 = bi0;
 283                   bi0 = apm->rx_buffers[thread_index][last_empty_buffer];
 284                   b0 = vlib_get_buffer (vm, bi0);
 285                   _vec_len (apm->rx_buffers[thread_index]) = last_empty_buffer;
 286                   n_free_bufs--;
 287
 288                   /* copy data */
 289                   u32 bytes_to_copy =
 290                     data_len > n_buffer_bytes ? n_buffer_bytes : data_len;
 291                   u32 vlan_len = 0;
 292                   u32 bytes_copied = 0;
 293                   b0->current_data = 0;
 294                   /* Kernel removes VLAN headers, so reconstruct VLAN */
 295                   if (PREDICT_FALSE (tph->tp_status & TP_STATUS_VLAN_VALID))
 296                     {
 297                       if (PREDICT_TRUE (offset == 0))
 298                         {
 299                           clib_memcpy_fast (vlib_buffer_get_current (b0),
 300                                             (u8 *) tph + tph->tp_mac,
 301                                             sizeof (ethernet_header_t));
 302                           ethernet_header_t *eth =
 303                             vlib_buffer_get_current (b0);
 304                           ethernet_vlan_header_t *vlan =
 305                             (ethernet_vlan_header_t *) (eth + 1);
 306                           vlan->priority_cfi_and_id =
 307                             clib_host_to_net_u16 (tph->hv1.tp_vlan_tci);
 308                           vlan->type = eth->type;
 309                           eth->type =
 310                             clib_host_to_net_u16 (ETHERNET_TYPE_VLAN);
 311                           vlan_len = sizeof (ethernet_vlan_header_t);
 312                           bytes_copied = sizeof (ethernet_header_t);
 313                         }
 314                     }
 315                   clib_memcpy_fast (((u8 *) vlib_buffer_get_current (b0)) +
 316                                       bytes_copied + vlan_len,
 317                                     (u8 *) tph + tph->tp_mac + offset +
 318                                       bytes_copied,
 319                                     (bytes_to_copy - bytes_copied));
 320
 321                   /* fill buffer header */
 322                   b0->current_length = bytes_to_copy + vlan_len;
 323
 324                   if (offset == 0)
 325                     {
 326                       b0->total_length_not_including_first_buffer = 0;
 327                       b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
 328                       vnet_buffer (b0)->sw_if_index[VLIB_RX] =
 329                         apif->sw_if_index;
 330                       vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~0;
 331                       first_bi0 = bi0;
 332                       first_b0 = vlib_get_buffer (vm, first_bi0);
 333                       if (tph->tp_status & TP_STATUS_CSUMNOTREADY)
 334                         mark_tcp_udp_cksum_calc (first_b0, &l4_hdr_sz);
 335                       /* This is a trade-off for GSO. As kernel isn't passing
 336                        * us the GSO state or size, we guess it by comparing it
 337                        * to the host MTU of the interface */
 338                       if (tph->tp_snaplen > (apif->host_mtu + eth_header_size))
 339                         fill_gso_buffer_flags (first_b0, apif->host_mtu,
 340                                                l4_hdr_sz);
 341                     }
 342                   else
 343                     buffer_add_to_chain (vm, bi0, first_bi0, prev_bi0);
 344
 345                   offset += bytes_to_copy;
 346                   data_len -= bytes_to_copy;
 347                 }
 348               n_rx_packets++;
 349               n_rx_bytes += tph->tp_snaplen;
 350               to_next[0] = first_bi0;
 351               to_next += 1;
 352               n_left_to_next--;
 353
 354               /* drop partial packets */
 355               if (PREDICT_FALSE (tph->tp_len != tph->tp_snaplen))
 356                 {
 357                   next0 = VNET_DEVICE_INPUT_NEXT_DROP;
 358                   first_b0->error =
 359                     node->errors[AF_PACKET_INPUT_ERROR_PARTIAL_PKT];
 360                 }
 361               else
 362                 {
 363                   if (PREDICT_FALSE (apif->mode == AF_PACKET_IF_MODE_IP))
 364                     {
 365                       switch (first_b0->data[0] & 0xf0)
 366                         {
 367                         case 0x40:
 368                           next0 = VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
 369                           break;
 370                         case 0x60:
 371                           next0 = VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
 372                           break;
 373                         default:
 374                           next0 = VNET_DEVICE_INPUT_NEXT_DROP;
 375                           break;
 376                         }
 377                       if (PREDICT_FALSE (apif->per_interface_next_index != ~0))
 378                         next0 = apif->per_interface_next_index;
 379                     }
 380                   else
 381                     {
 382                       /* copy feature arc data from template */
 383                       first_b0->current_config_index = bt.current_config_index;
 384                       vnet_buffer (first_b0)->feature_arc_index =
 385                         vnet_buffer (&bt)->feature_arc_index;
 386                     }
 387                 }
 388
 389               /* trace */
 390               if (PREDICT_FALSE (n_trace > 0 &&
 391                                  vlib_trace_buffer (vm, node, next0, first_b0,
 392                                                     /* follow_chain */ 0)))
 393                 {
 394                   af_packet_input_trace_t *tr;
 395                   vlib_set_trace_count (vm, node, --n_trace);
 396                   tr = vlib_add_trace (vm, node, first_b0, sizeof (*tr));
 397                   tr->next_index = next0;
 398                   tr->hw_if_index = apif->hw_if_index;
 399                   tr->block = block;
 400                   tr->block_start = bd;
 401                   tr->num_pkts = num_pkts;
 402                   clib_memcpy_fast (&tr->bd, bd, sizeof (block_desc_t));
 403                   clib_memcpy_fast (&tr->tph, tph, sizeof (tpacket3_hdr_t));
 404                 }
 405
 406               /* enque and take next packet */
 407               vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 408                                                n_left_to_next, first_bi0,
 409                                                next0);
 410
 411               /* next packet */
 412               num_pkts--;
 413               rx_frame_offset += tph->tp_next_offset;
 414             }
 415
 416           vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 417         }
 418
 419       if (PREDICT_TRUE (num_pkts == 0))
 420         {
 421           bd->hdr.bh1.block_status = TP_STATUS_KERNEL;
 422           block = (block + 1) % block_nr;
 423         }
 424     }
 425
 426   apif->next_rx_block = block;
 427   vlib_increment_combined_counter
 428     (vnet_get_main ()->interface_main.combined_sw_if_counters
 429      + VNET_INTERFACE_COUNTER_RX,
 430      vlib_get_thread_index (), apif->hw_if_index, n_rx_packets, n_rx_bytes);
 431
 432   vnet_device_increment_rx_packets (thread_index, n_rx_packets);
 433   return n_rx_packets;
 434 }
 435
 436 VLIB_NODE_FN (af_packet_input_node) (vlib_main_t * vm,
 437                                      vlib_node_runtime_t * node,
 438                                      vlib_frame_t * frame)
 439 {
 440   u32 n_rx_packets = 0;
 441   af_packet_main_t *apm = &af_packet_main;
 442   vnet_hw_if_rxq_poll_vector_t *pv;
 443   pv = vnet_hw_if_get_rxq_poll_vector (vm, node);
 444   for (int i = 0; i < vec_len (pv); i++)
 445     {
 446       af_packet_if_t *apif;
 447       apif = vec_elt_at_index (apm->interfaces, pv[i].dev_instance);
 448       if (apif->is_admin_up)
 449         n_rx_packets += af_packet_device_input_fn (vm, node, frame, apif);
 450     }
 451
 452   return n_rx_packets;
 453 }
 454
 455 VLIB_REGISTER_NODE (af_packet_input_node) = {
 456   .name = "af-packet-input",
 457   .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED,
 458   .sibling_of = "device-input",
 459   .format_trace = format_af_packet_input_trace,
 460   .type = VLIB_NODE_TYPE_INPUT,
 461   .state = VLIB_NODE_STATE_INTERRUPT,
 462   .n_errors = AF_PACKET_INPUT_N_ERROR,
 463   .error_strings = af_packet_input_error_strings,
 464 };
 465
 466
 467 /*
 468  * fd.io coding-style-patch-verification: ON
 469  *
 470  * Local Variables:
 471  * eval: (c-set-style "gnu")
 472  * End:
 473  */