src/plugins/vmxnet3/input.c

   1 /*
   2  *------------------------------------------------------------------
   3  * Copyright (c) 2018 Cisco and/or its affiliates.
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  *------------------------------------------------------------------
  16  */
  17
  18 #include <vlib/vlib.h>
  19 #include <vlib/unix/unix.h>
  20 #include <vlib/pci/pci.h>
  21 #include <vnet/ethernet/ethernet.h>
  22 #include <vnet/devices/devices.h>
  23
  24 #include <vmxnet3/vmxnet3.h>
  25
  26 #define foreach_vmxnet3_input_error \
  27   _(BUFFER_ALLOC, "buffer alloc error") \
  28   _(RX_PACKET_NO_SOP, "Rx packet error - no SOP") \
  29   _(RX_PACKET, "Rx packet error") \
  30   _(RX_PACKET_EOP, "Rx packet error found on EOP") \
  31   _(NO_BUFFER, "Rx no buffer error")
  32
  33 typedef enum
  34 {
  35 #define _(f,s) VMXNET3_INPUT_ERROR_##f,
  36   foreach_vmxnet3_input_error
  37 #undef _
  38     VMXNET3_INPUT_N_ERROR,
  39 } vmxnet3_input_error_t;
  40
  41 static __clib_unused char *vmxnet3_input_error_strings[] = {
  42 #define _(n,s) s,
  43   foreach_vmxnet3_input_error
  44 #undef _
  45 };
  46
  47 static_always_inline u16
  48 vmxnet3_find_rid (vmxnet3_device_t * vd, vmxnet3_rx_comp * rx_comp)
  49 {
  50   u32 rid;
  51
  52   // rid is bits 16-25 (10 bits number)
  53   rid = rx_comp->index & (0xffffffff >> 6);
  54   rid >>= 16;
  55   if ((rid >= vd->num_rx_queues) && (rid < (vd->num_rx_queues << 1)))
  56     return 1;
  57   else
  58     return 0;
  59 }
  60
  61 static_always_inline void
  62 vmxnet3_rx_comp_ring_advance_next (vmxnet3_rxq_t * rxq)
  63 {
  64   vmxnet3_rx_comp_ring *comp_ring = &rxq->rx_comp_ring;
  65
  66   comp_ring->next++;
  67   if (PREDICT_FALSE (comp_ring->next == rxq->size))
  68     {
  69       comp_ring->next = 0;
  70       comp_ring->gen ^= VMXNET3_RXCF_GEN;
  71     }
  72 }
  73
  74 static_always_inline uword
  75 vmxnet3_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
  76                              vlib_frame_t * frame, vmxnet3_device_t * vd,
  77                              u16 qid)
  78 {
  79   vnet_main_t *vnm = vnet_get_main ();
  80   uword n_trace = vlib_get_trace_count (vm, node);
  81   u32 n_rx_packets = 0, n_rx_bytes = 0;
  82   vmxnet3_rx_comp *rx_comp;
  83   u32 desc_idx;
  84   vmxnet3_rxq_t *rxq;
  85   u32 thread_index = vm->thread_index;
  86   u32 buffer_indices[VLIB_FRAME_SIZE], *bi;
  87   u16 nexts[VLIB_FRAME_SIZE], *next;
  88   vmxnet3_rx_ring *ring;
  89   vmxnet3_rx_comp_ring *comp_ring;
  90   u16 rid;
  91   vlib_buffer_t *prev_b0 = 0, *hb = 0;
  92   u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
  93   u8 known_next = 0, got_packet = 0;
  94   vmxnet3_rx_desc *rxd;
  95   clib_error_t *error;
  96
  97   rxq = vec_elt_at_index (vd->rxqs, qid);
  98   comp_ring = &rxq->rx_comp_ring;
  99   bi = buffer_indices;
 100   next = nexts;
 101   rx_comp = &rxq->rx_comp[comp_ring->next];
 102
 103   while (PREDICT_TRUE ((n_rx_packets < VLIB_FRAME_SIZE) &&
 104                        (comp_ring->gen ==
 105                         (rx_comp->flags & VMXNET3_RXCF_GEN))))
 106     {
 107       vlib_buffer_t *b0;
 108       u32 bi0;
 109
 110       rid = vmxnet3_find_rid (vd, rx_comp);
 111       ring = &rxq->rx_ring[rid];
 112
 113       if (PREDICT_TRUE (ring->fill >= 1))
 114         ring->fill--;
 115       else
 116         {
 117           vlib_error_count (vm, node->node_index,
 118                             VMXNET3_INPUT_ERROR_NO_BUFFER, 1);
 119           if (hb)
 120             {
 121               vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, hb));
 122               hb = 0;
 123             }
 124           prev_b0 = 0;
 125           break;
 126         }
 127
 128       desc_idx = rx_comp->index & VMXNET3_RXC_INDEX;
 129       ring->consume = desc_idx;
 130       rxd = &rxq->rx_desc[rid][desc_idx];
 131
 132       bi0 = ring->bufs[desc_idx];
 133       ring->bufs[desc_idx] = ~0;
 134
 135       b0 = vlib_get_buffer (vm, bi0);
 136       vnet_buffer (b0)->sw_if_index[VLIB_RX] = vd->sw_if_index;
 137       vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 138       vnet_buffer (b0)->feature_arc_index = 0;
 139       b0->current_length = rx_comp->len & VMXNET3_RXCL_LEN_MASK;
 140       b0->current_data = 0;
 141       b0->total_length_not_including_first_buffer = 0;
 142       b0->next_buffer = 0;
 143       b0->flags = 0;
 144       b0->error = 0;
 145       b0->current_config_index = 0;
 146       ASSERT (b0->current_length != 0);
 147
 148       if (PREDICT_FALSE ((rx_comp->index & VMXNET3_RXCI_EOP) &&
 149                          (rx_comp->len & VMXNET3_RXCL_ERROR)))
 150         {
 151           vlib_buffer_free_one (vm, bi0);
 152           vlib_error_count (vm, node->node_index,
 153                             VMXNET3_INPUT_ERROR_RX_PACKET_EOP, 1);
 154           if (hb && vlib_get_buffer_index (vm, hb) != bi0)
 155             {
 156               vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, hb));
 157               hb = 0;
 158             }
 159           prev_b0 = 0;
 160           goto next;
 161         }
 162
 163       if (rx_comp->index & VMXNET3_RXCI_SOP)
 164         {
 165           ASSERT (!(rxd->flags & VMXNET3_RXF_BTYPE));
 166           /* start segment */
 167           hb = b0;
 168           bi[0] = bi0;
 169           if (!(rx_comp->index & VMXNET3_RXCI_EOP))
 170             {
 171               hb->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
 172               prev_b0 = b0;
 173             }
 174           else
 175             {
 176               /*
 177                * Both start and end of packet is set. It is a complete packet
 178                */
 179               prev_b0 = 0;
 180               got_packet = 1;
 181             }
 182         }
 183       else if (rx_comp->index & VMXNET3_RXCI_EOP)
 184         {
 185           /* end of segment */
 186           if (prev_b0)
 187             {
 188               prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
 189               prev_b0->next_buffer = bi0;
 190               hb->total_length_not_including_first_buffer +=
 191                 b0->current_length;
 192               prev_b0 = 0;
 193               got_packet = 1;
 194             }
 195           else
 196             {
 197               /* EOP without SOP, error */
 198               vlib_error_count (vm, node->node_index,
 199                                 VMXNET3_INPUT_ERROR_RX_PACKET_NO_SOP, 1);
 200               vlib_buffer_free_one (vm, bi0);
 201               if (hb && vlib_get_buffer_index (vm, hb) != bi0)
 202                 {
 203                   vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, hb));
 204                   hb = 0;
 205                 }
 206               goto next;
 207             }
 208         }
 209       else if (prev_b0)         // !sop && !eop
 210         {
 211           /* mid chain */
 212           ASSERT (rxd->flags & VMXNET3_RXF_BTYPE);
 213           prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
 214           prev_b0->next_buffer = bi0;
 215           prev_b0 = b0;
 216           hb->total_length_not_including_first_buffer += b0->current_length;
 217         }
 218       else
 219         {
 220           vlib_error_count (vm, node->node_index,
 221                             VMXNET3_INPUT_ERROR_RX_PACKET, 1);
 222           vlib_buffer_free_one (vm, bi0);
 223           if (hb && vlib_get_buffer_index (vm, hb) != bi0)
 224             {
 225               vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, hb));
 226               hb = 0;
 227             }
 228           goto next;
 229         }
 230
 231       n_rx_bytes += b0->current_length;
 232
 233       if (got_packet)
 234         {
 235           ethernet_header_t *e = (ethernet_header_t *) hb->data;
 236
 237           if (PREDICT_FALSE (vd->per_interface_next_index != ~0))
 238             {
 239               next_index = vd->per_interface_next_index;
 240               known_next = 1;
 241             }
 242
 243           if (PREDICT_FALSE
 244               (vnet_device_input_have_features (vd->sw_if_index)))
 245             {
 246               vnet_feature_start_device_input_x1 (vd->sw_if_index,
 247                                                   &next_index, hb);
 248               known_next = 1;
 249             }
 250
 251           if (PREDICT_FALSE (known_next))
 252             {
 253               next[0] = next_index;
 254             }
 255           else
 256             {
 257               if (ethernet_frame_is_tagged (e->type))
 258                 next[0] = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
 259               else
 260                 {
 261                   if (rx_comp->flags & VMXNET3_RXCF_IP4)
 262                     {
 263                       next[0] = VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT;
 264                       hb->flags |= VNET_BUFFER_F_IS_IP4;
 265                       vlib_buffer_advance (hb,
 266                                            device_input_next_node_advance
 267                                            [next[0]]);
 268                     }
 269                   else if (rx_comp->flags & VMXNET3_RXCF_IP6)
 270                     {
 271                       next[0] = VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
 272                       hb->flags |= VNET_BUFFER_F_IS_IP6;
 273                       vlib_buffer_advance (hb,
 274                                            device_input_next_node_advance
 275                                            [next[0]]);
 276                     }
 277                   else
 278                     {
 279                       next[0] = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
 280                     }
 281                 }
 282             }
 283
 284           n_rx_packets++;
 285           next++;
 286           bi++;
 287           hb = 0;
 288           got_packet = 0;
 289         }
 290
 291     next:
 292       vmxnet3_rx_comp_ring_advance_next (rxq);
 293       rx_comp = &rxq->rx_comp[comp_ring->next];
 294     }
 295
 296   if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node))))
 297     {
 298       u32 n_left = n_rx_packets;
 299
 300       bi = buffer_indices;
 301       next = nexts;
 302       while (n_trace && n_left)
 303         {
 304           vlib_buffer_t *b;
 305           vmxnet3_input_trace_t *tr;
 306
 307           b = vlib_get_buffer (vm, bi[0]);
 308           vlib_trace_buffer (vm, node, next[0], b, /* follow_chain */ 0);
 309           tr = vlib_add_trace (vm, node, b, sizeof (*tr));
 310           tr->next_index = next[0];
 311           tr->hw_if_index = vd->hw_if_index;
 312           tr->buffer = *b;
 313
 314           n_trace--;
 315           n_left--;
 316           bi++;
 317           next++;
 318         }
 319       vlib_set_trace_count (vm, node, n_trace);
 320     }
 321
 322   if (PREDICT_TRUE (n_rx_packets))
 323     {
 324       vlib_buffer_enqueue_to_next (vm, node, buffer_indices, nexts,
 325                                    n_rx_packets);
 326       vlib_increment_combined_counter
 327         (vnm->interface_main.combined_sw_if_counters +
 328          VNET_INTERFACE_COUNTER_RX, thread_index,
 329          vd->hw_if_index, n_rx_packets, n_rx_bytes);
 330     }
 331
 332   error = vmxnet3_rxq_refill_ring0 (vm, vd, rxq);
 333   if (PREDICT_FALSE (error != 0))
 334     {
 335       vlib_error_count (vm, node->node_index,
 336                         VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1);
 337     }
 338   error = vmxnet3_rxq_refill_ring1 (vm, vd, rxq);
 339   if (PREDICT_FALSE (error != 0))
 340     {
 341       vlib_error_count (vm, node->node_index,
 342                         VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1);
 343     }
 344
 345   return n_rx_packets;
 346 }
 347
 348 VLIB_NODE_FN (vmxnet3_input_node) (vlib_main_t * vm,
 349                                    vlib_node_runtime_t * node,
 350                                    vlib_frame_t * frame)
 351 {
 352   u32 n_rx = 0;
 353   vmxnet3_main_t *vmxm = &vmxnet3_main;
 354   vnet_device_input_runtime_t *rt = (void *) node->runtime_data;
 355   vnet_device_and_queue_t *dq;
 356
 357   foreach_device_and_queue (dq, rt->devices_and_queues)
 358   {
 359     vmxnet3_device_t *vd;
 360     vd = vec_elt_at_index (vmxm->devices, dq->dev_instance);
 361     if ((vd->flags & VMXNET3_DEVICE_F_ADMIN_UP) == 0)
 362       continue;
 363     n_rx += vmxnet3_device_input_inline (vm, node, frame, vd, dq->queue_id);
 364   }
 365   return n_rx;
 366 }
 367
 368 #ifndef CLIB_MARCH_VARIANT
 369 /* *INDENT-OFF* */
 370 VLIB_REGISTER_NODE (vmxnet3_input_node) = {
 371   .name = "vmxnet3-input",
 372   .sibling_of = "device-input",
 373   .format_trace = format_vmxnet3_input_trace,
 374   .type = VLIB_NODE_TYPE_INPUT,
 375   .state = VLIB_NODE_STATE_DISABLED,
 376   .n_errors = VMXNET3_INPUT_N_ERROR,
 377   .error_strings = vmxnet3_input_error_strings,
 378 };
 379 #endif
 380
 381 /* *INDENT-ON* */
 382
 383 /*
 384  * fd.io coding-style-patch-verification: ON
 385  *
 386  * Local Variables:
 387  * eval: (c-set-style "gnu")
 388  * End:
 389  */