bond: tx perf improvement, part trois
[vpp.git] / src / vnet / bonding / device.c
index 79ca2fa..8a78728 100644 (file)
@@ -379,63 +379,28 @@ bond_load_balance_active_backup (vlib_main_t * vm,
   return 0;
 }
 
-static bond_load_balance_func_t bond_load_balance_table[] = {
-#define _(v,f,s, p) { bond_load_balance_##p },
-  foreach_bond_lb_algo
-#undef _
-};
-
-VNET_DEVICE_CLASS_TX_FN (bond_dev_class) (vlib_main_t * vm,
-                                         vlib_node_runtime_t * node,
-                                         vlib_frame_t * frame)
+static_always_inline void
+bond_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+               vlib_frame_t * frame, bond_if_t * bif,
+               uword slave_count, u32 lb_alg)
 {
-  vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
   bond_main_t *bm = &bond_main;
-  bond_if_t *bif = pool_elt_at_index (bm->interfaces, rund->dev_instance);
-  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
-  u32 *from = vlib_frame_vector_args (frame);
-  ethernet_header_t *eth;
-  u32 n_left;
-  u32 sw_if_index;
+  vnet_main_t *vnm = vnet_get_main ();
+  u16 thread_index = vm->thread_index;
   bond_packet_trace_t *t0;
   uword n_trace = vlib_get_trace_count (vm, node);
-  u16 thread_index = vm->thread_index;
-  vnet_main_t *vnm = vnet_get_main ();
   u32 *to_next;
   vlib_frame_t *f;
-  uword slave_count;
+  ethernet_header_t *eth;
+  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
+  u32 *from = vlib_frame_vector_args (frame);
+  u32 n_left = frame->n_vectors;
+  u32 sw_if_index;
   u32 port0 = 0, port1 = 0, port2 = 0, port3 = 0;
   bond_per_thread_data_t *ptd = vec_elt_at_index (bm->per_thread_data,
                                                  thread_index);
 
-  if (PREDICT_FALSE (bif->admin_up == 0))
-    {
-      vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
-      vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters +
-                                    VNET_INTERFACE_COUNTER_DROP,
-                                    thread_index, bif->sw_if_index,
-                                    frame->n_vectors);
-      vlib_error_count (vm, node->node_index, BOND_TX_ERROR_IF_DOWN,
-                       frame->n_vectors);
-      return frame->n_vectors;
-    }
-
-  n_left = frame->n_vectors;
   vlib_get_buffers (vm, from, bufs, n_left);
-
-  slave_count = vec_len (bif->active_slaves);
-  if (PREDICT_FALSE (slave_count == 0))
-    {
-      vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
-      vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters +
-                                    VNET_INTERFACE_COUNTER_DROP,
-                                    thread_index, bif->sw_if_index,
-                                    frame->n_vectors);
-      vlib_error_count (vm, node->node_index, BOND_TX_ERROR_NO_SLAVE,
-                       frame->n_vectors);
-      return frame->n_vectors;
-    }
-
   b = bufs;
   while (n_left >= 4)
     {
@@ -464,22 +429,72 @@ VNET_DEVICE_CLASS_TX_FN (bond_dev_class) (vlib_main_t * vm,
 
       if (PREDICT_TRUE (slave_count > 1))
        {
-         port0 =
-           (bond_load_balance_table[bif->lb]).load_balance (vm, node,
-                                                            bif, b[0],
-                                                            slave_count);
-         port1 =
-           (bond_load_balance_table[bif->lb]).load_balance (vm, node,
-                                                            bif, b[1],
-                                                            slave_count);
-         port2 =
-           (bond_load_balance_table[bif->lb]).load_balance (vm, node,
-                                                            bif, b[2],
-                                                            slave_count);
-         port3 =
-           (bond_load_balance_table[bif->lb]).load_balance (vm, node,
-                                                            bif, b[3],
-                                                            slave_count);
+         if (lb_alg == BOND_LB_L2)
+           {
+             port0 = bond_load_balance_l2 (vm, node, bif, b[0], slave_count);
+             port1 = bond_load_balance_l2 (vm, node, bif, b[1], slave_count);
+             port2 = bond_load_balance_l2 (vm, node, bif, b[2], slave_count);
+             port3 = bond_load_balance_l2 (vm, node, bif, b[3], slave_count);
+           }
+         else if (lb_alg == BOND_LB_L34)
+           {
+             port0 = bond_load_balance_l34 (vm, node, bif, b[0],
+                                            slave_count);
+             port1 = bond_load_balance_l34 (vm, node, bif, b[1],
+                                            slave_count);
+             port2 = bond_load_balance_l34 (vm, node, bif, b[2],
+                                            slave_count);
+             port3 = bond_load_balance_l34 (vm, node, bif, b[3],
+                                            slave_count);
+           }
+         else if (lb_alg == BOND_LB_L23)
+           {
+             port0 = bond_load_balance_l23 (vm, node, bif, b[0],
+                                            slave_count);
+             port1 = bond_load_balance_l23 (vm, node, bif, b[1],
+                                            slave_count);
+             port2 = bond_load_balance_l23 (vm, node, bif, b[2],
+                                            slave_count);
+             port3 = bond_load_balance_l23 (vm, node, bif, b[3],
+                                            slave_count);
+           }
+         else if (lb_alg == BOND_LB_RR)
+           {
+             port0 = bond_load_balance_round_robin (vm, node, bif, b[0],
+                                                    slave_count);
+             port1 = bond_load_balance_round_robin (vm, node, bif, b[1],
+                                                    slave_count);
+             port2 = bond_load_balance_round_robin (vm, node, bif, b[2],
+                                                    slave_count);
+             port3 = bond_load_balance_round_robin (vm, node, bif, b[3],
+                                                    slave_count);
+           }
+         else if (lb_alg == BOND_LB_BC)
+           {
+             port0 = bond_load_balance_broadcast (vm, node, bif, b[0],
+                                                  slave_count);
+             port1 = bond_load_balance_broadcast (vm, node, bif, b[1],
+                                                  slave_count);
+             port2 = bond_load_balance_broadcast (vm, node, bif, b[2],
+                                                  slave_count);
+             port3 = bond_load_balance_broadcast (vm, node, bif, b[3],
+                                                  slave_count);
+           }
+         else if (lb_alg == BOND_LB_AB)
+           {
+             port0 = bond_load_balance_active_backup (vm, node, bif, b[0],
+                                                      slave_count);
+             port1 = bond_load_balance_active_backup (vm, node, bif, b[1],
+                                                      slave_count);
+             port2 = bond_load_balance_active_backup (vm, node, bif, b[2],
+                                                      slave_count);
+             port3 = bond_load_balance_active_backup (vm, node, bif, b[3],
+                                                      slave_count);
+           }
+         else
+           {
+             ASSERT (0);
+           }
        }
 
       sif_if_index0 = *vec_elt_at_index (bif->active_slaves, port0);
@@ -574,9 +589,42 @@ VNET_DEVICE_CLASS_TX_FN (bond_dev_class) (vlib_main_t * vm,
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
 
       if (PREDICT_TRUE (slave_count > 1))
-       port0 =
-         (bond_load_balance_table[bif->lb]).load_balance (vm, node, bif,
-                                                          b[0], slave_count);
+       {
+         if (bif->lb == BOND_LB_L2)
+           {
+             port0 = bond_load_balance_l2 (vm, node, bif, b[0], slave_count);
+           }
+         else if (bif->lb == BOND_LB_L34)
+           {
+             port0 = bond_load_balance_l34 (vm, node, bif, b[0],
+                                            slave_count);
+           }
+         else if (bif->lb == BOND_LB_L23)
+           {
+             port0 = bond_load_balance_l23 (vm, node, bif, b[0],
+                                            slave_count);
+           }
+         else if (bif->lb == BOND_LB_RR)
+           {
+             port0 = bond_load_balance_round_robin (vm, node, bif, b[0],
+                                                    slave_count);
+           }
+         else if (bif->lb == BOND_LB_BC)
+           {
+             port0 = bond_load_balance_broadcast (vm, node, bif, b[0],
+                                                  slave_count);
+           }
+         else if (bif->lb == BOND_LB_AB)
+           {
+             port0 = bond_load_balance_active_backup (vm, node, bif, b[0],
+                                                      slave_count);
+           }
+         else
+           {
+             ASSERT (0);
+           }
+       }
+
       sif_if_index0 = *vec_elt_at_index (bif->active_slaves, port0);
 
       /* Do the tracing before the old interface is overwritten */
@@ -622,6 +670,57 @@ VNET_DEVICE_CLASS_TX_FN (bond_dev_class) (vlib_main_t * vm,
   vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters
                                 + VNET_INTERFACE_COUNTER_TX, thread_index,
                                 bif->sw_if_index, frame->n_vectors);
+}
+
+VNET_DEVICE_CLASS_TX_FN (bond_dev_class) (vlib_main_t * vm,
+                                         vlib_node_runtime_t * node,
+                                         vlib_frame_t * frame)
+{
+  vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
+  bond_main_t *bm = &bond_main;
+  u16 thread_index = vm->thread_index;
+  bond_if_t *bif = pool_elt_at_index (bm->interfaces, rund->dev_instance);
+  uword slave_count;
+
+  if (PREDICT_FALSE (bif->admin_up == 0))
+    {
+      vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
+      vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters +
+                                    VNET_INTERFACE_COUNTER_DROP,
+                                    thread_index, bif->sw_if_index,
+                                    frame->n_vectors);
+      vlib_error_count (vm, node->node_index, BOND_TX_ERROR_IF_DOWN,
+                       frame->n_vectors);
+      return frame->n_vectors;
+    }
+
+  slave_count = vec_len (bif->active_slaves);
+  if (PREDICT_FALSE (slave_count == 0))
+    {
+      vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
+      vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters +
+                                    VNET_INTERFACE_COUNTER_DROP,
+                                    thread_index, bif->sw_if_index,
+                                    frame->n_vectors);
+      vlib_error_count (vm, node->node_index, BOND_TX_ERROR_NO_SLAVE,
+                       frame->n_vectors);
+      return frame->n_vectors;
+    }
+
+  if (bif->lb == BOND_LB_L2)
+    bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_L2);
+  else if (bif->lb == BOND_LB_L34)
+    bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_L34);
+  else if (bif->lb == BOND_LB_L23)
+    bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_L23);
+  else if (bif->lb == BOND_LB_RR)
+    bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_RR);
+  else if (bif->lb == BOND_LB_BC)
+    bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_BC);
+  else if (bif->lb == BOND_LB_AB)
+    bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_AB);
+  else
+    ASSERT (0);
 
   return frame->n_vectors;
 }