Fix and improve Rx Scheduler mode

Fix support of atomic mode and add two new modes: parallel and ordered.
Three modes of synchronization between threads are supported now:
 - parallel - no synchronization between events
 - ordered - relative sequence of events is restored in output queues
 - atomic - only a single thread may process events from a single queue

Change-Id: I9399aa601f0927042ac0f7f1e5f265f4d47724b8
Signed-off-by: Michal Mazur <[email protected]>

diff --git a/README.vppodp b/README.vppodp
index fe1e298..9a84945 100644 (file)
--- a/README.vppodp
+++ b/README.vppodp
@@ -74,8 +74,9 @@ Below is a basic verification test.
 Note :For odp-dpdk the port has to bound with dpdk driver prior to test and interface name is passed as 0,1..etc.
 
 1)Configure odp packet interface with Rx/Tx mode and number of queues.
-  Rx mode can be set to 0-burst, 1-queue, 2-schedule (default mode is burst).
-  Tx mode can be set to 0-burst, 1-queue (default mode is burst)
+  Rx mode can be set to 0-burst (default), 1-queue, 2-schedule-atomic,
+      3-schedule-ordered, 4-schedule-parallel
+  Tx mode can be set to 0-burst (default), 1-queue (forced for schedule-ordered)
   Number of Rx queues can be 1 or more. It will determine number of threads and enable RSS (default is 1).
   Number of Tx queues can be 0 (equal to number of threads, default) or 1 (single shared queue).
 

diff --git a/src/plugins/odp/node.c b/src/plugins/odp/node.c
index 8eb1b0c..fea8bf6 100755 (executable)
--- a/src/plugins/odp/node.c
+++ b/src/plugins/odp/node.c
@@ -75,6 +75,8 @@ odp_packet_queue_mode (odp_packet_if_t * oif, odp_packet_t pkt_tbl[],
   odp_event_t evt_tbl[req_pkts];
   u64 sched_wait;
   odp_queue_t rxq = ODP_QUEUE_INVALID;
+  odp_thrmask_t mask;
+
 
   if (pktio == ODP_PKTIO_INVALID)
     {
@@ -90,6 +92,12 @@ odp_packet_queue_mode (odp_packet_if_t * oif, odp_packet_t pkt_tbl[],
          return 0;
        }
     }
+  else
+    {
+      odp_thrmask_zero (&mask);
+      odp_thrmask_set (&mask, odp_thread_id ());
+      odp_schedule_group_join (oif->sched_group, &mask);
+    }
 
   while (req_pkts)
     {
@@ -110,6 +118,11 @@ odp_packet_queue_mode (odp_packet_if_t * oif, odp_packet_t pkt_tbl[],
       req_pkts -= i;
     }
 
+  if (oif->m.rx_mode != APPL_MODE_PKT_QUEUE)
+    {
+      odp_schedule_group_leave (oif->sched_group, &mask);
+    }
+
   /* convert events to packets, discarding any non-packet events */
   for (i = 0; i < num_evts; i++)
     {
@@ -215,8 +228,7 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 
   do
     {
-      if ((oif->m.rx_mode == (APPL_MODE_PKT_QUEUE)) ||
-         (oif->m.rx_mode == (APPL_MODE_PKT_SCHED)))
+      if (oif->m.rx_mode != APPL_MODE_PKT_BURST)
        {
          n_left =
            odp_packet_queue_mode (oif, pkt_tbl, queue_id, n_left_to_next);

diff --git a/src/plugins/odp/odp_packet.c b/src/plugins/odp/odp_packet.c
index 271e7a3..f0c4007 100755 (executable)
--- a/src/plugins/odp/odp_packet.c
+++ b/src/plugins/odp/odp_packet.c
@@ -43,36 +43,54 @@ create_pktio (const char *dev, odp_pool_t pool, odp_packet_if_t * oif)
   odp_pktio_capability_t capa;
 
   odp_pktio_param_init (&pktio_param);
+  odp_pktio_config_init (&pktio_config);
+  odp_pktin_queue_param_init (&pktin_param);
+  odp_pktout_queue_param_init (&pktout_param);
 
-  switch (oif->m.rx_mode)
+  switch (oif->m.tx_mode)
     {
     case APPL_MODE_PKT_BURST:
-      pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT;
+      pktio_param.out_mode = ODP_PKTOUT_MODE_DIRECT;
       break;
     case APPL_MODE_PKT_QUEUE:
-      pktio_param.in_mode = ODP_PKTIN_MODE_QUEUE;
+      pktio_param.out_mode = ODP_PKTOUT_MODE_QUEUE;
       break;
-    case APPL_MODE_PKT_SCHED:
-      pktio_param.in_mode = ODP_PKTIN_MODE_SCHED;
+    case APPL_MODE_PKT_TM:
+      clib_error ("Traffic Manager mode not supported!\n");
+      pktio_param.out_mode = ODP_PKTOUT_MODE_TM;
       break;
     default:
-      clib_warning ("Invalid RX mode\n");
+      clib_warning ("Invalid TX mode\n");
     }
 
-  switch (oif->m.tx_mode)
+  switch (oif->m.rx_mode)
     {
     case APPL_MODE_PKT_BURST:
-      pktio_param.out_mode = ODP_PKTOUT_MODE_DIRECT;
+      pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT;
       break;
     case APPL_MODE_PKT_QUEUE:
+      pktio_param.in_mode = ODP_PKTIN_MODE_QUEUE;
+      break;
+    case APPL_MODE_PKT_SCHED_ATOMIC:
+      pktio_param.in_mode = ODP_PKTIN_MODE_SCHED;
+      pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_ATOMIC;
+      pktin_param.queue_param.sched.group = oif->sched_group;
+      break;
+    case APPL_MODE_PKT_SCHED_ORDERED:
+      pktio_param.in_mode = ODP_PKTIN_MODE_SCHED;
+      pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_ORDERED;
+      pktin_param.queue_param.sched.group = oif->sched_group;
+      /* Force Tx Queue mode */
+      oif->m.tx_mode = APPL_MODE_PKT_QUEUE;
       pktio_param.out_mode = ODP_PKTOUT_MODE_QUEUE;
       break;
-    case APPL_MODE_PKT_TM:
-      clib_error ("Traffic Manager mode not supported!\n");
-      pktio_param.out_mode = ODP_PKTOUT_MODE_TM;
+    case APPL_MODE_PKT_SCHED_PARALLEL:
+      pktio_param.in_mode = ODP_PKTIN_MODE_SCHED;
+      pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL;
+      pktin_param.queue_param.sched.group = oif->sched_group;
       break;
     default:
-      clib_warning ("Invalid TX mode\n");
+      clib_warning ("Invalid RX mode\n");
     }
 
   /* Open a packet IO instance */
@@ -90,13 +108,10 @@ create_pktio (const char *dev, odp_pool_t pool, odp_packet_if_t * oif)
       return 0;
     }
 
-  odp_pktio_config_init (&pktio_config);
+  /* Disable ODP parser for performance */
   pktio_config.parser.layer = ODP_PKTIO_PARSER_LAYER_NONE;
   odp_pktio_config (pktio, &pktio_config);
 
-  odp_pktin_queue_param_init (&pktin_param);
-  pktin_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE;
-
   if (oif->m.num_rx_queues > capa.max_input_queues)
     {
       oif->m.num_rx_queues = capa.max_input_queues;
@@ -109,9 +124,9 @@ create_pktio (const char *dev, odp_pool_t pool, odp_packet_if_t * oif)
       if (oif->m.num_rx_queues > MAX_QUEUES)
        oif->m.num_rx_queues = MAX_QUEUES;
 
+      pktin_param.num_queues = oif->m.num_rx_queues;
       pktin_param.classifier_enable = 0;
       pktin_param.hash_enable = 1;
-      pktin_param.num_queues = oif->m.num_rx_queues;
       pktin_param.hash_proto.proto.ipv4_udp = 1;
       pktin_param.hash_proto.proto.ipv4_tcp = 1;
       pktin_param.hash_proto.proto.ipv4 = 1;
@@ -119,15 +134,12 @@ create_pktio (const char *dev, odp_pool_t pool, odp_packet_if_t * oif)
   else
     oif->m.num_rx_queues = 1;
 
-  if (oif->m.rx_mode == APPL_MODE_PKT_SCHED)
-    pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_ATOMIC;
-
+  pktin_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE;
   if (odp_pktin_queue_config (pktio, &pktin_param))
     {
       clib_warning ("Error: pktin config failed");
     }
 
-  odp_pktout_queue_param_init (&pktout_param);
   if ((capa.max_output_queues >= tm->n_vlib_mains) &&
       (oif->m.num_tx_queues == 0))
     {
@@ -166,7 +178,7 @@ odp_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set,
   vnet_sw_interface_t *sw;
   vnet_main_t *vnm = vnet_get_main ();
   uword *p;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  odp_thrmask_t thrmask;
 
   p = mhash_get (&om->if_index_by_host_if_name, host_if_name);
   if (p)
@@ -178,8 +190,13 @@ odp_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set,
   oif->per_interface_next_index = ~0;
   oif->m = *mode;
 
-  if (mode->rx_mode == APPL_MODE_PKT_SCHED)
-    oif->m.num_rx_queues = tm->n_vlib_mains - 1;
+  /* Create new schedule group */
+  if ((oif->m.rx_mode != APPL_MODE_PKT_BURST) &&
+      (oif->m.rx_mode != APPL_MODE_PKT_QUEUE))
+    {
+      odp_thrmask_zero (&thrmask);
+      oif->sched_group = odp_schedule_group_create (NULL, &thrmask);
+    }
 
   /* Create a pktio instance */
   oif->pktio = create_pktio ((char *) host_if_name, om->pool, oif);
@@ -191,13 +208,13 @@ odp_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set,
 
   om->if_count++;
 
-  if (mode->rx_mode == APPL_MODE_PKT_BURST)
+  if (oif->m.rx_mode == APPL_MODE_PKT_BURST)
     odp_pktin_queue (oif->pktio, oif->inq, oif->m.num_rx_queues);
-  else if (mode->rx_mode == APPL_MODE_PKT_QUEUE)
+  else if (oif->m.rx_mode == APPL_MODE_PKT_QUEUE)
     odp_pktin_event_queue (oif->pktio, oif->rxq, oif->m.num_rx_queues);
-  if (mode->tx_mode == APPL_MODE_PKT_BURST)
+  if (oif->m.tx_mode == APPL_MODE_PKT_BURST)
     odp_pktout_queue (oif->pktio, oif->outq, oif->m.num_tx_queues);
-  else if (mode->tx_mode == APPL_MODE_PKT_QUEUE)
+  else if (oif->m.tx_mode == APPL_MODE_PKT_QUEUE)
     odp_pktout_event_queue (oif->pktio, oif->txq, oif->m.num_tx_queues);
 
   /* Use configured MAC or read MAC from DPDK */
@@ -243,12 +260,7 @@ odp_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set,
 
   /* Assign queues of the new interface to first available worker thread */
   for (j = 0; j < oif->m.num_rx_queues; j++)
-    {
-      if (mode->rx_mode == APPL_MODE_PKT_SCHED)
-       vnet_hw_interface_assign_rx_thread (vnm, oif->hw_if_index, 0, j + 1);
-      else
-       vnet_hw_interface_assign_rx_thread (vnm, oif->hw_if_index, j, -1);
-    }
+    vnet_hw_interface_assign_rx_thread (vnm, oif->hw_if_index, j, -1);
 
   return 0;
 
@@ -327,8 +339,12 @@ odp_device_config (char *name, unformat_input_t * input)
                mode.rx_mode = APPL_MODE_PKT_BURST;
              else if (!strcmp (val, "queue") || !strcmp (val, "1"))
                mode.rx_mode = APPL_MODE_PKT_QUEUE;
+             else if (!strcmp (val, "sched-p") || !strcmp (val, "4"))
+               mode.rx_mode = APPL_MODE_PKT_SCHED_PARALLEL;
+             else if (!strcmp (val, "sched-o") || !strcmp (val, "3"))
+               mode.rx_mode = APPL_MODE_PKT_SCHED_ORDERED;
              else if (!strcmp (val, "sched") || !strcmp (val, "2"))
-               mode.rx_mode = APPL_MODE_PKT_SCHED;
+               mode.rx_mode = APPL_MODE_PKT_SCHED_ATOMIC;
              vec_free (val);
            }
          else if (unformat (input, "tx-mode %s", &val))

diff --git a/src/plugins/odp/odp_packet.h b/src/plugins/odp/odp_packet.h
index f3b478b..edd3eca 100755 (executable)
--- a/src/plugins/odp/odp_packet.h
+++ b/src/plugins/odp/odp_packet.h
@@ -11,10 +11,13 @@
 #define SHM_PKT_POOL_BUF_SIZE  1856
 #define SHM_PKT_POOL_NB_PKTS   10240
 #define SHM_PKT_POOL_NAME      "packet_pool"
+
 #define APPL_MODE_PKT_BURST    0
 #define APPL_MODE_PKT_QUEUE    1
-#define APPL_MODE_PKT_SCHED    2
-#define APPL_MODE_PKT_TM       3
+#define APPL_MODE_PKT_SCHED_ATOMIC    2
+#define APPL_MODE_PKT_SCHED_ORDERED   3
+#define APPL_MODE_PKT_SCHED_PARALLEL  4
+#define APPL_MODE_PKT_TM       2
 
 #define MAX_WORKERS 32
 #define MAX_QUEUES (MAX_WORKERS + 1)
@@ -52,6 +55,7 @@ typedef struct
   odp_pktout_queue_t outq[MAX_QUEUES];
   odp_queue_t txq[MAX_QUEUES];
   odp_if_mode_t m;
+  odp_schedule_group_t sched_group;
 } odp_packet_if_t;
 
 typedef struct

diff --git a/src/vpp/conf/startup.conf b/src/vpp/conf/startup.conf
index 6438f77..d6d2d27 100644 (file)
--- a/src/vpp/conf/startup.conf
+++ b/src/vpp/conf/startup.conf
@@ -101,6 +101,7 @@ cpu {
 # odp {
        # platform-params {
        #   memory 100
+       #   cmdline { <additional-parameters> }
        # }
 
        ## Number of packets in allocated pool
@@ -113,8 +114,9 @@ cpu {
        # tx-burst-size 32
 
        ## Default configuration for all ODP devices.
-       ## rx-mode can be set to burst, queue or sched
-       ## tx-mode can be burst, queue or tm (traffic manager)
+       ## rx-mode can be set to burst (default), queue, sched(-atomic),
+       ##      sched-ordered, sched-parallel
+       ## tx-mode can be burst (default) or queue (forced for sched-ordered)
        ## num-rx-queues can be set to more than 1 to enable RSS
        ## num-tx-queues can be 'max' (separate queue for each thread) or 1
        # default {