dev: add ability for process node to wait for device config

This also reverts commit f36243c which was causing issues with queue
setup.

Change-Id: I5997d226c4bbf8c58d9ad538fa59563ea4fe2f69
Type: fix
Fixes: f36243c
Signed-off-by: Damjan Marion <[email protected]>

diff --git a/src/plugins/dev_ena/aq.c b/src/plugins/dev_ena/aq.c
index 8dfc3fc..290d5bd 100644 (file)
--- a/src/plugins/dev_ena/aq.c
+++ b/src/plugins/dev_ena/aq.c
@@ -153,7 +153,7 @@ ena_aq_req (vlib_main_t *vm, vnet_dev_t *dev, ena_aq_opcode_t opcode,
 
   while (cqe->phase != phase)
     {
-      vnet_dev_process_suspend (vm, suspend_time);
+      vlib_process_suspend (vm, suspend_time);
       suspend_time *= 2;
       if (suspend_time > 1e-3)
        {

diff --git a/src/plugins/dev_ena/reg.c b/src/plugins/dev_ena/reg.c
index 7aa4252..7f2cc0f 100644 (file)
--- a/src/plugins/dev_ena/reg.c
+++ b/src/plugins/dev_ena/reg.c
@@ -144,7 +144,7 @@ ena_reg_reset (vlib_main_t *vm, vnet_dev_t *dev, ena_reset_reason_t reason)
        break;
       if (i++ == 20)
        return ena_err (dev, VNET_DEV_ERR_BUS, "failed to initiate reset");
-      vnet_dev_process_suspend (vm, 0.001);
+      vlib_process_suspend (vm, 0.001);
     }
 
   ena_reg_write (dev, ENA_REG_DEV_CTL, &(ena_reg_dev_ctl_t){});
@@ -158,7 +158,7 @@ ena_reg_reset (vlib_main_t *vm, vnet_dev_t *dev, ena_reset_reason_t reason)
        break;
       if (i++ == 20)
        return ena_err (dev, VNET_DEV_ERR_BUS, "failed to complete reset");
-      vnet_dev_process_suspend (vm, 0.001);
+      vlib_process_suspend (vm, 0.001);
     }
 
   ena_reg_read (dev, ENA_REG_VERSION, &ver);

diff --git a/src/plugins/dev_iavf/adminq.c b/src/plugins/dev_iavf/adminq.c
index ec60288..2072c69 100644 (file)
--- a/src/plugins/dev_iavf/adminq.c
+++ b/src/plugins/dev_iavf/adminq.c
@@ -416,7 +416,7 @@ iavf_aq_atq_enq (vlib_main_t *vm, vnet_dev_t *dev, iavf_aq_desc_t *desc,
              return VNET_DEV_ERR_TIMEOUT;
            }
 
-         vnet_dev_process_suspend (vm, suspend_time);
+         vlib_process_suspend (vm, suspend_time);
          suspend_time *= 2;
        }
     }
@@ -458,7 +458,7 @@ iavf_aq_arq_next_acq (vlib_main_t *vm, vnet_dev_t *dev, iavf_aq_desc_t **dp,
          if (vlib_time_now (vm) - t0 > timeout)
            return 0;
 
-         vnet_dev_process_suspend (vm, suspend_time);
+         vlib_process_suspend (vm, suspend_time);
 
          suspend_time *= 2;
        }

diff --git a/src/plugins/dev_iavf/iavf.c b/src/plugins/dev_iavf/iavf.c
index d6543c6..f13440f 100644 (file)
--- a/src/plugins/dev_iavf/iavf.c
+++ b/src/plugins/dev_iavf/iavf.c
@@ -98,7 +98,7 @@ iavf_reset (vlib_main_t *vm, vnet_dev_t *dev)
     {
       if (n_tries-- == 0)
        return VNET_DEV_ERR_TIMEOUT;
-      vnet_dev_process_suspend (vm, 0.02);
+      vlib_process_suspend (vm, 0.02);
     }
   while ((iavf_reg_read (ad, IAVF_VFGEN_RSTAT) & 3) != 2);
 

diff --git a/src/vnet/dev/config.c b/src/vnet/dev/config.c
index df4ab31..ec1c35b 100644 (file)
--- a/src/vnet/dev/config.c
+++ b/src/vnet/dev/config.c
@@ -200,24 +200,34 @@ vnet_dev_config_one_device (vlib_main_t *vm, unformat_input_t *input,
   return err;
 }
 
-static clib_error_t *
-devices_config (vlib_main_t *vm, unformat_input_t *input)
+uword
+dev_config_process_node_fn (vlib_main_t *vm, vlib_node_runtime_t *rt,
+                           vlib_frame_t *f)
 {
+  vnet_dev_main_t *dm = &vnet_dev_main;
   vnet_dev_driver_name_t driver_name;
+  unformat_input_t input;
   clib_error_t *err = 0;
+  u32 *i;
+
+  if (dm->startup_config == 0)
+    goto done;
 
-  while (!err && unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+  unformat_init_vector (&input, dm->startup_config);
+  dm->startup_config = 0;
+
+  while (!err && unformat_check_input (&input) != UNFORMAT_END_OF_INPUT)
     {
       unformat_input_t sub_input;
       vnet_dev_device_id_t device_id;
-      if (unformat (input, "dev %U %U", unformat_c_string_array, device_id,
+      if (unformat (&input, "dev %U %U", unformat_c_string_array, device_id,
                    sizeof (device_id), unformat_vlib_cli_sub_input,
                    &sub_input))
        {
          err = vnet_dev_config_one_device (vm, &sub_input, device_id);
          unformat_free (&sub_input);
        }
-      else if (unformat (input, "dev %U", unformat_c_string_array, device_id,
+      else if (unformat (&input, "dev %U", unformat_c_string_array, device_id,
                         sizeof (device_id)))
        {
          unformat_input_t no_input = {};
@@ -225,7 +235,7 @@ devices_config (vlib_main_t *vm, unformat_input_t *input)
          err = vnet_dev_config_one_device (vm, &no_input, device_id);
          unformat_free (&no_input);
        }
-      else if (unformat (input, "driver %U %U", unformat_c_string_array,
+      else if (unformat (&input, "driver %U %U", unformat_c_string_array,
                         driver_name, sizeof (driver_name),
                         unformat_vlib_cli_sub_input, &sub_input))
        {
@@ -237,9 +247,58 @@ devices_config (vlib_main_t *vm, unformat_input_t *input)
                                 format_unformat_error, &input);
     }
 
-  unformat_free (input);
+  unformat_free (&input);
 
-  return err;
+  if (err)
+    {
+      log_err (0, "%U", format_clib_error, err);
+      clib_error_free (err);
+    }
+
+done:
+  dm->startup_config_completed = 1;
+  vec_foreach (i, dm->process_nodes_waiting_for_startup_conf)
+    vlib_process_signal_event (vm, *i, 0, 0);
+  vec_free (dm->process_nodes_waiting_for_startup_conf);
+
+  vlib_node_set_state (vm, rt->node_index, VLIB_NODE_STATE_DISABLED);
+  vlib_node_rename (vm, rt->node_index, "deleted-%u", rt->node_index);
+  vec_add1 (dm->free_process_node_indices, rt->node_index);
+  return 0;
+}
+
+VLIB_REGISTER_NODE (dev_config_process_node) = {
+  .function = dev_config_process_node_fn,
+  .type = VLIB_NODE_TYPE_PROCESS,
+  .name = "dev-config",
+};
+
+static clib_error_t *
+devices_config (vlib_main_t *vm, unformat_input_t *input)
+{
+  vnet_dev_main_t *dm = &vnet_dev_main;
+  uword c;
+
+  while ((c = unformat_get_input (input)) != UNFORMAT_END_OF_INPUT)
+    vec_add1 (dm->startup_config, c);
+
+  return 0;
 }
 
 VLIB_CONFIG_FUNCTION (devices_config, "devices");
+
+void
+vnet_dev_wait_for_startup_config_complete (vlib_main_t *vm)
+{
+  vnet_dev_main_t *dm = &vnet_dev_main;
+  uword *event_data = 0;
+
+  if (dm->startup_config_completed)
+    return;
+
+  vec_add1 (dm->process_nodes_waiting_for_startup_conf,
+           vlib_get_current_process_node_index (vm));
+  vlib_process_wait_for_event (vm);
+  vlib_process_get_events (vm, &event_data);
+  vec_free (event_data);
+}

diff --git a/src/vnet/dev/dev.c b/src/vnet/dev/dev.c
index bf83417..7954707 100644 (file)
--- a/src/vnet/dev/dev.c
+++ b/src/vnet/dev/dev.c
@@ -420,6 +420,9 @@ vnet_dev_main_init (vlib_main_t *vm)
        temp_space_sz = r->runtime_temp_space_sz;
     }
 
+  if (dm->startup_config)
+    log_debug (0, "startup config: %v", dm->startup_config);
+
   vec_free (drv);
 
   if (temp_space_sz > 0)

diff --git a/src/vnet/dev/dev.h b/src/vnet/dev/dev.h
index 6c79a8b..144003c 100644 (file)
--- a/src/vnet/dev/dev.h
+++ b/src/vnet/dev/dev.h
@@ -514,9 +514,13 @@ typedef struct
   u32 *free_rx_node_indices;
   uword *device_index_by_id;
 
+  /* startup config */
+  u8 *startup_config;
+  u32 *process_nodes_waiting_for_startup_conf;
+  u8 startup_config_completed;
+
   u16 next_rx_queue_thread;
   u8 eth_port_rx_feature_arc_index;
-  u8 main_loop_running;
 } vnet_dev_main_t;
 
 extern vnet_dev_main_t vnet_dev_main;
@@ -576,6 +580,9 @@ format_function_t format_vnet_dev_arg_type;
 format_function_t format_vnet_dev_arg_value;
 format_function_t format_vnet_dev_args;
 
+/* config.c */
+void vnet_dev_wait_for_startup_config_complete (vlib_main_t *);
+
 /* dev.c */
 vnet_dev_t *vnet_dev_alloc (vlib_main_t *, vnet_dev_device_id_t,
                            vnet_dev_driver_t *);

diff --git a/src/vnet/dev/dev_funcs.h b/src/vnet/dev/dev_funcs.h
index 95ea22e..27172f7 100644 (file)
--- a/src/vnet/dev/dev_funcs.h
+++ b/src/vnet/dev/dev_funcs.h
@@ -168,19 +168,15 @@ vnet_dev_get_bus (vnet_dev_t *dev)
 static_always_inline void
 vnet_dev_validate (vlib_main_t *vm, vnet_dev_t *dev)
 {
-  vnet_dev_main_t *dm = &vnet_dev_main;
-  ASSERT (!dm->main_loop_running ||
-         dev->process_node_index == vlib_get_current_process_node_index (vm));
+  ASSERT (dev->process_node_index == vlib_get_current_process_node_index (vm));
   ASSERT (vm->thread_index == 0);
 }
 
 static_always_inline void
 vnet_dev_port_validate (vlib_main_t *vm, vnet_dev_port_t *port)
 {
-  vnet_dev_main_t *dm = &vnet_dev_main;
-  ASSERT (!dm->main_loop_running ||
-         port->dev->process_node_index ==
-           vlib_get_current_process_node_index (vm));
+  ASSERT (port->dev->process_node_index ==
+         vlib_get_current_process_node_index (vm));
   ASSERT (vm->thread_index == 0);
 }
 
@@ -437,25 +433,4 @@ vnet_dev_arg_get_string (vnet_dev_arg_t *arg)
   return arg->val_set ? arg->val.string : arg->default_val.string;
 }
 
-static_always_inline void
-vnet_dev_process_suspend (vlib_main_t *vm, f64 t)
-{
-  if (!vnet_dev_main.main_loop_running)
-    {
-      t += vlib_time_now (vm);
-
-      while (vlib_time_now (vm) < t)
-       CLIB_PAUSE ();
-    }
-  else
-    vlib_process_suspend (vm, t);
-}
-
-static_always_inline void
-vnet_dev_process_yield (vlib_main_t *vm, f64 t)
-{
-  if (vnet_dev_main.main_loop_running)
-    vlib_process_yield (vm);
-}
-
 #endif /* _VNET_DEV_FUNCS_H_ */

diff --git a/src/vnet/dev/process.c b/src/vnet/dev/process.c
index 55dce8d..4df5629 100644 (file)
--- a/src/vnet/dev/process.c
+++ b/src/vnet/dev/process.c
@@ -320,20 +320,11 @@ static vnet_dev_rv_t
 vnet_dev_process_event_send_and_wait (vlib_main_t *vm, vnet_dev_t *dev,
                                      vnet_dev_event_data_t *ed)
 {
-  vnet_dev_main_t *dm = &vnet_dev_main;
+  ed->calling_process_index = vlib_get_current_process_node_index (vm);
   vnet_dev_rv_t rv = VNET_DEV_ERR_PROCESS_REPLY;
 
   ed->reply_needed = 1;
 
-  if (!dm->main_loop_running)
-    {
-      vnet_dev_process_one_event (vm, dev, ed);
-      rv = ed->rv;
-      goto done;
-    }
-
-  ed->calling_process_index = vlib_get_current_process_node_index (vm);
-
   if (ed->calling_process_index == dev->process_node_index)
     {
       vnet_dev_process_one_event (vm, dev, ed);
@@ -586,13 +577,3 @@ vnet_dev_process_call_port_op_no_wait (vlib_main_t *vm, vnet_dev_port_t *port,
 
   vnet_dev_process_event_send (vm, port->dev, ed);
 }
-
-static clib_error_t *
-vnet_dev_main_loop_init (vlib_main_t *vm)
-{
-  vnet_dev_main_t *dm = &vnet_dev_main;
-  dm->main_loop_running = 1;
-  return 0;
-}
-
-VLIB_MAIN_LOOP_ENTER_FUNCTION (vnet_dev_main_loop_init);