#include <vnet/ip/ip.h>
#include <vnet/ethernet/ethernet.h>
#include <vnet/interface/rx_queue_funcs.h>
+#include <vnet/interface/tx_queue_funcs.h>
#include <vlib/unix/unix.h>
VLIB_REGISTER_LOG_CLASS (if_rxq_log, static) = {
vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
u32 node_index = hi->input_node_index;
vnet_hw_if_rx_queue_t *rxq;
- vnet_hw_if_rxq_poll_vector_t *pv, **d = 0;
+ vnet_hw_if_rxq_poll_vector_t *pv, **d = 0, **a = 0;
+ vnet_hw_if_output_node_runtime_t *new_out_runtimes = 0;
vlib_node_state_t *per_thread_node_state = 0;
- u32 n_threads = vec_len (vlib_mains);
+ u32 n_threads = vlib_get_n_threads ();
u16 *per_thread_node_adaptive = 0;
- int something_changed = 0;
+ int something_changed_on_rx = 0;
+ int something_changed_on_tx = 0;
clib_bitmap_t *pending_int = 0;
int last_int = -1;
format_vlib_node_name, vm, node_index, hi->name);
vec_validate (d, n_threads - 1);
+ vec_validate (a, n_threads - 1);
vec_validate_init_empty (per_thread_node_state, n_threads - 1,
VLIB_NODE_STATE_DISABLED);
vec_validate_init_empty (per_thread_node_adaptive, n_threads - 1, 0);
pool_foreach (rxq, im->hw_if_rx_queues)
{
u32 ti = rxq->thread_index;
+ vnet_hw_interface_t *rxq_hi;
ASSERT (rxq->mode != VNET_HW_IF_RX_MODE_UNKNOWN);
ASSERT (rxq->mode != VNET_HW_IF_RX_MODE_DEFAULT);
- hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
+ rxq_hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
- if (hi->input_node_index != node_index)
+ if (rxq_hi->input_node_index != node_index)
continue;
if (rxq->mode == VNET_HW_IF_RX_MODE_POLLING)
pool_foreach (rxq, im->hw_if_rx_queues)
{
u32 ti = rxq->thread_index;
+ vnet_hw_interface_t *rxq_hi;
- hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
+ rxq_hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
- if (hi->input_node_index != node_index)
+ if (rxq_hi->input_node_index != node_index)
continue;
if (rxq->mode == VNET_HW_IF_RX_MODE_INTERRUPT ||
rxq->mode == VNET_HW_IF_RX_MODE_ADAPTIVE)
last_int = clib_max (last_int, rxq - im->hw_if_rx_queues);
+ if (per_thread_node_adaptive[ti])
+ {
+ vec_add2_aligned (a[ti], pv, 1, CLIB_CACHE_LINE_BYTES);
+ pv->dev_instance = rxq->dev_instance;
+ pv->queue_id = rxq->queue_id;
+ }
+
if (per_thread_node_state[ti] != VLIB_NODE_STATE_POLLING)
continue;
* unnecesary barrier */
for (int i = 0; i < n_threads; i++)
{
+ vlib_main_t *ovm = vlib_get_main_by_index (i);
vlib_node_state_t old_state;
vec_sort_with_function (d[i], poll_data_sort);
- old_state = vlib_node_get_state (vlib_mains[i], node_index);
+ old_state = vlib_node_get_state (ovm, node_index);
if (per_thread_node_state[i] != old_state)
{
- something_changed = 1;
+ something_changed_on_rx = 1;
log_debug ("state changed for node %U on thread %u from %s to %s",
format_vlib_node_name, vm, node_index, i,
node_state_str[old_state],
}
/* check if something changed */
- if (something_changed == 0)
+ if (something_changed_on_rx == 0)
{
vnet_hw_if_rx_node_runtime_t *rt;
- rt = vlib_node_get_runtime_data (vlib_mains[i], node_index);
- if (vec_len (rt->rxq_poll_vector) != vec_len (d[i]))
- something_changed = 1;
- else if (memcmp (d[i], rt->rxq_poll_vector,
- vec_len (d[i]) * sizeof (*d)))
- something_changed = 1;
+ rt = vlib_node_get_runtime_data (ovm, node_index);
+ if (vec_len (rt->rxq_vector_int) != vec_len (d[i]))
+ something_changed_on_rx = 1;
+ else if (memcmp (d[i], rt->rxq_vector_int,
+ vec_len (d[i]) * sizeof (**d)))
+ something_changed_on_rx = 1;
if (clib_interrupt_get_n_int (rt->rxq_interrupts) != last_int + 1)
- something_changed = 1;
+ something_changed_on_rx = 1;
+
+ if (something_changed_on_rx == 0 && per_thread_node_adaptive[i])
+ {
+ if (vec_len (rt->rxq_vector_poll) != vec_len (a[i]))
+ something_changed_on_rx = 1;
+ else if (memcmp (a[i], rt->rxq_vector_poll,
+ vec_len (a[i]) * sizeof (**a)))
+ something_changed_on_rx = 1;
+ }
}
}
- if (something_changed)
+ new_out_runtimes =
+ vec_dup_aligned (hi->output_node_thread_runtimes, CLIB_CACHE_LINE_BYTES);
+ vec_validate_aligned (new_out_runtimes, n_threads - 1,
+ CLIB_CACHE_LINE_BYTES);
+
+ if (vec_len (hi->output_node_thread_runtimes) != vec_len (new_out_runtimes))
+ something_changed_on_tx = 1;
+
+ for (int i = 0; i < vec_len (hi->tx_queue_indices); i++)
+ {
+ u32 thread_index;
+ u32 queue_index = hi->tx_queue_indices[i];
+ vnet_hw_if_tx_queue_t *txq = vnet_hw_if_get_tx_queue (vnm, queue_index);
+ uword n_threads = clib_bitmap_count_set_bits (txq->threads);
+
+ clib_bitmap_foreach (thread_index, txq->threads)
+ {
+ vnet_hw_if_output_node_runtime_t *rt;
+ rt = vec_elt_at_index (new_out_runtimes, thread_index);
+ if ((rt->frame.queue_id != txq->queue_id) ||
+ (rt->n_threads != n_threads))
+ {
+ log_debug ("tx queue data changed for interface %v, thread %u "
+ "(queue_id %u -> %u, n_threads %u -> %u)",
+ hi->name, thread_index, rt->frame.queue_id,
+ txq->queue_id, rt->n_threads, n_threads);
+ something_changed_on_tx = 1;
+ rt->frame.queue_id = txq->queue_id;
+ rt->frame.shared_queue = txq->shared_queue;
+ rt->n_threads = n_threads;
+ }
+ }
+ }
+
+ if (something_changed_on_rx || something_changed_on_tx)
{
int with_barrier;
if (with_barrier)
vlib_worker_thread_barrier_sync (vm);
- for (int i = 0; i < n_threads; i++)
+ if (something_changed_on_rx)
{
- vlib_main_t *vm = vlib_mains[i];
- vnet_hw_if_rx_node_runtime_t *rt;
- rt = vlib_node_get_runtime_data (vm, node_index);
- pv = rt->rxq_poll_vector;
- rt->rxq_poll_vector = d[i];
- d[i] = pv;
-
- if (rt->rxq_interrupts)
+ for (int i = 0; i < n_threads; i++)
{
- void *in = rt->rxq_interrupts;
- int int_num = -1;
- while ((int_num = clib_interrupt_get_next (in, int_num)) != -1)
+ vlib_main_t *vm = vlib_get_main_by_index (i);
+ vnet_hw_if_rx_node_runtime_t *rt;
+ rt = vlib_node_get_runtime_data (vm, node_index);
+ pv = rt->rxq_vector_int;
+ rt->rxq_vector_int = d[i];
+ d[i] = pv;
+
+ if (per_thread_node_adaptive[i])
{
- clib_interrupt_clear (in, int_num);
- pending_int = clib_bitmap_set (pending_int, int_num, 1);
- last_int = clib_max (last_int, int_num);
+ pv = rt->rxq_vector_poll;
+ rt->rxq_vector_poll = a[i];
+ a[i] = pv;
}
- }
- vlib_node_set_state (vm, node_index, per_thread_node_state[i]);
- vlib_node_set_flag (vm, node_index, VLIB_NODE_FLAG_ADAPTIVE_MODE,
- per_thread_node_adaptive[i]);
+ if (rt->rxq_interrupts)
+ {
+ void *in = rt->rxq_interrupts;
+ int int_num = -1;
+ while ((int_num = clib_interrupt_get_next (in, int_num)) !=
+ -1)
+ {
+ clib_interrupt_clear (in, int_num);
+ pending_int = clib_bitmap_set (pending_int, int_num, 1);
+ last_int = clib_max (last_int, int_num);
+ }
+ }
- if (last_int >= 0)
- clib_interrupt_resize (&rt->rxq_interrupts, last_int + 1);
- else
- clib_interrupt_free (&rt->rxq_interrupts);
+ vlib_node_set_state (vm, node_index, per_thread_node_state[i]);
+ vlib_node_set_flag (vm, node_index, VLIB_NODE_FLAG_ADAPTIVE_MODE,
+ per_thread_node_adaptive[i]);
+
+ if (last_int >= 0)
+ clib_interrupt_resize (&rt->rxq_interrupts, last_int + 1);
+ else
+ clib_interrupt_free (&rt->rxq_interrupts);
+ }
+ }
+ if (something_changed_on_tx)
+ {
+ vnet_hw_if_output_node_runtime_t *t;
+ t = hi->output_node_thread_runtimes;
+ hi->output_node_thread_runtimes = new_out_runtimes;
+ new_out_runtimes = t;
}
if (with_barrier)
}
for (int i = 0; i < n_threads; i++)
- vec_free (d[i]);
+ {
+ vec_free (d[i]);
+ vec_free (a[i]);
+ }
vec_free (d);
+ vec_free (a);
vec_free (per_thread_node_state);
vec_free (per_thread_node_adaptive);
+ vec_free (new_out_runtimes);
}
Code Review / vpp.git / blobdiff