return s;
}
-#ifndef CLIB_MARCH_VARIANT
-inline void
-virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring)
+static_always_inline void
+virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring)
{
u16 used = vring->desc_in_use;
u16 sz = vring->size;
while (n_left)
{
struct vring_used_elem *e = &vring->used->ring[last & mask];
- u16 slot = e->id;
+ u16 slot, n_buffers;
+ slot = n_buffers = e->id;
- vlib_buffer_free (vm, &vring->buffers[slot], 1);
- used--;
- last++;
- n_left--;
+ while (e->id == n_buffers)
+ {
+ n_left--;
+ last++;
+ n_buffers++;
+ if (n_left == 0)
+ break;
+ e = &vring->used->ring[last & mask];
+ }
+ vlib_buffer_free_from_ring (vm, vring->buffers, slot,
+ sz, (n_buffers - slot));
+ used -= (n_buffers - slot);
+
+ if (n_left > 0)
+ {
+ slot = e->id;
+
+ vlib_buffer_free (vm, &vring->buffers[slot], 1);
+ used--;
+ last++;
+ n_left--;
+ }
}
vring->desc_in_use = used;
vring->last_used_idx = last;
}
-#endif /* CLIB_MARCH_VARIANT */
static_always_inline u16
add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif,
* It can easily support 65535 bytes of Jumbo frames with
* each data buffer size of 512 bytes minimum.
*/
- vlib_buffer_t *indirect_desc =
- vlib_get_buffer (vm, vring->indirect_buffers[next]);
+ u32 indirect_buffer = 0;
+ if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
+ return n_added;
+
+ vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
indirect_desc->current_data = 0;
+ indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
+ indirect_desc->next_buffer = bi;
+ bi = indirect_buffer;
struct vring_desc *id =
(struct vring_desc *) vlib_buffer_get_current (indirect_desc);
virtio_kick (vm, vring, vif);
/* free consumed buffers */
- virtio_free_used_desc (vm, vring);
+ virtio_free_used_device_desc (vm, vring);
used = vring->desc_in_use;
next = vring->desc_next;
ASSERT (vring->buffers == 0);
vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
- ASSERT (vring->indirect_buffers == 0);
- vec_validate_aligned (vring->indirect_buffers, queue_size,
- CLIB_CACHE_LINE_BYTES);
if (queue_num % 2)
{
- u32 n_alloc = 0;
- do
- {
- if (n_alloc < queue_size)
- n_alloc +=
- vlib_buffer_alloc (vm, vring->indirect_buffers + n_alloc,
- queue_size - n_alloc);
- }
- while (n_alloc != queue_size);
virtio_log_debug (vim, vif, "tx-queue: number %u, size %u", queue_num,
queue_size);
}
virtio_free_rx_buffers (vm, vring);
}
vec_free (vring->buffers);
- vec_free (vring->indirect_buffers);
vlib_physmem_free (vm, vring->desc);
}
{
virtio_free_used_desc (vm, vring);
}
- if (vring->queue_id % 2)
- {
- vlib_buffer_free_no_next (vm, vring->indirect_buffers, vring->size);
- }
vec_free (vring->buffers);
- vec_free (vring->indirect_buffers);
vlib_physmem_free (vm, vring->desc);
}
vring->queue_id = idx;
ASSERT (vring->buffers == 0);
vec_validate_aligned (vring->buffers, sz, CLIB_CACHE_LINE_BYTES);
- ASSERT (vring->indirect_buffers == 0);
- vec_validate_aligned (vring->indirect_buffers, sz, CLIB_CACHE_LINE_BYTES);
- if (idx % 2)
- {
- u32 n_alloc = 0;
- do
- {
- if (n_alloc < sz)
- n_alloc +=
- vlib_buffer_alloc (vm, vring->indirect_buffers + n_alloc,
- sz - n_alloc);
- }
- while (n_alloc != sz);
- }
vring->size = sz;
vring->call_fd = eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC);
if (vring->avail)
clib_mem_free (vring->avail);
vec_free (vring->buffers);
- vec_free (vring->indirect_buffers);
return 0;
}
+inline void
+virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring)
+{
+ u16 used = vring->desc_in_use;
+ u16 sz = vring->size;
+ u16 mask = sz - 1;
+ u16 last = vring->last_used_idx;
+ u16 n_left = vring->used->idx - last;
+
+ if (n_left == 0)
+ return;
+
+ while (n_left)
+ {
+ struct vring_used_elem *e = &vring->used->ring[last & mask];
+ u16 slot = e->id;
+
+ vlib_buffer_free (vm, &vring->buffers[slot], 1);
+ used--;
+ last++;
+ n_left--;
+ }
+ vring->desc_in_use = used;
+ vring->last_used_idx = last;
+}
+
clib_error_t *
virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx)
{
clib_mem_free (vring->desc);
if (vring->avail)
clib_mem_free (vring->avail);
- vlib_buffer_free_no_next (vm, vring->indirect_buffers, vring->size);
vec_free (vring->buffers);
- vec_free (vring->indirect_buffers);
return 0;
}
u16 flags;
u32 call_file_index;
u32 *buffers;
- u32 *indirect_buffers;
u16 last_used_idx;
u16 last_kick_avail_idx;
} virtio_vring_t;