recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi,
vlib_buffer_t * b)
{
- vlib_buffer_free_list_t *fl;
u32 thread_index = vlib_get_thread_index ();
- vlib_buffer_free_list_index_t fi;
- fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
- /* The only current use of this callback: multicast recycle */
- if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
- {
- int j;
-
- vlib_buffer_add_to_free_list (vm, fl, bi,
- (b->flags & VLIB_BUFFER_RECYCLE) == 0);
-
- for (j = 0; j < vec_len (vm->buffer_announce_list); j++)
- {
- if (fl == vm->buffer_announce_list[j])
- goto already_announced;
- }
- vec_add1 (vm->buffer_announce_list, fl);
- already_announced:
- ;
- }
- else if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
- dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
+ dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
}
static_always_inline void
vlib_buffer_t *bufp[n_buffers], **b = bufp;
u32 thread_index = vlib_get_thread_index ();
int i = 0;
- u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST | VLIB_BUFFER_RECYCLE |
+ u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST |
VLIB_BUFFER_NEXT_PRESENT);
u32 n_left, *bi;
u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
b += 1;
n_left -= 1;
}
- if (vec_len (vm->buffer_announce_list))
- {
- vlib_buffer_free_list_t *fl;
- for (i = 0; i < vec_len (vm->buffer_announce_list); i++)
- {
- fl = vm->buffer_announce_list[i];
- fl->buffers_added_to_freelist_function (vm, fl);
- }
- _vec_len (vm->buffer_announce_list) = 0;
- }
}
void
#define foreach_dpdk_tx_func_error \
_(BAD_RETVAL, "DPDK tx function returned an error") \
- _(PKT_DROP, "Tx packet drops (dpdk tx failure)") \
- _(REPL_FAIL, "Tx packet drops (replication failure)")
+ _(PKT_DROP, "Tx packet drops (dpdk tx failure)")
typedef enum
{
}
}
-static struct rte_mbuf *
-dpdk_replicate_packet_mb (vlib_buffer_t * b)
-{
- dpdk_main_t *dm = &dpdk_main;
- struct rte_mbuf **mbufs = 0, *s, *d;
- u8 nb_segs;
- unsigned socket_id = rte_socket_id ();
- int i;
-
- ASSERT (dm->pktmbuf_pools[socket_id]);
- s = rte_mbuf_from_vlib_buffer (b);
- nb_segs = s->nb_segs;
- vec_validate (mbufs, nb_segs - 1);
-
- if (rte_pktmbuf_alloc_bulk (dm->pktmbuf_pools[socket_id], mbufs, nb_segs))
- {
- vec_free (mbufs);
- return 0;
- }
-
- d = mbufs[0];
- d->nb_segs = s->nb_segs;
- d->data_len = s->data_len;
- d->pkt_len = s->pkt_len;
- d->data_off = s->data_off;
- clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len);
-
- for (i = 1; i < nb_segs; i++)
- {
- d->next = mbufs[i];
- d = mbufs[i];
- s = s->next;
- d->data_len = s->data_len;
- clib_memcpy (d->buf_addr, s->buf_addr,
- RTE_PKTMBUF_HEADROOM + s->data_len);
- }
-
- d = mbufs[0];
- vec_free (mbufs);
- return d;
-}
-
static void
dpdk_tx_trace_buffer (dpdk_main_t * dm, vlib_node_runtime_t * node,
dpdk_device_t * xd, u16 queue_id,
CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
}
-static_always_inline void
-dpdk_buffer_recycle (vlib_main_t * vm, vlib_node_runtime_t * node,
- vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp)
-{
- dpdk_main_t *dm = &dpdk_main;
- struct rte_mbuf *mb_new;
-
- if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0)
- return;
-
- mb_new = dpdk_replicate_packet_mb (b);
- if (PREDICT_FALSE (mb_new == 0))
- {
- vlib_error_count (vm, node->node_index,
- DPDK_TX_FUNC_ERROR_REPL_FAIL, 1);
- b->flags |= VLIB_BUFFER_REPL_FAIL;
- }
- else
- *mbp = mb_new;
-
- vec_add1 (dm->recycle[vm->thread_index], bi);
-}
-
static_always_inline void
dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b,
struct rte_mbuf *mb)
n_left--;
}
- /* run inly if we have buffers to recycle */
- if (PREDICT_FALSE (all_or_flags & VLIB_BUFFER_RECYCLE))
- {
- struct rte_mbuf **mb_old;
- from = vlib_frame_vector_args (f);
- n_left = n_packets;
- mb_old = mb = ptd->mbufs;
- while (n_left > 0)
- {
- b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
- dpdk_buffer_recycle (vm, node, b[0], from[0], &mb_old[0]);
-
- /* in case of REPL_FAIL we need to shift data */
- mb[0] = mb_old[0];
-
- if (PREDICT_TRUE ((b[0]->flags & VLIB_BUFFER_REPL_FAIL) == 0))
- mb++;
- mb_old++;
- from++;
- n_left--;
- }
- }
-
/* transmit as many packets as possible */
tx_pkts = n_packets = mb - ptd->mbufs;
n_left = tx_burst_vector_internal (vm, xd, ptd->mbufs, n_packets);
}
}
- /* Recycle replicated buffers */
- if (PREDICT_FALSE (vec_len (dm->recycle[thread_index])))
- {
- vlib_buffer_free (vm, dm->recycle[thread_index],
- vec_len (dm->recycle[thread_index]));
- _vec_len (dm->recycle[thread_index]) = 0;
- }
-
return tx_pkts;
}
dpdk_device_and_queue_t **devices_by_hqos_cpu;
dpdk_per_thread_data_t *per_thread_data;
- /* per-thread recycle lists */
- u32 **recycle;
-
/* buffer flags template, configurable to enable/disable tcp / udp cksum */
u32 buffer_flags_template;
{
dpdk_main_t *dm = &dpdk_main;
clib_error_t *error = 0;
- vlib_thread_main_t *tm = vlib_get_thread_main ();
/* verify that structs are cacheline aligned */
STATIC_ASSERT (offsetof (dpdk_device_t, cacheline0) == 0,
dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF;
vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet");
- vec_validate (dm->recycle, tm->n_thread_stacks - 1);
-
/* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */
dm->buffer_flags_template =
(VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_EXT_HDR_VALID
{
vlib_buffer_free_list_t *fl;
vlib_buffer_free_list_index_t fi;
- fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
+ u32 flags, next;
- /* The only current use of this callback:
- * multicast recycle */
- if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
- {
- int j;
+ fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
- vlib_buffer_add_to_free_list (vm, fl, bi,
- (b->flags & VLIB_BUFFER_RECYCLE) == 0);
- for (j = 0; j < vec_len (vm->buffer_announce_list); j++)
- {
- if (fl == vm->buffer_announce_list[j])
- goto already_announced;
- }
- vec_add1 (vm->buffer_announce_list, fl);
- already_announced:
- ;
- }
- else
+ do
{
- if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
+ vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
+ flags = nb->flags;
+ next = nb->next_buffer;
+ if (nb->n_add_refs)
+ nb->n_add_refs--;
+ else
{
- u32 flags, next;
-
- do
- {
- vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
- flags = nb->flags;
- next = nb->next_buffer;
- if (nb->n_add_refs)
- nb->n_add_refs--;
- else
- {
- vlib_buffer_validate_alloc_free (vm, &bi, 1,
- VLIB_BUFFER_KNOWN_ALLOCATED);
- vlib_buffer_add_to_free_list (vm, fl, bi, 1);
- }
- bi = next;
- }
- while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT));
-
+ vlib_buffer_validate_alloc_free (vm, &bi, 1,
+ VLIB_BUFFER_KNOWN_ALLOCATED);
+ vlib_buffer_add_to_free_list (vm, fl, bi, 1);
}
+ bi = next;
}
+ while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT));
}
static_always_inline void
recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
i++;
}
-
- if (vec_len (vm->buffer_announce_list))
- {
- vlib_buffer_free_list_t *fl;
- for (i = 0; i < vec_len (vm->buffer_announce_list); i++)
- {
- fl = vm->buffer_announce_list[i];
- fl->buffers_added_to_freelist_function (vm, fl);
- }
- _vec_len (vm->buffer_announce_list) = 0;
- }
}
static void
_( 0, NON_DEFAULT_FREELIST, "non-default-fl") \
_( 1, IS_TRACED, 0) \
_( 2, NEXT_PRESENT, 0) \
- _( 3, IS_RECYCLED, "is-recycled") \
- _( 4, TOTAL_LENGTH_VALID, 0) \
- _( 5, REPL_FAIL, "repl-fail") \
- _( 6, RECYCLE, "recycle") \
- _( 7, EXT_HDR_VALID, "ext-hdr-valid")
+ _( 3, TOTAL_LENGTH_VALID, 0) \
+ _( 4, EXT_HDR_VALID, "ext-hdr-valid")
/* NOTE: only buffer generic flags should be defined here, please consider
using user flags. i.e. src/vnet/buffer.h */
<br> VLIB_BUFFER_IS_TRACED: trace this buffer.
<br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
<br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
- <br> VLIB_BUFFER_REPL_FAIL: packet replication failure
- <br> VLIB_BUFFER_RECYCLE: as it says
<br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
set to avoid adding it to a flow report
<br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
struct vlib_buffer_free_list_t * fl,
u32 * buffers, u32 n_buffers);
- /* Callback function to announce that buffers have been
- added to the freelist */
- void (*buffers_added_to_freelist_function)
- (struct vlib_main_t * vm, struct vlib_buffer_free_list_t * fl);
-
uword buffer_init_function_opaque;
} __attribute__ ((aligned (16))) vlib_buffer_free_list_t;
/* Pool of buffer free lists. */
vlib_buffer_free_list_t *buffer_free_list_pool;
- /* List of free-lists needing Blue Light Special announcements */
- vlib_buffer_free_list_t **buffer_announce_list;
-
/* Allocate/free buffer memory for DMA transfers, descriptor rings, etc.
buffer memory is guaranteed to be cache-aligned. */
interface_output.c
interface_stats.c
misc.c
- replication.c
)
list(APPEND VNET_HEADERS
l3_types.h
plugin/plugin.h
pipeline.h
- replication.h
vnet.h
vnet_all_api_h.h
vnet_msg_enum.h
#include <vnet/l2/l2_input.h>
#include <vnet/l2/feat_bitmap.h>
#include <vnet/l2/l2_bvi.h>
-#include <vnet/replication.h>
#include <vnet/l2/l2_fib.h>
#include <vppinfra/error.h>
+++ /dev/null
-/*
- * replication.c : packet replication
- *
- * Copyright (c) 2013 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <vlib/vlib.h>
-#include <vnet/vnet.h>
-#include <vppinfra/error.h>
-#include <vnet/ip/ip4_packet.h>
-#include <vnet/replication.h>
-
-
-replication_main_t replication_main;
-
-
-replication_context_t *
-replication_prep (vlib_main_t * vm,
- vlib_buffer_t * b0, u32 recycle_node_index, u32 l2_packet)
-{
- replication_main_t *rm = &replication_main;
- replication_context_t *ctx;
- uword thread_index = vm->thread_index;
- ip4_header_t *ip;
- u32 ctx_id;
-
- /* Allocate a context, reserve context 0 */
- if (PREDICT_FALSE (rm->contexts[thread_index] == 0))
- pool_get_aligned (rm->contexts[thread_index], ctx, CLIB_CACHE_LINE_BYTES);
-
- pool_get_aligned (rm->contexts[thread_index], ctx, CLIB_CACHE_LINE_BYTES);
- ctx_id = ctx - rm->contexts[thread_index];
-
- /* Save state from vlib buffer */
- ctx->saved_free_list_index = vlib_buffer_get_free_list_index (b0);
- ctx->current_data = b0->current_data;
- ctx->flags = b0->flags & VNET_BUFFER_FLAGS_VLAN_BITS;
-
- /* Set up vlib buffer hooks */
- b0->recycle_count = ctx_id;
- vlib_buffer_set_free_list_index (b0, rm->recycle_list_index);
- b0->flags |= VLIB_BUFFER_RECYCLE;
-
- /* Save feature state */
- ctx->recycle_node_index = recycle_node_index;
-
- /* Save vnet state */
- clib_memcpy (ctx->vnet_buffer, vnet_buffer (b0),
- sizeof (vnet_buffer_opaque_t));
-
- /* Save packet contents */
- ctx->l2_packet = l2_packet;
- ip = (ip4_header_t *) vlib_buffer_get_current (b0);
- if (l2_packet)
- {
- /* Save ethernet header */
- ctx->l2_header[0] = ((u64 *) ip)[0];
- ctx->l2_header[1] = ((u64 *) ip)[1];
- ctx->l2_header[2] = ((u64 *) ip)[2];
- /* set ip to the true ip header */
- ip = (ip4_header_t *) (((u8 *) ip) + vnet_buffer (b0)->l2.l2_len);
- }
-
- /*
- * Copy L3 fields.
- * We need to save TOS for ip4 and ip6 packets.
- * Fortunately the TOS field is
- * in the first two bytes of both the ip4 and ip6 headers.
- */
- ctx->ip_tos = *((u16 *) (ip));
-
- /*
- * Save the ip4 checksum as well. We just blindly save the corresponding two
- * bytes even for ip6 packets.
- */
- ctx->ip4_checksum = ip->checksum;
-
- return ctx;
-}
-
-
-replication_context_t *
-replication_recycle (vlib_main_t * vm, vlib_buffer_t * b0, u32 is_last)
-{
- replication_main_t *rm = &replication_main;
- replication_context_t *ctx;
- uword thread_index = vm->thread_index;
- ip4_header_t *ip;
-
- /* Get access to the replication context */
- ctx = pool_elt_at_index (rm->contexts[thread_index], b0->recycle_count);
-
- /* Restore vnet buffer state */
- clib_memcpy (vnet_buffer (b0), ctx->vnet_buffer,
- sizeof (vnet_buffer_opaque_t));
-
- /* Restore the vlan flags */
- b0->flags &= ~VNET_BUFFER_FLAGS_VLAN_BITS;
- b0->flags |= ctx->flags;
-
- /* Restore the packet start (current_data) and length */
- vlib_buffer_advance (b0, ctx->current_data - b0->current_data);
-
- /* Restore packet contents */
- ip = (ip4_header_t *) vlib_buffer_get_current (b0);
- if (ctx->l2_packet)
- {
- /* Restore ethernet header */
- ((u64 *) ip)[0] = ctx->l2_header[0];
- ((u64 *) ip)[1] = ctx->l2_header[1];
- ((u64 *) ip)[2] = ctx->l2_header[2];
- /* set ip to the true ip header */
- ip = (ip4_header_t *) (((u8 *) ip) + vnet_buffer (b0)->l2.l2_len);
- }
-
- // Restore L3 fields
- *((u16 *) (ip)) = ctx->ip_tos;
- ip->checksum = ctx->ip4_checksum;
-
- if (is_last)
- {
- /*
- * This is the last replication in the list.
- * Restore original buffer free functionality.
- */
- vlib_buffer_set_free_list_index (b0, ctx->saved_free_list_index);
- b0->flags &= ~VLIB_BUFFER_RECYCLE;
-
- /* Free context back to its pool */
- pool_put (rm->contexts[thread_index], ctx);
- }
-
- return ctx;
-}
-
-
-
-/*
- * fish pkts back from the recycle queue/freelist
- * un-flatten the context chains
- */
-static void
-replication_recycle_callback (vlib_main_t * vm, vlib_buffer_free_list_t * fl)
-{
- vlib_frame_t *f = 0;
- u32 n_left_from;
- u32 n_left_to_next = 0;
- u32 n_this_frame = 0;
- u32 *from;
- u32 *to_next = 0;
- u32 bi0, pi0;
- vlib_buffer_t *b0;
- int i;
- replication_main_t *rm = &replication_main;
- replication_context_t *ctx;
- u32 feature_node_index = 0;
- uword thread_index = vm->thread_index;
-
- /*
- * All buffers in the list are destined to the same recycle node.
- * Pull the recycle node index from the first buffer.
- * Note: this could be sped up if the node index were stuffed into
- * the freelist itself.
- */
- if (vec_len (fl->buffers) > 0)
- {
- bi0 = fl->buffers[0];
- b0 = vlib_get_buffer (vm, bi0);
- ctx = pool_elt_at_index (rm->contexts[thread_index], b0->recycle_count);
- feature_node_index = ctx->recycle_node_index;
- }
-
- /* buffers */
- for (i = 0; i < 2; i++)
- {
- if (i == 0)
- {
- from = fl->buffers;
- n_left_from = vec_len (from);
- }
-
- while (n_left_from > 0)
- {
- if (PREDICT_FALSE (n_left_to_next == 0))
- {
- if (f)
- {
- f->n_vectors = n_this_frame;
- vlib_put_frame_to_node (vm, feature_node_index, f);
- }
-
- f = vlib_get_frame_to_node (vm, feature_node_index);
- to_next = vlib_frame_vector_args (f);
- n_left_to_next = VLIB_FRAME_SIZE;
- n_this_frame = 0;
- }
-
- bi0 = from[0];
- if (PREDICT_TRUE (n_left_from > 1))
- {
- pi0 = from[1];
- vlib_prefetch_buffer_with_index (vm, pi0, LOAD);
- }
-
- b0 = vlib_get_buffer (vm, bi0);
-
- /* Mark that this buffer was just recycled */
- b0->flags |= VLIB_BUFFER_IS_RECYCLED;
-
-#if (CLIB_DEBUG > 0)
- if (buffer_main.callbacks_registered == 0)
- vlib_buffer_set_known_state (bi0, VLIB_BUFFER_KNOWN_ALLOCATED);
-#endif
-
- /* If buffer is traced, mark frame as traced */
- if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
- f->flags |= VLIB_FRAME_TRACE;
-
- to_next[0] = bi0;
-
- from++;
- to_next++;
- n_this_frame++;
- n_left_to_next--;
- n_left_from--;
- }
- }
-
- vec_reset_length (fl->buffers);
-
- if (f)
- {
- ASSERT (n_this_frame);
- f->n_vectors = n_this_frame;
- vlib_put_frame_to_node (vm, feature_node_index, f);
- }
-}
-
-clib_error_t *
-replication_init (vlib_main_t * vm)
-{
- replication_main_t *rm = &replication_main;
- vlib_buffer_free_list_t *fl;
- __attribute__ ((unused)) replication_context_t *ctx;
- vlib_thread_main_t *tm = vlib_get_thread_main ();
-
- rm->vlib_main = vm;
- rm->vnet_main = vnet_get_main ();
- rm->recycle_list_index =
- vlib_buffer_create_free_list (vm, 1024 /* fictional */ ,
- "replication-recycle");
-
- fl = pool_elt_at_index (vm->buffer_free_list_pool, rm->recycle_list_index);
-
- fl->buffers_added_to_freelist_function = replication_recycle_callback;
-
- /* Verify the replication context is the expected size */
- ASSERT (sizeof (replication_context_t) == 128); /* 2 cache lines */
-
- vec_validate (rm->contexts, tm->n_vlib_mains - 1);
- return 0;
-}
-
-VLIB_INIT_FUNCTION (replication_init);
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
+++ /dev/null
-/*
- * replication.h : packet replication
- *
- * Copyright (c) 2013 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef included_replication_h
-#define included_replication_h
-
-
-#include <vlib/vlib.h>
-#include <vnet/vnet.h>
-#include <vnet/replication.h>
-
-
-typedef struct
-{
- /* The entire vnet buffer header restored for each replica */
- u8 vnet_buffer[40]; /* 16B aligned to allow vector unit copy */
- u8 reserved[24]; /* space for future expansion of vnet buffer header */
-
- /* feature state used during this replication */
- u64 feature_replicas; /* feature's id for its set of replicas */
- u32 feature_counter; /* feature's current index into set of replicas */
- u32 recycle_node_index; /* feature's recycle node index */
-
- /*
- * data saved from the start of replication and restored
- * at the end of replication
- */
- vlib_buffer_free_list_index_t saved_free_list_index; /* from vlib buffer */
-
- /* data saved from the original packet and restored for each replica */
- u64 l2_header[3]; /* 24B (must be at least 22B for l2 packets) */
- u32 flags; /* vnet buffer flags */
- u16 ip_tos; /* v4 and v6 */
- u16 ip4_checksum; /* needed for v4 only */
-
- /* data saved from the vlib buffer header and restored for each replica */
- i16 current_data; /* offset of first byte of packet in packet data */
- u8 pad[2]; /* to 64B */
- u8 l2_packet; /* flag for l2 vs l3 packet data */
-
-} replication_context_t; /* 128B */
-
-
-typedef struct
-{
-
- u32 recycle_list_index;
-
- /* per-thread pools of replication contexts */
- replication_context_t **contexts;
-
- vlib_main_t *vlib_main;
- vnet_main_t *vnet_main;
-
-} replication_main_t;
-
-
-extern replication_main_t replication_main;
-
-
-/* Return 1 if this buffer just came from the replication recycle handler. */
-always_inline u32
-replication_is_recycled (vlib_buffer_t * b0)
-{
- return b0->flags & VLIB_BUFFER_IS_RECYCLED;
-}
-
-/*
- * Clear the recycle flag. If buffer came from the replication recycle
- * handler, this flag must be cleared before the packet is transmitted again.
- */
-always_inline void
-replication_clear_recycled (vlib_buffer_t * b0)
-{
- b0->flags &= ~VLIB_BUFFER_IS_RECYCLED;
-}
-
-/*
- * Return the active replication context if this buffer has
- * been recycled, otherwise return 0. (Note that this essentially
- * restricts access to the replication context to the replication
- * feature's prep and recycle nodes.)
- */
-always_inline replication_context_t *
-replication_get_ctx (vlib_buffer_t * b0)
-{
- replication_main_t *rm = &replication_main;
-
- return replication_is_recycled (b0) ?
- pool_elt_at_index (rm->contexts[vlib_get_thread_index ()],
- b0->recycle_count) : 0;
-}
-
-/* Prefetch the replication context for this buffer, if it exists */
-always_inline void
-replication_prefetch_ctx (vlib_buffer_t * b0)
-{
- replication_context_t *ctx = replication_get_ctx (b0);
-
- if (ctx)
- {
- CLIB_PREFETCH (ctx, (2 * CLIB_CACHE_LINE_BYTES), STORE);
- }
-}
-
-replication_context_t *replication_prep (vlib_main_t * vm,
- vlib_buffer_t * b0,
- u32 recycle_node_index,
- u32 l2_packet);
-
-replication_context_t *replication_recycle (vlib_main_t * vm,
- vlib_buffer_t * b0, u32 is_last);
-
-
-#endif
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
Code Review / vpp.git / commitdiff