#include <vlib/log.h>
#include <vnet/interface.h>
-#include <bpf/xsk.h>
+#include <xdp/xsk.h>
#define AF_XDP_NUM_RX_QUEUES_ALL ((u16)-1)
#define af_xdp_log(lvl, dev, f, ...) \
vlib_log(lvl, af_xdp_main.log_class, "%v: " f, (dev)->name, ##__VA_ARGS__)
-#define foreach_af_xdp_device_flags \
- _(0, INITIALIZED, "initialized") \
- _(1, ERROR, "error") \
- _(2, ADMIN_UP, "admin-up") \
- _(4, LINK_UP, "link-up") \
- _(8, ZEROCOPY, "zero-copy") \
+#define foreach_af_xdp_device_flags \
+ _ (0, INITIALIZED, "initialized") \
+ _ (1, ERROR, "error") \
+ _ (2, ADMIN_UP, "admin-up") \
+ _ (3, LINK_UP, "link-up") \
+ _ (4, ZEROCOPY, "zero-copy") \
+ _ (5, SYSCALL_LOCK, "syscall-lock")
enum
{
clib_error_free(err_); \
}
+typedef enum
+{
+ AF_XDP_RXQ_MODE_UNKNOWN,
+ AF_XDP_RXQ_MODE_POLLING,
+ AF_XDP_RXQ_MODE_INTERRUPT,
+} __clib_packed af_xdp_rxq_mode_t;
+
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
/* fields below are accessed in data-plane (hot) */
+ clib_spinlock_t syscall_lock;
struct xsk_ring_cons rx;
struct xsk_ring_prod fq;
int xsk_fd;
/* fields below are accessed in control-plane only (cold) */
uword file_index;
+ u32 queue_index;
+ af_xdp_rxq_mode_t mode;
} af_xdp_rxq_t;
typedef struct
/* fields below are accessed in data-plane (hot) */
clib_spinlock_t lock;
+ clib_spinlock_t syscall_lock;
struct xsk_ring_prod tx;
struct xsk_ring_cons cq;
int xsk_fd;
+
+ /* fields below are accessed in control-plane only (cold) */
+
+ u32 queue_index;
} af_xdp_txq_t;
typedef struct
u32 dev_instance;
u8 hwaddr[6];
+ u8 rxq_num;
+
+ char *netns;
+
struct xsk_umem **umem;
struct xsk_socket **xsk;
AF_XDP_MODE_ZERO_COPY = 2,
} af_xdp_mode_t;
+typedef enum
+{
+ AF_XDP_CREATE_FLAGS_NO_SYSCALL_LOCK = 1,
+} af_xdp_create_flag_t;
+
typedef struct
{
char *linux_ifname;
char *name;
char *prog;
+ char *netns;
af_xdp_mode_t mode;
+ af_xdp_create_flag_t flags;
u32 rxq_size;
u32 txq_size;
u32 rxq_num;
void af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args);
void af_xdp_delete_if (vlib_main_t * vm, af_xdp_device_t * ad);
+void af_xdp_device_input_refill (af_xdp_device_t *ad);
+
extern vlib_node_registration_t af_xdp_input_node;
extern vnet_device_class_t af_xdp_device_class;
u32 hw_if_index;
} af_xdp_input_trace_t;
-#define foreach_af_xdp_tx_func_error \
-_(NO_FREE_SLOTS, "no free tx slots") \
-_(SENDTO_REQUIRED, "sendto required") \
-_(SENDTO_FAILURES, "sendto failures")
+#define foreach_af_xdp_tx_func_error \
+ _ (NO_FREE_SLOTS, "no free tx slots") \
+ _ (SYSCALL_REQUIRED, "syscall required") \
+ _ (SYSCALL_FAILURES, "syscall failures")
typedef enum
{