src/vnet/dev/dev.h

   1 /* SPDX-License-Identifier: Apache-2.0
   2  * Copyright (c) 2023 Cisco Systems, Inc.
   3  */
   4
   5 #ifndef _VNET_DEV_H_
   6 #define _VNET_DEV_H_
   7
   8 #include <vppinfra/clib.h>
   9 #include <vppinfra/error_bootstrap.h>
  10 #include <vppinfra/format.h>
  11 #include <vnet/vnet.h>
  12 #include <vnet/dev/types.h>
  13
  14 #define VNET_DEV_DEVICE_ID_PREFIX_DELIMITER "/"
  15
  16 #define foreach_vnet_dev_port_type                                            \
  17   _ (0, UNKNOWN)                                                              \
  18   _ (1, ETHERNET)
  19
  20 typedef char vnet_dev_device_id_t[32];
  21
  22 typedef enum
  23 {
  24 #define _(b, n) VNET_DEV_PORT_TYPE_##n = (1U << (b)),
  25   foreach_vnet_dev_port_type
  26 #undef _
  27 } vnet_dev_port_type_t;
  28
  29 #define foreach_vnet_dev_port_caps                                            \
  30   _ (interrupt_mode)                                                          \
  31   _ (rss)
  32
  33 typedef union
  34 {
  35   struct
  36   {
  37 #define _(n) u8 n : 1;
  38     foreach_vnet_dev_port_caps
  39 #undef _
  40   };
  41   u8 as_number;
  42 } vnet_dev_port_caps_t;
  43
  44 typedef union
  45 {
  46   u8 eth_mac[6];
  47   u8 raw[8];
  48 } vnet_dev_hw_addr_t;
  49
  50 typedef struct vnet_dev_bus_registration vnet_dev_bus_registration_t;
  51 typedef struct vnet_dev_driver_registration vnet_dev_driver_registration_t;
  52
  53 typedef struct vnet_dev vnet_dev_t;
  54 typedef struct vnet_dev_port vnet_dev_port_t;
  55 typedef struct vnet_dev_rx_queue vnet_dev_rx_queue_t;
  56 typedef struct vnet_dev_tx_queue vnet_dev_tx_queue_t;
  57 typedef struct vnet_dev_bus_registration vnet_dev_bus_registration_t;
  58 typedef struct vnet_dev_driver_registration vnet_dev_driver_registration_t;
  59 typedef struct vnet_dev_counter vnet_dev_counter_t;
  60 typedef struct vnet_dev_counter_main vnet_dev_counter_main_t;
  61 typedef struct vnet_dev_port_cfg_change_req vnet_dev_port_cfg_change_req_t;
  62
  63 typedef vnet_dev_rv_t (vnet_dev_op_t) (vlib_main_t *, vnet_dev_t *);
  64 typedef vnet_dev_rv_t (vnet_dev_port_op_t) (vlib_main_t *, vnet_dev_port_t *);
  65 typedef vnet_dev_rv_t (vnet_dev_port_cfg_change_op_t) (
  66   vlib_main_t *, vnet_dev_port_t *, vnet_dev_port_cfg_change_req_t *);
  67 typedef vnet_dev_rv_t (vnet_dev_rx_queue_op_t) (vlib_main_t *,
  68                                                 vnet_dev_rx_queue_t *);
  69 typedef vnet_dev_rv_t (vnet_dev_tx_queue_op_t) (vlib_main_t *,
  70                                                 vnet_dev_tx_queue_t *);
  71 typedef void (vnet_dev_op_no_rv_t) (vlib_main_t *, vnet_dev_t *);
  72 typedef void (vnet_dev_port_op_no_rv_t) (vlib_main_t *, vnet_dev_port_t *);
  73 typedef void (vnet_dev_rx_queue_op_no_rv_t) (vlib_main_t *,
  74                                              vnet_dev_rx_queue_t *);
  75 typedef void (vnet_dev_tx_queue_op_no_rv_t) (vlib_main_t *,
  76                                              vnet_dev_tx_queue_t *);
  77
  78 typedef u16 vnet_dev_queue_id_t;
  79 typedef u16 vnet_dev_bus_index_t;
  80 typedef u16 vnet_dev_driver_index_t;
  81
  82 typedef struct
  83 {
  84   vnet_dev_rx_queue_op_t *alloc;
  85   vnet_dev_rx_queue_op_t *start;
  86   vnet_dev_rx_queue_op_no_rv_t *stop;
  87   vnet_dev_rx_queue_op_no_rv_t *free;
  88 } vnet_dev_rx_queue_ops_t;
  89
  90 typedef struct
  91 {
  92   vnet_dev_tx_queue_op_t *alloc;
  93   vnet_dev_tx_queue_op_t *start;
  94   vnet_dev_tx_queue_op_no_rv_t *stop;
  95   vnet_dev_tx_queue_op_no_rv_t *free;
  96 } vnet_dev_tx_queue_ops_t;
  97
  98 typedef struct
  99 {
 100   u16 data_size;
 101   u16 min_size;
 102   u16 max_size;
 103   u16 default_size;
 104   u8 multiplier;
 105   u8 size_is_power_of_two : 1;
 106 } vnet_dev_queue_config_t;
 107
 108 #define foreach_vnet_dev_port_cfg_type                                        \
 109   _ (PROMISC_MODE)                                                            \
 110   _ (MAX_FRAME_SIZE)                                                          \
 111   _ (CHANGE_PRIMARY_HW_ADDR)                                                  \
 112   _ (ADD_SECONDARY_HW_ADDR)                                                   \
 113   _ (REMOVE_SECONDARY_HW_ADDR)
 114
 115 typedef enum
 116 {
 117   VNET_DEV_PORT_CFG_UNKNOWN,
 118 #define _(n) VNET_DEV_PORT_CFG_##n,
 119   foreach_vnet_dev_port_cfg_type
 120 #undef _
 121 } __clib_packed vnet_dev_port_cfg_type_t;
 122
 123 typedef struct vnet_dev_port_cfg_change_req
 124 {
 125   vnet_dev_port_cfg_type_t type;
 126   u8 validated : 1;
 127
 128   union
 129   {
 130     u8 promisc : 1;
 131     vnet_dev_hw_addr_t addr;
 132     u16 max_frame_size;
 133   };
 134
 135 } vnet_dev_port_cfg_change_req_t;
 136
 137 typedef struct
 138 {
 139   vnet_dev_hw_addr_t hw_addr;
 140   u16 max_rx_queues;
 141   u16 max_tx_queues;
 142   u16 max_supported_frame_size;
 143   vnet_dev_port_type_t type;
 144   vnet_dev_port_caps_t caps;
 145 } vnet_dev_port_attr_t;
 146
 147 typedef enum
 148 {
 149   VNET_DEV_PERIODIC_OP_TYPE_DEV = 1,
 150   VNET_DEV_PERIODIC_OP_TYPE_PORT = 2,
 151 } __clib_packed vnet_dev_periodic_op_type_t;
 152
 153 typedef struct
 154 {
 155   f64 interval;
 156   f64 last_run;
 157   vnet_dev_periodic_op_type_t type;
 158   union
 159   {
 160     vnet_dev_t *dev;
 161     vnet_dev_port_t *port;
 162     void *arg;
 163   };
 164   union
 165   {
 166     vnet_dev_op_no_rv_t *dev_op;
 167     vnet_dev_port_op_no_rv_t *port_op;
 168     void *op;
 169   };
 170 } vnet_dev_periodic_op_t;
 171
 172 typedef struct
 173 {
 174   struct _vlib_node_fn_registration *registrations;
 175   format_function_t *format_trace;
 176   vlib_error_desc_t *error_counters;
 177   u16 n_error_counters;
 178 } vnet_dev_node_t;
 179
 180 typedef struct
 181 {
 182   vnet_dev_op_t *alloc;
 183   vnet_dev_op_t *init;
 184   vnet_dev_op_no_rv_t *deinit;
 185   vnet_dev_op_t *reset;
 186   vnet_dev_op_no_rv_t *free;
 187   u8 *(*probe) (vlib_main_t *, vnet_dev_bus_index_t, void *);
 188   format_function_t *format_info;
 189 } vnet_dev_ops_t;
 190
 191 typedef struct
 192 {
 193   vnet_dev_port_op_t *alloc;
 194   vnet_dev_port_op_t *init;
 195   vnet_dev_port_cfg_change_op_t *config_change;
 196   vnet_dev_port_cfg_change_op_t *config_change_validate;
 197   vnet_dev_port_op_t *start;
 198   vnet_dev_port_op_no_rv_t *stop;
 199   vnet_dev_port_op_no_rv_t *deinit;
 200   vnet_dev_port_op_no_rv_t *free;
 201   format_function_t *format_status;
 202 } vnet_dev_port_ops_t;
 203
 204 typedef union
 205 {
 206   struct
 207   {
 208     u8 update_next_index : 1;
 209     u8 update_feature_arc : 1;
 210     u8 suspend_off : 1;
 211     u8 suspend_on : 1;
 212   };
 213   u8 as_number;
 214 } vnet_dev_rx_queue_rt_req_t;
 215
 216 typedef struct vnet_dev_rx_queue
 217 {
 218   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 219   vnet_dev_port_t *port;
 220   u16 rx_thread_index;
 221   u16 index;
 222   vnet_dev_counter_main_t *counter_main;
 223   CLIB_CACHE_LINE_ALIGN_MARK (runtime0);
 224   u8 enabled : 1;
 225   u8 started : 1;
 226   u8 suspended : 1;
 227   vnet_dev_queue_id_t queue_id;
 228   u16 size;
 229   u16 next_index;
 230   vnet_dev_rx_queue_rt_req_t runtime_request;
 231   CLIB_CACHE_LINE_ALIGN_MARK (runtime1);
 232   vlib_buffer_template_t buffer_template;
 233   CLIB_ALIGN_MARK (private_data, 16);
 234   u8 data[];
 235 } vnet_dev_rx_queue_t;
 236
 237 STATIC_ASSERT_SIZEOF (vnet_dev_rx_queue_t, 3 * CLIB_CACHE_LINE_BYTES);
 238
 239 typedef struct vnet_dev_tx_queue
 240 {
 241   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 242   vnet_dev_port_t *port;
 243   clib_bitmap_t *assigned_threads;
 244   u16 index;
 245   vnet_dev_counter_main_t *counter_main;
 246   CLIB_CACHE_LINE_ALIGN_MARK (runtime0);
 247   vnet_dev_queue_id_t queue_id;
 248   u8 started : 1;
 249   u8 enabled : 1;
 250   u8 lock_needed : 1;
 251   u8 lock;
 252   u16 size;
 253   CLIB_ALIGN_MARK (private_data, 16);
 254   u8 data[];
 255 } vnet_dev_tx_queue_t;
 256
 257 STATIC_ASSERT_SIZEOF (vnet_dev_tx_queue_t, 2 * CLIB_CACHE_LINE_BYTES);
 258
 259 typedef struct vnet_dev_port
 260 {
 261   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 262   vnet_dev_t *dev;
 263   vnet_dev_port_id_t port_id;
 264   vnet_dev_driver_index_t driver_index;
 265   u8 initialized : 1;
 266   u8 started : 1;
 267   u8 link_up : 1;
 268   u8 promisc : 1;
 269   u8 interface_created : 1;
 270   u8 rx_node_assigned : 1;
 271   vnet_dev_counter_main_t *counter_main;
 272   vnet_dev_queue_config_t rx_queue_config;
 273   vnet_dev_queue_config_t tx_queue_config;
 274   vnet_dev_port_attr_t attr;
 275   u32 max_frame_size;
 276   vnet_dev_hw_addr_t primary_hw_addr;
 277   vnet_dev_hw_addr_t *secondary_hw_addr;
 278   u32 index;
 279   u32 speed;
 280   vnet_dev_rx_queue_t **rx_queues;
 281   vnet_dev_tx_queue_t **tx_queues;
 282   vnet_dev_port_ops_t port_ops;
 283   vnet_dev_rx_queue_ops_t rx_queue_ops;
 284   vnet_dev_tx_queue_ops_t tx_queue_ops;
 285   vnet_dev_node_t rx_node;
 286   vnet_dev_node_t tx_node;
 287
 288   struct
 289   {
 290     vnet_dev_if_name_t name;
 291     u32 dev_instance;
 292     u32 rx_node_index;
 293     u32 current_config_index;
 294     u16 rx_next_index;
 295     u16 redirect_to_node_next_index;
 296     u8 feature_arc_index;
 297     u8 feature_arc : 1;
 298     u8 redirect_to_node : 1;
 299     u32 tx_node_index;
 300     u32 hw_if_index;
 301     u32 sw_if_index;
 302     u16 num_rx_queues;
 303     u16 num_tx_queues;
 304     u16 txq_sz;
 305     u16 rxq_sz;
 306   } intf;
 307
 308   CLIB_CACHE_LINE_ALIGN_MARK (data0);
 309   u8 data[];
 310 } vnet_dev_port_t;
 311
 312 typedef struct vnet_dev
 313 {
 314   vnet_dev_device_id_t device_id;
 315   u16 initialized : 1;
 316   u16 not_first_init : 1;
 317   u16 va_dma : 1;
 318   u16 process_node_quit : 1;
 319   u16 process_node_periodic : 1;
 320   u16 poll_stats : 1;
 321   u16 bus_index;
 322   u8 numa_node;
 323   u16 max_rx_queues;
 324   u16 max_tx_queues;
 325   vnet_dev_driver_index_t driver_index;
 326   u32 index;
 327   u32 process_node_index;
 328   u8 bus_data[32] __clib_aligned (16);
 329   vnet_dev_ops_t ops;
 330   vnet_dev_port_t **ports;
 331   vnet_dev_periodic_op_t *periodic_ops;
 332   u8 *description;
 333   u8 __clib_aligned (16)
 334   data[];
 335 } vnet_dev_t;
 336
 337 typedef struct
 338 {
 339   u16 vendor_id, device_id;
 340   char *description;
 341 } vnet_dev_match_t;
 342
 343 #define VNET_DEV_MATCH(...)                                                   \
 344   (vnet_dev_match_t[])                                                        \
 345   {                                                                           \
 346     __VA_ARGS__, {}                                                           \
 347   }
 348
 349 typedef struct
 350 {
 351   vnet_dev_op_t *device_open;
 352   vnet_dev_op_no_rv_t *device_close;
 353   vnet_dev_rv_t (*dma_mem_alloc_fn) (vlib_main_t *, vnet_dev_t *, u32, u32,
 354                                      void **);
 355   void (*dma_mem_free_fn) (vlib_main_t *, vnet_dev_t *, void *);
 356   void *(*get_device_info) (vlib_main_t *, char *);
 357   void (*free_device_info) (vlib_main_t *, void *);
 358   format_function_t *format_device_info;
 359   format_function_t *format_device_addr;
 360 } vnet_dev_bus_ops_t;
 361
 362 struct vnet_dev_bus_registration
 363 {
 364   vnet_dev_bus_registration_t *next_registration;
 365   vnet_dev_driver_name_t name;
 366   u16 device_data_size;
 367   vnet_dev_bus_ops_t ops;
 368 };
 369
 370 struct vnet_dev_driver_registration
 371 {
 372   vnet_dev_driver_registration_t *next_registration;
 373   u8 bus_master_enable : 1;
 374   vnet_dev_driver_name_t name;
 375   vnet_dev_bus_name_t bus;
 376   u16 device_data_sz;
 377   u16 runtime_temp_space_sz;
 378   vnet_dev_match_t *match;
 379   int priority;
 380   vnet_dev_ops_t ops;
 381 };
 382
 383 typedef struct
 384 {
 385   u32 index;
 386   vnet_dev_bus_registration_t *registration;
 387   vnet_dev_bus_ops_t ops;
 388 } vnet_dev_bus_t;
 389
 390 typedef struct
 391 {
 392   u32 index;
 393   void *dev_data;
 394   vnet_dev_driver_registration_t *registration;
 395   u32 dev_class_index;
 396   vnet_dev_bus_index_t bus_index;
 397   vnet_dev_ops_t ops;
 398 } vnet_dev_driver_t;
 399
 400 typedef struct
 401 {
 402   vnet_dev_bus_t *buses;
 403   vnet_dev_driver_t *drivers;
 404   vnet_dev_t **devices;
 405   vnet_dev_port_t **ports_by_dev_instance;
 406   vnet_dev_bus_registration_t *bus_registrations;
 407   vnet_dev_driver_registration_t *driver_registrations;
 408   void *runtime_temp_spaces;
 409   u32 log2_runtime_temp_space_sz;
 410   u32 *free_process_node_indices;
 411   u32 *free_rx_node_indices;
 412   uword *device_index_by_id;
 413
 414   u8 *startup_config;
 415   u16 next_rx_queue_thread;
 416   u8 eth_port_rx_feature_arc_index;
 417 } vnet_dev_main_t;
 418
 419 extern vnet_dev_main_t vnet_dev_main;
 420
 421 typedef struct
 422 {
 423   struct
 424   {
 425     vnet_dev_port_attr_t attr;
 426     vnet_dev_port_ops_t ops;
 427     u16 data_size;
 428     void *initial_data;
 429   } port;
 430
 431   vnet_dev_node_t *rx_node;
 432   vnet_dev_node_t *tx_node;
 433
 434   struct
 435   {
 436     vnet_dev_queue_config_t config;
 437     vnet_dev_rx_queue_ops_t ops;
 438   } rx_queue;
 439
 440   struct
 441   {
 442     vnet_dev_queue_config_t config;
 443     vnet_dev_tx_queue_ops_t ops;
 444   } tx_queue;
 445 } vnet_dev_port_add_args_t;
 446
 447 typedef struct
 448 {
 449   union
 450   {
 451     struct
 452     {
 453       u8 link_speed : 1;
 454       u8 link_state : 1;
 455       u8 link_duplex : 1;
 456     };
 457     u8 any;
 458   } change;
 459   u8 link_state : 1;
 460   u8 full_duplex : 1;
 461   u32 link_speed;
 462 } vnet_dev_port_state_changes_t;
 463
 464 /* dev.c */
 465 vnet_dev_t *vnet_dev_alloc (vlib_main_t *, vnet_dev_device_id_t,
 466                             vnet_dev_driver_t *);
 467 void vnet_dev_free (vlib_main_t *, vnet_dev_t *);
 468 vnet_dev_rv_t vnet_dev_init (vlib_main_t *, vnet_dev_t *);
 469 void vnet_dev_deinit (vlib_main_t *, vnet_dev_t *);
 470 vnet_dev_rv_t vnet_dev_reset (vlib_main_t *, vnet_dev_t *);
 471 void vnet_dev_detach (vlib_main_t *, vnet_dev_t *);
 472 vnet_dev_rv_t vnet_dev_port_add (vlib_main_t *, vnet_dev_t *,
 473                                  vnet_dev_port_id_t,
 474                                  vnet_dev_port_add_args_t *);
 475 vnet_dev_rv_t vnet_dev_dma_mem_alloc (vlib_main_t *, vnet_dev_t *, u32, u32,
 476                                       void **);
 477 void vnet_dev_dma_mem_free (vlib_main_t *, vnet_dev_t *, void *);
 478 vnet_dev_bus_t *vnet_dev_find_device_bus (vlib_main_t *, vnet_dev_device_id_t);
 479 void *vnet_dev_get_device_info (vlib_main_t *, vnet_dev_device_id_t);
 480
 481 /* error.c */
 482 clib_error_t *vnet_dev_port_err (vlib_main_t *, vnet_dev_port_t *,
 483                                  vnet_dev_rv_t, char *, ...);
 484
 485 /* handlers.c */
 486 clib_error_t *vnet_dev_port_set_max_frame_size (vnet_main_t *,
 487                                                 vnet_hw_interface_t *, u32);
 488 u32 vnet_dev_port_eth_flag_change (vnet_main_t *, vnet_hw_interface_t *, u32);
 489 clib_error_t *vnet_dev_port_mac_change (vnet_hw_interface_t *, const u8 *,
 490                                         const u8 *);
 491 clib_error_t *vnet_dev_add_del_mac_address (vnet_hw_interface_t *, const u8 *,
 492                                             u8);
 493 int vnet_dev_flow_ops_fn (vnet_main_t *, vnet_flow_dev_op_t, u32, u32,
 494                           uword *);
 495 clib_error_t *vnet_dev_interface_set_rss_queues (vnet_main_t *,
 496                                                  vnet_hw_interface_t *,
 497                                                  clib_bitmap_t *);
 498 void vnet_dev_clear_hw_interface_counters (u32);
 499 clib_error_t *vnet_dev_rx_mode_change_fn (vnet_main_t *, u32, u32,
 500                                           vnet_hw_if_rx_mode);
 501 void vnet_dev_set_interface_next_node (vnet_main_t *, u32, u32);
 502
 503 /* port.c */
 504 vnet_dev_rv_t vnet_dev_port_start (vlib_main_t *, vnet_dev_port_t *);
 505 vnet_dev_rv_t vnet_dev_port_start_all_rx_queues (vlib_main_t *,
 506                                                  vnet_dev_port_t *);
 507 vnet_dev_rv_t vnet_dev_port_start_all_tx_queues (vlib_main_t *,
 508                                                  vnet_dev_port_t *);
 509 void vnet_dev_port_stop (vlib_main_t *, vnet_dev_port_t *);
 510 void vnet_dev_port_deinit (vlib_main_t *, vnet_dev_port_t *);
 511 void vnet_dev_port_free (vlib_main_t *, vnet_dev_port_t *);
 512 void vnet_dev_port_add_counters (vlib_main_t *, vnet_dev_port_t *,
 513                                  vnet_dev_counter_t *, u16);
 514 void vnet_dev_port_free_counters (vlib_main_t *, vnet_dev_port_t *);
 515 void vnet_dev_port_update_tx_node_runtime (vlib_main_t *, vnet_dev_port_t *);
 516 void vnet_dev_port_state_change (vlib_main_t *, vnet_dev_port_t *,
 517                                  vnet_dev_port_state_changes_t);
 518 void vnet_dev_port_clear_counters (vlib_main_t *, vnet_dev_port_t *);
 519 vnet_dev_rv_t
 520 vnet_dev_port_cfg_change_req_validate (vlib_main_t *, vnet_dev_port_t *,
 521                                        vnet_dev_port_cfg_change_req_t *);
 522 vnet_dev_rv_t vnet_dev_port_cfg_change (vlib_main_t *, vnet_dev_port_t *,
 523                                         vnet_dev_port_cfg_change_req_t *);
 524 vnet_dev_rv_t vnet_dev_port_if_create (vlib_main_t *, vnet_dev_port_t *);
 525 vnet_dev_rv_t vnet_dev_port_if_remove (vlib_main_t *, vnet_dev_port_t *);
 526
 527 /* queue.c */
 528 vnet_dev_rv_t vnet_dev_rx_queue_alloc (vlib_main_t *, vnet_dev_port_t *, u16);
 529 vnet_dev_rv_t vnet_dev_tx_queue_alloc (vlib_main_t *, vnet_dev_port_t *, u16);
 530 void vnet_dev_rx_queue_free (vlib_main_t *, vnet_dev_rx_queue_t *);
 531 void vnet_dev_tx_queue_free (vlib_main_t *, vnet_dev_tx_queue_t *);
 532 void vnet_dev_rx_queue_add_counters (vlib_main_t *, vnet_dev_rx_queue_t *,
 533                                      vnet_dev_counter_t *, u16);
 534 void vnet_dev_rx_queue_free_counters (vlib_main_t *, vnet_dev_rx_queue_t *);
 535 void vnet_dev_tx_queue_add_counters (vlib_main_t *, vnet_dev_tx_queue_t *,
 536                                      vnet_dev_counter_t *, u16);
 537 void vnet_dev_tx_queue_free_counters (vlib_main_t *, vnet_dev_tx_queue_t *);
 538 vnet_dev_rv_t vnet_dev_rx_queue_start (vlib_main_t *, vnet_dev_rx_queue_t *);
 539 vnet_dev_rv_t vnet_dev_tx_queue_start (vlib_main_t *, vnet_dev_tx_queue_t *);
 540 void vnet_dev_rx_queue_stop (vlib_main_t *, vnet_dev_rx_queue_t *);
 541 void vnet_dev_tx_queue_stop (vlib_main_t *, vnet_dev_tx_queue_t *);
 542
 543 /* process.c */
 544 vnet_dev_rv_t vnet_dev_process_create (vlib_main_t *, vnet_dev_t *);
 545 vnet_dev_rv_t vnet_dev_process_call_op (vlib_main_t *, vnet_dev_t *,
 546                                         vnet_dev_op_t *);
 547 vnet_dev_rv_t vnet_dev_process_call_op_no_rv (vlib_main_t *, vnet_dev_t *,
 548                                               vnet_dev_op_no_rv_t *);
 549 void vnet_dev_process_call_op_no_wait (vlib_main_t *, vnet_dev_t *,
 550                                        vnet_dev_op_no_rv_t *);
 551 vnet_dev_rv_t vnet_dev_process_call_port_op (vlib_main_t *, vnet_dev_port_t *,
 552                                              vnet_dev_port_op_t *);
 553 vnet_dev_rv_t vnet_dev_process_call_port_op_no_rv (vlib_main_t *vm,
 554                                                    vnet_dev_port_t *,
 555                                                    vnet_dev_port_op_no_rv_t *);
 556 void vnet_dev_process_call_port_op_no_wait (vlib_main_t *, vnet_dev_port_t *,
 557                                             vnet_dev_port_op_no_rv_t *);
 558 vnet_dev_rv_t
 559 vnet_dev_process_port_cfg_change_req (vlib_main_t *, vnet_dev_port_t *,
 560                                       vnet_dev_port_cfg_change_req_t *);
 561 void vnet_dev_process_quit (vlib_main_t *, vnet_dev_t *);
 562 void vnet_dev_poll_dev_add (vlib_main_t *, vnet_dev_t *, f64,
 563                             vnet_dev_op_no_rv_t *);
 564 void vnet_dev_poll_dev_remove (vlib_main_t *, vnet_dev_t *,
 565                                vnet_dev_op_no_rv_t *);
 566 void vnet_dev_poll_port_add (vlib_main_t *, vnet_dev_port_t *, f64,
 567                              vnet_dev_port_op_no_rv_t *);
 568 void vnet_dev_poll_port_remove (vlib_main_t *, vnet_dev_port_t *,
 569                                 vnet_dev_port_op_no_rv_t *);
 570
 571 /* runtime.c */
 572 typedef enum
 573 {
 574   VNET_DEV_RT_OP_TYPE_UNKNOWN,
 575   VNET_DEV_RT_OP_TYPE_RX_QUEUE,
 576 } __clib_packed vnet_dev_rt_op_type_t;
 577
 578 typedef enum
 579 {
 580   VNET_DEV_RT_OP_ACTION_UNKNOWN,
 581   VNET_DEV_RT_OP_ACTION_START,
 582   VNET_DEV_RT_OP_ACTION_STOP,
 583 } __clib_packed vnet_dev_rt_op_action_t;
 584
 585 typedef struct
 586 {
 587   u16 thread_index;
 588   u8 type : 4;
 589   u8 action : 4;
 590   u8 completed;
 591   vnet_dev_rx_queue_t *rx_queue;
 592 } vnet_dev_rt_op_t;
 593
 594 vnet_dev_rv_t vnet_dev_rt_exec_ops (vlib_main_t *, vnet_dev_t *,
 595                                     vnet_dev_rt_op_t *, u32);
 596
 597 /* format.c */
 598 typedef struct
 599 {
 600   u8 counters : 1;
 601   u8 show_zero_counters : 1;
 602   u8 debug : 1;
 603 } vnet_dev_format_args_t;
 604
 605 format_function_t format_vnet_dev_addr;
 606 format_function_t format_vnet_dev_hw_addr;
 607 format_function_t format_vnet_dev_info;
 608 format_function_t format_vnet_dev_interface_info;
 609 format_function_t format_vnet_dev_interface_name;
 610 format_function_t format_vnet_dev_port_info;
 611 format_function_t format_vnet_dev_rv;
 612 format_function_t format_vnet_dev_rx_queue_info;
 613 format_function_t format_vnet_dev_tx_queue_info;
 614 format_function_t format_vnet_dev_flags;
 615 format_function_t format_vnet_dev_port_flags;
 616 unformat_function_t unformat_vnet_dev_flags;
 617 unformat_function_t unformat_vnet_dev_port_flags;
 618
 619 typedef struct
 620 {
 621   u8 n_rx_queues;
 622   vnet_dev_rx_queue_t *rx_queues[4];
 623 } vnet_dev_rx_node_runtime_t;
 624
 625 STATIC_ASSERT (sizeof (vnet_dev_rx_node_runtime_t) <=
 626                  VLIB_NODE_RUNTIME_DATA_SIZE,
 627                "must fit into runtime data");
 628
 629 #define foreach_vnet_dev_port_rx_next                                         \
 630   _ (ETH_INPUT, "ethernet-input")                                             \
 631   _ (DROP, "error-drop")
 632
 633 typedef enum
 634 {
 635 #define _(n, s) VNET_DEV_ETH_RX_PORT_NEXT_##n,
 636   foreach_vnet_dev_port_rx_next
 637 #undef _
 638     VNET_DEV_ETH_RX_PORT_N_NEXTS
 639 } vnet_dev_eth_port_rx_next_t;
 640
 641 extern u16 vnet_dev_default_next_index_by_port_type[];
 642 extern vlib_node_registration_t port_rx_eth_node;
 643
 644 typedef vnet_interface_output_runtime_t vnet_dev_tx_node_runtime_t;
 645
 646 STATIC_ASSERT (sizeof (vnet_dev_tx_node_runtime_t) <=
 647                  VLIB_NODE_RUNTIME_DATA_SIZE,
 648                "must fit into runtime data");
 649
 650 #define VNET_DEV_REGISTER_BUS(x, ...)                                         \
 651   __VA_ARGS__ vnet_dev_bus_registration_t __vnet_dev_bus_registration_##x;    \
 652   static void __clib_constructor __vnet_dev_bus_registration_fn_##x (void)    \
 653   {                                                                           \
 654     vnet_dev_main_t *dm = &vnet_dev_main;                                     \
 655     __vnet_dev_bus_registration_##x.next_registration =                       \
 656       dm->bus_registrations;                                                  \
 657     dm->bus_registrations = &__vnet_dev_bus_registration_##x;                 \
 658   }                                                                           \
 659   __VA_ARGS__ vnet_dev_bus_registration_t __vnet_dev_bus_registration_##x
 660
 661 #define VNET_DEV_REGISTER_DRIVER(x, ...)                                      \
 662   __VA_ARGS__ vnet_dev_driver_registration_t                                  \
 663     __vnet_dev_driver_registration_##x;                                       \
 664   static void __clib_constructor __vnet_dev_driver_registration_fn_##x (void) \
 665   {                                                                           \
 666     vnet_dev_main_t *dm = &vnet_dev_main;                                     \
 667     __vnet_dev_driver_registration_##x.next_registration =                    \
 668       dm->driver_registrations;                                               \
 669     dm->driver_registrations = &__vnet_dev_driver_registration_##x;           \
 670   }                                                                           \
 671   __VA_ARGS__ vnet_dev_driver_registration_t __vnet_dev_driver_registration_##x
 672
 673 #define VNET_DEV_NODE_FN(node)                                                \
 674   uword CLIB_MARCH_SFX (node##_fn) (vlib_main_t *, vlib_node_runtime_t *,     \
 675                                     vlib_frame_t *);                          \
 676   static vlib_node_fn_registration_t CLIB_MARCH_SFX (                         \
 677     node##_fn_registration) = {                                               \
 678     .function = &CLIB_MARCH_SFX (node##_fn),                                  \
 679   };                                                                          \
 680                                                                               \
 681   static void __clib_constructor CLIB_MARCH_SFX (                             \
 682     node##_fn_multiarch_register) (void)                                      \
 683   {                                                                           \
 684     extern vnet_dev_node_t node;                                              \
 685     vlib_node_fn_registration_t *r;                                           \
 686     r = &CLIB_MARCH_SFX (node##_fn_registration);                             \
 687     r->march_variant = CLIB_MARCH_SFX (CLIB_MARCH_VARIANT_TYPE);              \
 688     r->next_registration = (node).registrations;                              \
 689     (node).registrations = r;                                                 \
 690   }                                                                           \
 691   uword CLIB_MARCH_SFX (node##_fn)
 692
 693 #define foreach_vnet_dev_port(p, d) pool_foreach_pointer (p, d->ports)
 694 #define foreach_vnet_dev_port_rx_queue(q, p)                                  \
 695   pool_foreach_pointer (q, p->rx_queues)
 696 #define foreach_vnet_dev_port_tx_queue(q, p)                                  \
 697   pool_foreach_pointer (q, p->tx_queues)
 698
 699 #include <vnet/dev/dev_funcs.h>
 700
 701 #endif /* _VNET_DEV_H_ */