src/vnet/dev/dev.h

   1 /* SPDX-License-Identifier: Apache-2.0
   2  * Copyright (c) 2023 Cisco Systems, Inc.
   3  */
   4
   5 #ifndef _VNET_DEV_H_
   6 #define _VNET_DEV_H_
   7
   8 #include <vppinfra/clib.h>
   9 #include <vppinfra/error_bootstrap.h>
  10 #include <vppinfra/format.h>
  11 #include <vnet/vnet.h>
  12 #include <vnet/dev/types.h>
  13 #include <vnet/dev/args.h>
  14
  15 #define VNET_DEV_DEVICE_ID_PREFIX_DELIMITER "/"
  16
  17 #define foreach_vnet_dev_port_type                                            \
  18   _ (0, UNKNOWN)                                                              \
  19   _ (1, ETHERNET)
  20
  21 typedef enum
  22 {
  23 #define _(b, n) VNET_DEV_PORT_TYPE_##n = (1U << (b)),
  24   foreach_vnet_dev_port_type
  25 #undef _
  26 } vnet_dev_port_type_t;
  27
  28 #define foreach_vnet_dev_port_caps                                            \
  29   _ (interrupt_mode)                                                          \
  30   _ (rss)                                                                     \
  31   _ (change_max_rx_frame_size)                                                \
  32   _ (mac_filter)
  33
  34 #define foreach_vnet_dev_port_rx_offloads _ (ip4_cksum)
  35
  36 #define foreach_vnet_dev_port_tx_offloads                                     \
  37   _ (ip4_cksum)                                                               \
  38   _ (tcp_gso)                                                                 \
  39   _ (udp_gso)
  40
  41 typedef union
  42 {
  43   struct
  44   {
  45 #define _(n) u8 n : 1;
  46     foreach_vnet_dev_port_caps
  47 #undef _
  48   };
  49   u8 as_number;
  50 } vnet_dev_port_caps_t;
  51
  52 typedef union
  53 {
  54   struct
  55   {
  56 #define _(n) u8 n : 1;
  57     foreach_vnet_dev_port_rx_offloads
  58 #undef _
  59   };
  60   u8 as_number;
  61 } vnet_dev_port_rx_offloads_t;
  62
  63 typedef union
  64 {
  65   struct
  66   {
  67 #define _(n) u8 n : 1;
  68     foreach_vnet_dev_port_tx_offloads
  69 #undef _
  70   };
  71   u8 as_number;
  72 } vnet_dev_port_tx_offloads_t;
  73
  74 typedef union
  75 {
  76   u8 eth_mac[6];
  77   u8 raw[8];
  78 } vnet_dev_hw_addr_t;
  79
  80 typedef struct vnet_dev_bus_registration vnet_dev_bus_registration_t;
  81 typedef struct vnet_dev_driver_registration vnet_dev_driver_registration_t;
  82
  83 typedef struct vnet_dev vnet_dev_t;
  84 typedef struct vnet_dev_port vnet_dev_port_t;
  85 typedef struct vnet_dev_rx_queue vnet_dev_rx_queue_t;
  86 typedef struct vnet_dev_tx_queue vnet_dev_tx_queue_t;
  87 typedef struct vnet_dev_bus_registration vnet_dev_bus_registration_t;
  88 typedef struct vnet_dev_driver_registration vnet_dev_driver_registration_t;
  89 typedef struct vnet_dev_counter vnet_dev_counter_t;
  90 typedef struct vnet_dev_counter_main vnet_dev_counter_main_t;
  91 typedef struct vnet_dev_port_cfg_change_req vnet_dev_port_cfg_change_req_t;
  92
  93 typedef vnet_dev_rv_t (vnet_dev_op_t) (vlib_main_t *, vnet_dev_t *);
  94 typedef vnet_dev_rv_t (vnet_dev_port_op_t) (vlib_main_t *, vnet_dev_port_t *);
  95 typedef vnet_dev_rv_t (vnet_dev_port_cfg_change_op_t) (
  96   vlib_main_t *, vnet_dev_port_t *, vnet_dev_port_cfg_change_req_t *);
  97 typedef vnet_dev_rv_t (vnet_dev_rx_queue_op_t) (vlib_main_t *,
  98                                                 vnet_dev_rx_queue_t *);
  99 typedef vnet_dev_rv_t (vnet_dev_tx_queue_op_t) (vlib_main_t *,
 100                                                 vnet_dev_tx_queue_t *);
 101 typedef void (vnet_dev_op_no_rv_t) (vlib_main_t *, vnet_dev_t *);
 102 typedef void (vnet_dev_port_op_no_rv_t) (vlib_main_t *, vnet_dev_port_t *);
 103 typedef void (vnet_dev_rx_queue_op_no_rv_t) (vlib_main_t *,
 104                                              vnet_dev_rx_queue_t *);
 105 typedef void (vnet_dev_tx_queue_op_no_rv_t) (vlib_main_t *,
 106                                              vnet_dev_tx_queue_t *);
 107
 108 typedef u16 vnet_dev_queue_id_t;
 109 typedef u16 vnet_dev_bus_index_t;
 110 typedef u16 vnet_dev_driver_index_t;
 111
 112 typedef struct
 113 {
 114   vnet_dev_rx_queue_op_t *alloc;
 115   vnet_dev_rx_queue_op_t *start;
 116   vnet_dev_rx_queue_op_no_rv_t *stop;
 117   vnet_dev_rx_queue_op_no_rv_t *free;
 118   vnet_dev_rx_queue_op_no_rv_t *clear_counters;
 119   format_function_t *format_info;
 120 } vnet_dev_rx_queue_ops_t;
 121
 122 typedef struct
 123 {
 124   vnet_dev_tx_queue_op_t *alloc;
 125   vnet_dev_tx_queue_op_t *start;
 126   vnet_dev_tx_queue_op_no_rv_t *stop;
 127   vnet_dev_tx_queue_op_no_rv_t *free;
 128   vnet_dev_tx_queue_op_no_rv_t *clear_counters;
 129   format_function_t *format_info;
 130 } vnet_dev_tx_queue_ops_t;
 131
 132 typedef struct
 133 {
 134   u16 data_size;
 135   u16 min_size;
 136   u16 max_size;
 137   u16 default_size;
 138   u8 multiplier;
 139   u8 size_is_power_of_two : 1;
 140 } vnet_dev_queue_config_t;
 141
 142 #define foreach_vnet_dev_port_cfg_type                                        \
 143   _ (PROMISC_MODE)                                                            \
 144   _ (MAX_RX_FRAME_SIZE)                                                       \
 145   _ (CHANGE_PRIMARY_HW_ADDR)                                                  \
 146   _ (ADD_SECONDARY_HW_ADDR)                                                   \
 147   _ (REMOVE_SECONDARY_HW_ADDR)                                                \
 148   _ (RXQ_INTR_MODE_ENABLE)                                                    \
 149   _ (RXQ_INTR_MODE_DISABLE)                                                   \
 150   _ (ADD_RX_FLOW)                                                             \
 151   _ (DEL_RX_FLOW)                                                             \
 152   _ (GET_RX_FLOW_COUNTER)                                                     \
 153   _ (RESET_RX_FLOW_COUNTER)
 154
 155 typedef enum
 156 {
 157   VNET_DEV_PORT_CFG_UNKNOWN,
 158 #define _(n) VNET_DEV_PORT_CFG_##n,
 159   foreach_vnet_dev_port_cfg_type
 160 #undef _
 161 } __clib_packed vnet_dev_port_cfg_type_t;
 162
 163 typedef struct vnet_dev_port_cfg_change_req
 164 {
 165   vnet_dev_port_cfg_type_t type;
 166   u8 validated : 1;
 167   u8 all_queues : 1;
 168
 169   union
 170   {
 171     u8 promisc : 1;
 172     vnet_dev_hw_addr_t addr;
 173     u16 max_rx_frame_size;
 174     vnet_dev_queue_id_t queue_id;
 175     struct
 176     {
 177       u32 flow_index;
 178       uword *private_data;
 179     };
 180   };
 181
 182 } vnet_dev_port_cfg_change_req_t;
 183
 184 typedef struct
 185 {
 186   vnet_dev_hw_addr_t hw_addr;
 187   u16 max_rx_queues;
 188   u16 max_tx_queues;
 189   u16 max_supported_rx_frame_size;
 190   vnet_dev_port_type_t type;
 191   vnet_dev_port_caps_t caps;
 192   vnet_dev_port_rx_offloads_t rx_offloads;
 193   vnet_dev_port_tx_offloads_t tx_offloads;
 194 } vnet_dev_port_attr_t;
 195
 196 typedef enum
 197 {
 198   VNET_DEV_PERIODIC_OP_TYPE_DEV = 1,
 199   VNET_DEV_PERIODIC_OP_TYPE_PORT = 2,
 200 } __clib_packed vnet_dev_periodic_op_type_t;
 201
 202 typedef struct
 203 {
 204   f64 interval;
 205   f64 last_run;
 206   vnet_dev_periodic_op_type_t type;
 207   union
 208   {
 209     vnet_dev_t *dev;
 210     vnet_dev_port_t *port;
 211     void *arg;
 212   };
 213   union
 214   {
 215     vnet_dev_op_no_rv_t *dev_op;
 216     vnet_dev_port_op_no_rv_t *port_op;
 217     void *op;
 218   };
 219 } vnet_dev_periodic_op_t;
 220
 221 typedef struct
 222 {
 223   struct _vlib_node_fn_registration *registrations;
 224   format_function_t *format_trace;
 225   vlib_error_desc_t *error_counters;
 226   u16 n_error_counters;
 227 } vnet_dev_node_t;
 228
 229 typedef struct
 230 {
 231   vnet_dev_op_t *alloc;
 232   vnet_dev_op_t *init;
 233   vnet_dev_op_no_rv_t *deinit;
 234   vnet_dev_op_t *reset;
 235   vnet_dev_op_no_rv_t *free;
 236   u8 *(*probe) (vlib_main_t *, vnet_dev_bus_index_t, void *);
 237   format_function_t *format_info;
 238 } vnet_dev_ops_t;
 239
 240 typedef struct
 241 {
 242   vnet_dev_port_op_t *alloc;
 243   vnet_dev_port_op_t *init;
 244   vnet_dev_port_cfg_change_op_t *config_change;
 245   vnet_dev_port_cfg_change_op_t *config_change_validate;
 246   vnet_dev_port_op_t *start;
 247   vnet_dev_port_op_no_rv_t *stop;
 248   vnet_dev_port_op_no_rv_t *deinit;
 249   vnet_dev_port_op_no_rv_t *free;
 250   vnet_dev_port_op_no_rv_t *clear_counters;
 251   format_function_t *format_status;
 252   format_function_t *format_flow;
 253 } vnet_dev_port_ops_t;
 254
 255 typedef union
 256 {
 257   struct
 258   {
 259     u8 update_next_index : 1;
 260     u8 update_feature_arc : 1;
 261     u8 suspend_off : 1;
 262     u8 suspend_on : 1;
 263   };
 264   u8 as_number;
 265 } vnet_dev_rx_queue_rt_req_t;
 266
 267 typedef struct vnet_dev_rx_queue
 268 {
 269   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 270   vnet_dev_port_t *port;
 271   u16 rx_thread_index;
 272   u16 index;
 273   vnet_dev_counter_main_t *counter_main;
 274   CLIB_CACHE_LINE_ALIGN_MARK (runtime0);
 275   vnet_dev_rx_queue_t *next_on_thread;
 276   u8 interrupt_mode : 1;
 277   u8 enabled : 1;
 278   u8 started : 1;
 279   u8 suspended : 1;
 280   vnet_dev_queue_id_t queue_id;
 281   u16 size;
 282   u16 next_index;
 283   vnet_dev_rx_queue_rt_req_t runtime_request;
 284   CLIB_CACHE_LINE_ALIGN_MARK (runtime1);
 285   vlib_buffer_template_t buffer_template;
 286   CLIB_CACHE_LINE_ALIGN_MARK (driver_data);
 287   u8 data[];
 288 } vnet_dev_rx_queue_t;
 289
 290 STATIC_ASSERT_SIZEOF (vnet_dev_rx_queue_t, 3 * CLIB_CACHE_LINE_BYTES);
 291
 292 typedef struct vnet_dev_tx_queue
 293 {
 294   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 295   vnet_dev_port_t *port;
 296   clib_bitmap_t *assigned_threads;
 297   u16 index;
 298   vnet_dev_counter_main_t *counter_main;
 299   CLIB_CACHE_LINE_ALIGN_MARK (runtime0);
 300   vnet_dev_queue_id_t queue_id;
 301   u8 started : 1;
 302   u8 enabled : 1;
 303   u8 lock_needed : 1;
 304   u8 lock;
 305   u16 size;
 306   CLIB_ALIGN_MARK (private_data, 16);
 307   u8 data[];
 308 } vnet_dev_tx_queue_t;
 309
 310 STATIC_ASSERT_SIZEOF (vnet_dev_tx_queue_t, 2 * CLIB_CACHE_LINE_BYTES);
 311
 312 typedef struct vnet_dev_port
 313 {
 314   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 315   vnet_dev_t *dev;
 316   vnet_dev_port_id_t port_id;
 317   vnet_dev_driver_index_t driver_index;
 318   u8 initialized : 1;
 319   u8 started : 1;
 320   u8 link_up : 1;
 321   u8 promisc : 1;
 322   u8 interface_created : 1;
 323   u8 rx_node_assigned : 1;
 324   vnet_dev_counter_main_t *counter_main;
 325   vnet_dev_queue_config_t rx_queue_config;
 326   vnet_dev_queue_config_t tx_queue_config;
 327   vnet_dev_port_attr_t attr;
 328   u32 max_rx_frame_size;
 329   vnet_dev_hw_addr_t primary_hw_addr;
 330   vnet_dev_hw_addr_t *secondary_hw_addr;
 331   u32 index;
 332   u32 speed;
 333   vnet_dev_rx_queue_t **rx_queues;
 334   vnet_dev_tx_queue_t **tx_queues;
 335   vnet_dev_port_ops_t port_ops;
 336   vnet_dev_arg_t *args;
 337   vnet_dev_rx_queue_ops_t rx_queue_ops;
 338   vnet_dev_tx_queue_ops_t tx_queue_ops;
 339   vnet_dev_node_t rx_node;
 340   vnet_dev_node_t tx_node;
 341
 342   struct
 343   {
 344     vnet_dev_if_name_t name;
 345     u32 dev_instance;
 346     u32 rx_node_index;
 347     u32 current_config_index;
 348     u16 rx_next_index;
 349     u16 redirect_to_node_next_index;
 350     u8 feature_arc_index;
 351     u8 feature_arc : 1;
 352     u8 redirect_to_node : 1;
 353     u8 default_is_intr_mode : 1;
 354     u32 tx_node_index;
 355     u32 hw_if_index;
 356     u32 sw_if_index;
 357     u16 num_rx_queues;
 358     u16 num_tx_queues;
 359     u16 txq_sz;
 360     u16 rxq_sz;
 361   } intf;
 362
 363   CLIB_CACHE_LINE_ALIGN_MARK (data0);
 364   u8 data[];
 365 } vnet_dev_port_t;
 366
 367 typedef struct vnet_dev
 368 {
 369   vnet_dev_device_id_t device_id;
 370   u16 initialized : 1;
 371   u16 not_first_init : 1;
 372   u16 va_dma : 1;
 373   u16 process_node_quit : 1;
 374   u16 process_node_periodic : 1;
 375   u16 poll_stats : 1;
 376   u16 bus_index;
 377   u8 numa_node;
 378   u16 max_rx_queues;
 379   u16 max_tx_queues;
 380   vnet_dev_driver_index_t driver_index;
 381   u32 index;
 382   u32 process_node_index;
 383   u8 bus_data[32] __clib_aligned (16);
 384   vnet_dev_ops_t ops;
 385   vnet_dev_port_t **ports;
 386   vnet_dev_periodic_op_t *periodic_ops;
 387   u8 *description;
 388   vnet_dev_arg_t *args;
 389   u8 __clib_aligned (16)
 390   data[];
 391 } vnet_dev_t;
 392
 393 typedef struct
 394 {
 395   u16 vendor_id, device_id;
 396   char *description;
 397 } vnet_dev_match_t;
 398
 399 #define VNET_DEV_MATCH(...)                                                   \
 400   (vnet_dev_match_t[])                                                        \
 401   {                                                                           \
 402     __VA_ARGS__, {}                                                           \
 403   }
 404
 405 typedef struct
 406 {
 407   vnet_dev_op_t *device_open;
 408   vnet_dev_op_no_rv_t *device_close;
 409   vnet_dev_rv_t (*dma_mem_alloc_fn) (vlib_main_t *, vnet_dev_t *, u32, u32,
 410                                      void **);
 411   void (*dma_mem_free_fn) (vlib_main_t *, vnet_dev_t *, void *);
 412   void *(*get_device_info) (vlib_main_t *, char *);
 413   void (*free_device_info) (vlib_main_t *, void *);
 414   format_function_t *format_device_info;
 415   format_function_t *format_device_addr;
 416 } vnet_dev_bus_ops_t;
 417
 418 struct vnet_dev_bus_registration
 419 {
 420   vnet_dev_bus_registration_t *next_registration;
 421   vnet_dev_driver_name_t name;
 422   u16 device_data_size;
 423   vnet_dev_bus_ops_t ops;
 424 };
 425
 426 struct vnet_dev_driver_registration
 427 {
 428   vnet_dev_driver_registration_t *next_registration;
 429   u8 bus_master_enable : 1;
 430   vnet_dev_driver_name_t name;
 431   vnet_dev_bus_name_t bus;
 432   u16 device_data_sz;
 433   u16 runtime_temp_space_sz;
 434   vnet_dev_match_t *match;
 435   int priority;
 436   vnet_dev_ops_t ops;
 437   vnet_dev_arg_t *args;
 438 };
 439
 440 typedef struct
 441 {
 442   u32 index;
 443   vnet_dev_bus_registration_t *registration;
 444   vnet_dev_bus_ops_t ops;
 445 } vnet_dev_bus_t;
 446
 447 typedef struct
 448 {
 449   u32 index;
 450   void *dev_data;
 451   vnet_dev_driver_registration_t *registration;
 452   u32 dev_class_index;
 453   vnet_dev_bus_index_t bus_index;
 454   vnet_dev_ops_t ops;
 455 } vnet_dev_driver_t;
 456
 457 typedef struct
 458 {
 459   vnet_dev_bus_t *buses;
 460   vnet_dev_driver_t *drivers;
 461   vnet_dev_t **devices;
 462   vnet_dev_port_t **ports_by_dev_instance;
 463   vnet_dev_bus_registration_t *bus_registrations;
 464   vnet_dev_driver_registration_t *driver_registrations;
 465   void *runtime_temp_spaces;
 466   u32 log2_runtime_temp_space_sz;
 467   u32 *free_process_node_indices;
 468   u32 *free_rx_node_indices;
 469   uword *device_index_by_id;
 470
 471   u8 *startup_config;
 472   u16 next_rx_queue_thread;
 473   u8 eth_port_rx_feature_arc_index;
 474 } vnet_dev_main_t;
 475
 476 extern vnet_dev_main_t vnet_dev_main;
 477
 478 typedef struct
 479 {
 480   struct
 481   {
 482     vnet_dev_port_attr_t attr;
 483     vnet_dev_port_ops_t ops;
 484     vnet_dev_arg_t *args;
 485     u16 data_size;
 486     void *initial_data;
 487   } port;
 488
 489   vnet_dev_node_t *rx_node;
 490   vnet_dev_node_t *tx_node;
 491
 492   struct
 493   {
 494     vnet_dev_queue_config_t config;
 495     vnet_dev_rx_queue_ops_t ops;
 496   } rx_queue;
 497
 498   struct
 499   {
 500     vnet_dev_queue_config_t config;
 501     vnet_dev_tx_queue_ops_t ops;
 502   } tx_queue;
 503 } vnet_dev_port_add_args_t;
 504
 505 typedef struct
 506 {
 507   union
 508   {
 509     struct
 510     {
 511       u8 link_speed : 1;
 512       u8 link_state : 1;
 513       u8 link_duplex : 1;
 514     };
 515     u8 any;
 516   } change;
 517   u8 link_state : 1;
 518   u8 full_duplex : 1;
 519   u32 link_speed;
 520 } vnet_dev_port_state_changes_t;
 521
 522 /* args.c */
 523 vnet_dev_rv_t vnet_dev_arg_parse (vlib_main_t *, vnet_dev_t *,
 524                                   vnet_dev_arg_t *, u8 *);
 525 void vnet_dev_arg_free (vnet_dev_arg_t **);
 526 void vnet_dev_arg_clear_value (vnet_dev_arg_t *);
 527 format_function_t format_vnet_dev_arg_type;
 528 format_function_t format_vnet_dev_arg_value;
 529 format_function_t format_vnet_dev_args;
 530
 531 /* dev.c */
 532 vnet_dev_t *vnet_dev_alloc (vlib_main_t *, vnet_dev_device_id_t,
 533                             vnet_dev_driver_t *);
 534 void vnet_dev_free (vlib_main_t *, vnet_dev_t *);
 535 vnet_dev_rv_t vnet_dev_init (vlib_main_t *, vnet_dev_t *);
 536 void vnet_dev_deinit (vlib_main_t *, vnet_dev_t *);
 537 vnet_dev_rv_t vnet_dev_reset (vlib_main_t *, vnet_dev_t *);
 538 void vnet_dev_detach (vlib_main_t *, vnet_dev_t *);
 539 vnet_dev_rv_t vnet_dev_port_add (vlib_main_t *, vnet_dev_t *,
 540                                  vnet_dev_port_id_t,
 541                                  vnet_dev_port_add_args_t *);
 542 vnet_dev_rv_t vnet_dev_dma_mem_alloc (vlib_main_t *, vnet_dev_t *, u32, u32,
 543                                       void **);
 544 void vnet_dev_dma_mem_free (vlib_main_t *, vnet_dev_t *, void *);
 545 vnet_dev_bus_t *vnet_dev_find_device_bus (vlib_main_t *, vnet_dev_device_id_t);
 546 void *vnet_dev_get_device_info (vlib_main_t *, vnet_dev_device_id_t);
 547
 548 /* error.c */
 549 clib_error_t *vnet_dev_port_err (vlib_main_t *, vnet_dev_port_t *,
 550                                  vnet_dev_rv_t, char *, ...);
 551 int vnet_dev_flow_err (vlib_main_t *, vnet_dev_rv_t);
 552
 553 /* handlers.c */
 554 clib_error_t *vnet_dev_port_set_max_frame_size (vnet_main_t *,
 555                                                 vnet_hw_interface_t *, u32);
 556 u32 vnet_dev_port_eth_flag_change (vnet_main_t *, vnet_hw_interface_t *, u32);
 557 clib_error_t *vnet_dev_port_mac_change (vnet_hw_interface_t *, const u8 *,
 558                                         const u8 *);
 559 clib_error_t *vnet_dev_add_del_mac_address (vnet_hw_interface_t *, const u8 *,
 560                                             u8);
 561 int vnet_dev_flow_ops_fn (vnet_main_t *, vnet_flow_dev_op_t, u32, u32,
 562                           uword *);
 563 clib_error_t *vnet_dev_interface_set_rss_queues (vnet_main_t *,
 564                                                  vnet_hw_interface_t *,
 565                                                  clib_bitmap_t *);
 566 void vnet_dev_clear_hw_interface_counters (u32);
 567 void vnet_dev_set_interface_next_node (vnet_main_t *, u32, u32);
 568
 569 /* port.c */
 570 vnet_dev_rv_t vnet_dev_port_start (vlib_main_t *, vnet_dev_port_t *);
 571 vnet_dev_rv_t vnet_dev_port_start_all_rx_queues (vlib_main_t *,
 572                                                  vnet_dev_port_t *);
 573 vnet_dev_rv_t vnet_dev_port_start_all_tx_queues (vlib_main_t *,
 574                                                  vnet_dev_port_t *);
 575 void vnet_dev_port_stop (vlib_main_t *, vnet_dev_port_t *);
 576 void vnet_dev_port_deinit (vlib_main_t *, vnet_dev_port_t *);
 577 void vnet_dev_port_free (vlib_main_t *, vnet_dev_port_t *);
 578 void vnet_dev_port_add_counters (vlib_main_t *, vnet_dev_port_t *,
 579                                  vnet_dev_counter_t *, u16);
 580 void vnet_dev_port_free_counters (vlib_main_t *, vnet_dev_port_t *);
 581 void vnet_dev_port_update_tx_node_runtime (vlib_main_t *, vnet_dev_port_t *);
 582 void vnet_dev_port_state_change (vlib_main_t *, vnet_dev_port_t *,
 583                                  vnet_dev_port_state_changes_t);
 584 void vnet_dev_port_clear_counters (vlib_main_t *, vnet_dev_port_t *);
 585 vnet_dev_rv_t
 586 vnet_dev_port_cfg_change_req_validate (vlib_main_t *, vnet_dev_port_t *,
 587                                        vnet_dev_port_cfg_change_req_t *);
 588 vnet_dev_rv_t vnet_dev_port_cfg_change (vlib_main_t *, vnet_dev_port_t *,
 589                                         vnet_dev_port_cfg_change_req_t *);
 590 vnet_dev_rv_t vnet_dev_port_if_create (vlib_main_t *, vnet_dev_port_t *);
 591 vnet_dev_rv_t vnet_dev_port_if_remove (vlib_main_t *, vnet_dev_port_t *);
 592
 593 /* queue.c */
 594 vnet_dev_rv_t vnet_dev_rx_queue_alloc (vlib_main_t *, vnet_dev_port_t *, u16);
 595 vnet_dev_rv_t vnet_dev_tx_queue_alloc (vlib_main_t *, vnet_dev_port_t *, u16);
 596 void vnet_dev_rx_queue_free (vlib_main_t *, vnet_dev_rx_queue_t *);
 597 void vnet_dev_tx_queue_free (vlib_main_t *, vnet_dev_tx_queue_t *);
 598 void vnet_dev_rx_queue_add_counters (vlib_main_t *, vnet_dev_rx_queue_t *,
 599                                      vnet_dev_counter_t *, u16);
 600 void vnet_dev_rx_queue_free_counters (vlib_main_t *, vnet_dev_rx_queue_t *);
 601 void vnet_dev_tx_queue_add_counters (vlib_main_t *, vnet_dev_tx_queue_t *,
 602                                      vnet_dev_counter_t *, u16);
 603 void vnet_dev_tx_queue_free_counters (vlib_main_t *, vnet_dev_tx_queue_t *);
 604 vnet_dev_rv_t vnet_dev_rx_queue_start (vlib_main_t *, vnet_dev_rx_queue_t *);
 605 vnet_dev_rv_t vnet_dev_tx_queue_start (vlib_main_t *, vnet_dev_tx_queue_t *);
 606 void vnet_dev_rx_queue_stop (vlib_main_t *, vnet_dev_rx_queue_t *);
 607 void vnet_dev_tx_queue_stop (vlib_main_t *, vnet_dev_tx_queue_t *);
 608
 609 /* process.c */
 610 vnet_dev_rv_t vnet_dev_process_create (vlib_main_t *, vnet_dev_t *);
 611 vnet_dev_rv_t vnet_dev_process_call_op (vlib_main_t *, vnet_dev_t *,
 612                                         vnet_dev_op_t *);
 613 vnet_dev_rv_t vnet_dev_process_call_op_no_rv (vlib_main_t *, vnet_dev_t *,
 614                                               vnet_dev_op_no_rv_t *);
 615 void vnet_dev_process_call_op_no_wait (vlib_main_t *, vnet_dev_t *,
 616                                        vnet_dev_op_no_rv_t *);
 617 vnet_dev_rv_t vnet_dev_process_call_port_op (vlib_main_t *, vnet_dev_port_t *,
 618                                              vnet_dev_port_op_t *);
 619 vnet_dev_rv_t vnet_dev_process_call_port_op_no_rv (vlib_main_t *vm,
 620                                                    vnet_dev_port_t *,
 621                                                    vnet_dev_port_op_no_rv_t *);
 622 void vnet_dev_process_call_port_op_no_wait (vlib_main_t *, vnet_dev_port_t *,
 623                                             vnet_dev_port_op_no_rv_t *);
 624 vnet_dev_rv_t
 625 vnet_dev_process_port_cfg_change_req (vlib_main_t *, vnet_dev_port_t *,
 626                                       vnet_dev_port_cfg_change_req_t *);
 627 void vnet_dev_process_quit (vlib_main_t *, vnet_dev_t *);
 628 void vnet_dev_poll_dev_add (vlib_main_t *, vnet_dev_t *, f64,
 629                             vnet_dev_op_no_rv_t *);
 630 void vnet_dev_poll_dev_remove (vlib_main_t *, vnet_dev_t *,
 631                                vnet_dev_op_no_rv_t *);
 632 void vnet_dev_poll_port_add (vlib_main_t *, vnet_dev_port_t *, f64,
 633                              vnet_dev_port_op_no_rv_t *);
 634 void vnet_dev_poll_port_remove (vlib_main_t *, vnet_dev_port_t *,
 635                                 vnet_dev_port_op_no_rv_t *);
 636
 637 typedef struct
 638 {
 639   u16 thread_index;
 640   u8 completed;
 641   u8 in_order;
 642   vnet_dev_port_t *port;
 643 } vnet_dev_rt_op_t;
 644
 645 vnet_dev_rv_t vnet_dev_rt_exec_ops (vlib_main_t *, vnet_dev_t *,
 646                                     vnet_dev_rt_op_t *, u32);
 647
 648 /* format.c */
 649 typedef struct
 650 {
 651   u8 counters : 1;
 652   u8 show_zero_counters : 1;
 653   u8 debug : 1;
 654 } vnet_dev_format_args_t;
 655
 656 format_function_t format_vnet_dev_addr;
 657 format_function_t format_vnet_dev_flags;
 658 format_function_t format_vnet_dev_hw_addr;
 659 format_function_t format_vnet_dev_info;
 660 format_function_t format_vnet_dev_interface_info;
 661 format_function_t format_vnet_dev_interface_name;
 662 format_function_t format_vnet_dev_log;
 663 format_function_t format_vnet_dev_port_caps;
 664 format_function_t format_vnet_dev_port_flags;
 665 format_function_t format_vnet_dev_port_info;
 666 format_function_t format_vnet_dev_port_rx_offloads;
 667 format_function_t format_vnet_dev_port_tx_offloads;
 668 format_function_t format_vnet_dev_rv;
 669 format_function_t format_vnet_dev_rx_queue_info;
 670 format_function_t format_vnet_dev_tx_queue_info;
 671 format_function_t format_vnet_dev_flow;
 672 unformat_function_t unformat_vnet_dev_flags;
 673 unformat_function_t unformat_vnet_dev_port_flags;
 674
 675 typedef struct
 676 {
 677   vnet_dev_rx_queue_t *first_rx_queue;
 678 } vnet_dev_rx_node_runtime_t;
 679
 680 STATIC_ASSERT (sizeof (vnet_dev_rx_node_runtime_t) <=
 681                  VLIB_NODE_RUNTIME_DATA_SIZE,
 682                "must fit into runtime data");
 683
 684 #define foreach_vnet_dev_port_rx_next                                         \
 685   _ (ETH_INPUT, "ethernet-input")                                             \
 686   _ (DROP, "error-drop")
 687
 688 typedef enum
 689 {
 690 #define _(n, s) VNET_DEV_ETH_RX_PORT_NEXT_##n,
 691   foreach_vnet_dev_port_rx_next
 692 #undef _
 693     VNET_DEV_ETH_RX_PORT_N_NEXTS
 694 } vnet_dev_eth_port_rx_next_t;
 695
 696 extern u16 vnet_dev_default_next_index_by_port_type[];
 697 extern vlib_node_registration_t port_rx_eth_node;
 698
 699 typedef vnet_interface_output_runtime_t vnet_dev_tx_node_runtime_t;
 700
 701 STATIC_ASSERT (sizeof (vnet_dev_tx_node_runtime_t) <=
 702                  VLIB_NODE_RUNTIME_DATA_SIZE,
 703                "must fit into runtime data");
 704
 705 #define VNET_DEV_REGISTER_BUS(x, ...)                                         \
 706   __VA_ARGS__ vnet_dev_bus_registration_t __vnet_dev_bus_registration_##x;    \
 707   static void __clib_constructor __vnet_dev_bus_registration_fn_##x (void)    \
 708   {                                                                           \
 709     vnet_dev_main_t *dm = &vnet_dev_main;                                     \
 710     __vnet_dev_bus_registration_##x.next_registration =                       \
 711       dm->bus_registrations;                                                  \
 712     dm->bus_registrations = &__vnet_dev_bus_registration_##x;                 \
 713   }                                                                           \
 714   __VA_ARGS__ vnet_dev_bus_registration_t __vnet_dev_bus_registration_##x
 715
 716 #define VNET_DEV_REGISTER_DRIVER(x, ...)                                      \
 717   __VA_ARGS__ vnet_dev_driver_registration_t                                  \
 718     __vnet_dev_driver_registration_##x;                                       \
 719   static void __clib_constructor __vnet_dev_driver_registration_fn_##x (void) \
 720   {                                                                           \
 721     vnet_dev_main_t *dm = &vnet_dev_main;                                     \
 722     __vnet_dev_driver_registration_##x.next_registration =                    \
 723       dm->driver_registrations;                                               \
 724     dm->driver_registrations = &__vnet_dev_driver_registration_##x;           \
 725   }                                                                           \
 726   __VA_ARGS__ vnet_dev_driver_registration_t __vnet_dev_driver_registration_##x
 727
 728 #define VNET_DEV_NODE_FN(node)                                                \
 729   uword CLIB_MARCH_SFX (node##_fn) (vlib_main_t *, vlib_node_runtime_t *,     \
 730                                     vlib_frame_t *);                          \
 731   static vlib_node_fn_registration_t CLIB_MARCH_SFX (                         \
 732     node##_fn_registration) = {                                               \
 733     .function = &CLIB_MARCH_SFX (node##_fn),                                  \
 734   };                                                                          \
 735                                                                               \
 736   static void __clib_constructor CLIB_MARCH_SFX (                             \
 737     node##_fn_multiarch_register) (void)                                      \
 738   {                                                                           \
 739     extern vnet_dev_node_t node;                                              \
 740     vlib_node_fn_registration_t *r;                                           \
 741     r = &CLIB_MARCH_SFX (node##_fn_registration);                             \
 742     r->march_variant = CLIB_MARCH_SFX (CLIB_MARCH_VARIANT_TYPE);              \
 743     r->next_registration = (node).registrations;                              \
 744     (node).registrations = r;                                                 \
 745   }                                                                           \
 746   uword CLIB_MARCH_SFX (node##_fn)
 747
 748 #define foreach_vnet_dev_port(p, d) pool_foreach_pointer (p, d->ports)
 749 #define foreach_vnet_dev_port_rx_queue(q, p)                                  \
 750   pool_foreach_pointer (q, p->rx_queues)
 751 #define foreach_vnet_dev_port_tx_queue(q, p)                                  \
 752   pool_foreach_pointer (q, p->tx_queues)
 753
 754 #include <vnet/dev/dev_funcs.h>
 755
 756 #endif /* _VNET_DEV_H_ */