src/vnet/dev/dev.h

   1 /* SPDX-License-Identifier: Apache-2.0
   2  * Copyright (c) 2023 Cisco Systems, Inc.
   3  */
   4
   5 #ifndef _VNET_DEV_H_
   6 #define _VNET_DEV_H_
   7
   8 #include <vppinfra/clib.h>
   9 #include <vppinfra/error_bootstrap.h>
  10 #include <vppinfra/format.h>
  11 #include <vnet/vnet.h>
  12 #include <vnet/dev/types.h>
  13 #include <vnet/dev/args.h>
  14
  15 #define VNET_DEV_DEVICE_ID_PREFIX_DELIMITER "/"
  16
  17 #define foreach_vnet_dev_port_type                                            \
  18   _ (0, UNKNOWN)                                                              \
  19   _ (1, ETHERNET)
  20
  21 typedef char vnet_dev_device_id_t[32];
  22
  23 typedef enum
  24 {
  25 #define _(b, n) VNET_DEV_PORT_TYPE_##n = (1U << (b)),
  26   foreach_vnet_dev_port_type
  27 #undef _
  28 } vnet_dev_port_type_t;
  29
  30 #define foreach_vnet_dev_port_caps                                            \
  31   _ (interrupt_mode)                                                          \
  32   _ (rss)                                                                     \
  33   _ (change_max_rx_frame_size)
  34
  35 typedef union
  36 {
  37   struct
  38   {
  39 #define _(n) u8 n : 1;
  40     foreach_vnet_dev_port_caps
  41 #undef _
  42   };
  43   u8 as_number;
  44 } vnet_dev_port_caps_t;
  45
  46 typedef union
  47 {
  48   u8 eth_mac[6];
  49   u8 raw[8];
  50 } vnet_dev_hw_addr_t;
  51
  52 typedef struct vnet_dev_bus_registration vnet_dev_bus_registration_t;
  53 typedef struct vnet_dev_driver_registration vnet_dev_driver_registration_t;
  54
  55 typedef struct vnet_dev vnet_dev_t;
  56 typedef struct vnet_dev_port vnet_dev_port_t;
  57 typedef struct vnet_dev_rx_queue vnet_dev_rx_queue_t;
  58 typedef struct vnet_dev_tx_queue vnet_dev_tx_queue_t;
  59 typedef struct vnet_dev_bus_registration vnet_dev_bus_registration_t;
  60 typedef struct vnet_dev_driver_registration vnet_dev_driver_registration_t;
  61 typedef struct vnet_dev_counter vnet_dev_counter_t;
  62 typedef struct vnet_dev_counter_main vnet_dev_counter_main_t;
  63 typedef struct vnet_dev_port_cfg_change_req vnet_dev_port_cfg_change_req_t;
  64
  65 typedef vnet_dev_rv_t (vnet_dev_op_t) (vlib_main_t *, vnet_dev_t *);
  66 typedef vnet_dev_rv_t (vnet_dev_port_op_t) (vlib_main_t *, vnet_dev_port_t *);
  67 typedef vnet_dev_rv_t (vnet_dev_port_cfg_change_op_t) (
  68   vlib_main_t *, vnet_dev_port_t *, vnet_dev_port_cfg_change_req_t *);
  69 typedef vnet_dev_rv_t (vnet_dev_rx_queue_op_t) (vlib_main_t *,
  70                                                 vnet_dev_rx_queue_t *);
  71 typedef vnet_dev_rv_t (vnet_dev_tx_queue_op_t) (vlib_main_t *,
  72                                                 vnet_dev_tx_queue_t *);
  73 typedef void (vnet_dev_op_no_rv_t) (vlib_main_t *, vnet_dev_t *);
  74 typedef void (vnet_dev_port_op_no_rv_t) (vlib_main_t *, vnet_dev_port_t *);
  75 typedef void (vnet_dev_rx_queue_op_no_rv_t) (vlib_main_t *,
  76                                              vnet_dev_rx_queue_t *);
  77 typedef void (vnet_dev_tx_queue_op_no_rv_t) (vlib_main_t *,
  78                                              vnet_dev_tx_queue_t *);
  79
  80 typedef u16 vnet_dev_queue_id_t;
  81 typedef u16 vnet_dev_bus_index_t;
  82 typedef u16 vnet_dev_driver_index_t;
  83
  84 typedef struct
  85 {
  86   vnet_dev_rx_queue_op_t *alloc;
  87   vnet_dev_rx_queue_op_t *start;
  88   vnet_dev_rx_queue_op_no_rv_t *stop;
  89   vnet_dev_rx_queue_op_no_rv_t *free;
  90 } vnet_dev_rx_queue_ops_t;
  91
  92 typedef struct
  93 {
  94   vnet_dev_tx_queue_op_t *alloc;
  95   vnet_dev_tx_queue_op_t *start;
  96   vnet_dev_tx_queue_op_no_rv_t *stop;
  97   vnet_dev_tx_queue_op_no_rv_t *free;
  98 } vnet_dev_tx_queue_ops_t;
  99
 100 typedef struct
 101 {
 102   u16 data_size;
 103   u16 min_size;
 104   u16 max_size;
 105   u16 default_size;
 106   u8 multiplier;
 107   u8 size_is_power_of_two : 1;
 108 } vnet_dev_queue_config_t;
 109
 110 #define foreach_vnet_dev_port_cfg_type                                        \
 111   _ (PROMISC_MODE)                                                            \
 112   _ (MAX_RX_FRAME_SIZE)                                                       \
 113   _ (CHANGE_PRIMARY_HW_ADDR)                                                  \
 114   _ (ADD_SECONDARY_HW_ADDR)                                                   \
 115   _ (REMOVE_SECONDARY_HW_ADDR)                                                \
 116   _ (RXQ_INTR_MODE_ENABLE)                                                    \
 117   _ (RXQ_INTR_MODE_DISABLE)
 118
 119 typedef enum
 120 {
 121   VNET_DEV_PORT_CFG_UNKNOWN,
 122 #define _(n) VNET_DEV_PORT_CFG_##n,
 123   foreach_vnet_dev_port_cfg_type
 124 #undef _
 125 } __clib_packed vnet_dev_port_cfg_type_t;
 126
 127 typedef struct vnet_dev_port_cfg_change_req
 128 {
 129   vnet_dev_port_cfg_type_t type;
 130   u8 validated : 1;
 131   u8 all_queues : 1;
 132
 133   union
 134   {
 135     u8 promisc : 1;
 136     vnet_dev_hw_addr_t addr;
 137     u16 max_rx_frame_size;
 138     vnet_dev_queue_id_t queue_id;
 139   };
 140
 141 } vnet_dev_port_cfg_change_req_t;
 142
 143 typedef struct
 144 {
 145   vnet_dev_hw_addr_t hw_addr;
 146   u16 max_rx_queues;
 147   u16 max_tx_queues;
 148   u16 max_supported_rx_frame_size;
 149   vnet_dev_port_type_t type;
 150   vnet_dev_port_caps_t caps;
 151 } vnet_dev_port_attr_t;
 152
 153 typedef enum
 154 {
 155   VNET_DEV_PERIODIC_OP_TYPE_DEV = 1,
 156   VNET_DEV_PERIODIC_OP_TYPE_PORT = 2,
 157 } __clib_packed vnet_dev_periodic_op_type_t;
 158
 159 typedef struct
 160 {
 161   f64 interval;
 162   f64 last_run;
 163   vnet_dev_periodic_op_type_t type;
 164   union
 165   {
 166     vnet_dev_t *dev;
 167     vnet_dev_port_t *port;
 168     void *arg;
 169   };
 170   union
 171   {
 172     vnet_dev_op_no_rv_t *dev_op;
 173     vnet_dev_port_op_no_rv_t *port_op;
 174     void *op;
 175   };
 176 } vnet_dev_periodic_op_t;
 177
 178 typedef struct
 179 {
 180   struct _vlib_node_fn_registration *registrations;
 181   format_function_t *format_trace;
 182   vlib_error_desc_t *error_counters;
 183   u16 n_error_counters;
 184 } vnet_dev_node_t;
 185
 186 typedef struct
 187 {
 188   vnet_dev_op_t *alloc;
 189   vnet_dev_op_t *init;
 190   vnet_dev_op_no_rv_t *deinit;
 191   vnet_dev_op_t *reset;
 192   vnet_dev_op_no_rv_t *free;
 193   u8 *(*probe) (vlib_main_t *, vnet_dev_bus_index_t, void *);
 194   format_function_t *format_info;
 195 } vnet_dev_ops_t;
 196
 197 typedef struct
 198 {
 199   vnet_dev_port_op_t *alloc;
 200   vnet_dev_port_op_t *init;
 201   vnet_dev_port_cfg_change_op_t *config_change;
 202   vnet_dev_port_cfg_change_op_t *config_change_validate;
 203   vnet_dev_port_op_t *start;
 204   vnet_dev_port_op_no_rv_t *stop;
 205   vnet_dev_port_op_no_rv_t *deinit;
 206   vnet_dev_port_op_no_rv_t *free;
 207   format_function_t *format_status;
 208 } vnet_dev_port_ops_t;
 209
 210 typedef union
 211 {
 212   struct
 213   {
 214     u8 update_next_index : 1;
 215     u8 update_feature_arc : 1;
 216     u8 suspend_off : 1;
 217     u8 suspend_on : 1;
 218   };
 219   u8 as_number;
 220 } vnet_dev_rx_queue_rt_req_t;
 221
 222 typedef struct vnet_dev_rx_queue
 223 {
 224   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 225   vnet_dev_port_t *port;
 226   u16 rx_thread_index;
 227   u16 index;
 228   vnet_dev_counter_main_t *counter_main;
 229   CLIB_CACHE_LINE_ALIGN_MARK (runtime0);
 230   vnet_dev_rx_queue_t *next_on_thread;
 231   u8 interrupt_mode : 1;
 232   u8 enabled : 1;
 233   u8 started : 1;
 234   u8 suspended : 1;
 235   vnet_dev_queue_id_t queue_id;
 236   u16 size;
 237   u16 next_index;
 238   vnet_dev_rx_queue_rt_req_t runtime_request;
 239   CLIB_CACHE_LINE_ALIGN_MARK (runtime1);
 240   vlib_buffer_template_t buffer_template;
 241   CLIB_CACHE_LINE_ALIGN_MARK (driver_data);
 242   u8 data[];
 243 } vnet_dev_rx_queue_t;
 244
 245 STATIC_ASSERT_SIZEOF (vnet_dev_rx_queue_t, 3 * CLIB_CACHE_LINE_BYTES);
 246
 247 typedef struct vnet_dev_tx_queue
 248 {
 249   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 250   vnet_dev_port_t *port;
 251   clib_bitmap_t *assigned_threads;
 252   u16 index;
 253   vnet_dev_counter_main_t *counter_main;
 254   CLIB_CACHE_LINE_ALIGN_MARK (runtime0);
 255   vnet_dev_queue_id_t queue_id;
 256   u8 started : 1;
 257   u8 enabled : 1;
 258   u8 lock_needed : 1;
 259   u8 lock;
 260   u16 size;
 261   CLIB_ALIGN_MARK (private_data, 16);
 262   u8 data[];
 263 } vnet_dev_tx_queue_t;
 264
 265 STATIC_ASSERT_SIZEOF (vnet_dev_tx_queue_t, 2 * CLIB_CACHE_LINE_BYTES);
 266
 267 typedef struct vnet_dev_port
 268 {
 269   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 270   vnet_dev_t *dev;
 271   vnet_dev_port_id_t port_id;
 272   vnet_dev_driver_index_t driver_index;
 273   u8 initialized : 1;
 274   u8 started : 1;
 275   u8 link_up : 1;
 276   u8 promisc : 1;
 277   u8 interface_created : 1;
 278   u8 rx_node_assigned : 1;
 279   vnet_dev_counter_main_t *counter_main;
 280   vnet_dev_queue_config_t rx_queue_config;
 281   vnet_dev_queue_config_t tx_queue_config;
 282   vnet_dev_port_attr_t attr;
 283   u32 max_rx_frame_size;
 284   vnet_dev_hw_addr_t primary_hw_addr;
 285   vnet_dev_hw_addr_t *secondary_hw_addr;
 286   u32 index;
 287   u32 speed;
 288   vnet_dev_rx_queue_t **rx_queues;
 289   vnet_dev_tx_queue_t **tx_queues;
 290   vnet_dev_port_ops_t port_ops;
 291   vnet_dev_arg_t *args;
 292   vnet_dev_rx_queue_ops_t rx_queue_ops;
 293   vnet_dev_tx_queue_ops_t tx_queue_ops;
 294   vnet_dev_node_t rx_node;
 295   vnet_dev_node_t tx_node;
 296
 297   struct
 298   {
 299     vnet_dev_if_name_t name;
 300     u32 dev_instance;
 301     u32 rx_node_index;
 302     u32 current_config_index;
 303     u16 rx_next_index;
 304     u16 redirect_to_node_next_index;
 305     u8 feature_arc_index;
 306     u8 feature_arc : 1;
 307     u8 redirect_to_node : 1;
 308     u8 default_is_intr_mode : 1;
 309     u32 tx_node_index;
 310     u32 hw_if_index;
 311     u32 sw_if_index;
 312     u16 num_rx_queues;
 313     u16 num_tx_queues;
 314     u16 txq_sz;
 315     u16 rxq_sz;
 316   } intf;
 317
 318   CLIB_CACHE_LINE_ALIGN_MARK (data0);
 319   u8 data[];
 320 } vnet_dev_port_t;
 321
 322 typedef struct vnet_dev
 323 {
 324   vnet_dev_device_id_t device_id;
 325   u16 initialized : 1;
 326   u16 not_first_init : 1;
 327   u16 va_dma : 1;
 328   u16 process_node_quit : 1;
 329   u16 process_node_periodic : 1;
 330   u16 poll_stats : 1;
 331   u16 bus_index;
 332   u8 numa_node;
 333   u16 max_rx_queues;
 334   u16 max_tx_queues;
 335   vnet_dev_driver_index_t driver_index;
 336   u32 index;
 337   u32 process_node_index;
 338   u8 bus_data[32] __clib_aligned (16);
 339   vnet_dev_ops_t ops;
 340   vnet_dev_port_t **ports;
 341   vnet_dev_periodic_op_t *periodic_ops;
 342   u8 *description;
 343   vnet_dev_arg_t *args;
 344   u8 __clib_aligned (16)
 345   data[];
 346 } vnet_dev_t;
 347
 348 typedef struct
 349 {
 350   u16 vendor_id, device_id;
 351   char *description;
 352 } vnet_dev_match_t;
 353
 354 #define VNET_DEV_MATCH(...)                                                   \
 355   (vnet_dev_match_t[])                                                        \
 356   {                                                                           \
 357     __VA_ARGS__, {}                                                           \
 358   }
 359
 360 typedef struct
 361 {
 362   vnet_dev_op_t *device_open;
 363   vnet_dev_op_no_rv_t *device_close;
 364   vnet_dev_rv_t (*dma_mem_alloc_fn) (vlib_main_t *, vnet_dev_t *, u32, u32,
 365                                      void **);
 366   void (*dma_mem_free_fn) (vlib_main_t *, vnet_dev_t *, void *);
 367   void *(*get_device_info) (vlib_main_t *, char *);
 368   void (*free_device_info) (vlib_main_t *, void *);
 369   format_function_t *format_device_info;
 370   format_function_t *format_device_addr;
 371 } vnet_dev_bus_ops_t;
 372
 373 struct vnet_dev_bus_registration
 374 {
 375   vnet_dev_bus_registration_t *next_registration;
 376   vnet_dev_driver_name_t name;
 377   u16 device_data_size;
 378   vnet_dev_bus_ops_t ops;
 379 };
 380
 381 struct vnet_dev_driver_registration
 382 {
 383   vnet_dev_driver_registration_t *next_registration;
 384   u8 bus_master_enable : 1;
 385   vnet_dev_driver_name_t name;
 386   vnet_dev_bus_name_t bus;
 387   u16 device_data_sz;
 388   u16 runtime_temp_space_sz;
 389   vnet_dev_match_t *match;
 390   int priority;
 391   vnet_dev_ops_t ops;
 392   vnet_dev_arg_t *args;
 393 };
 394
 395 typedef struct
 396 {
 397   u32 index;
 398   vnet_dev_bus_registration_t *registration;
 399   vnet_dev_bus_ops_t ops;
 400 } vnet_dev_bus_t;
 401
 402 typedef struct
 403 {
 404   u32 index;
 405   void *dev_data;
 406   vnet_dev_driver_registration_t *registration;
 407   u32 dev_class_index;
 408   vnet_dev_bus_index_t bus_index;
 409   vnet_dev_ops_t ops;
 410 } vnet_dev_driver_t;
 411
 412 typedef struct
 413 {
 414   vnet_dev_bus_t *buses;
 415   vnet_dev_driver_t *drivers;
 416   vnet_dev_t **devices;
 417   vnet_dev_port_t **ports_by_dev_instance;
 418   vnet_dev_bus_registration_t *bus_registrations;
 419   vnet_dev_driver_registration_t *driver_registrations;
 420   void *runtime_temp_spaces;
 421   u32 log2_runtime_temp_space_sz;
 422   u32 *free_process_node_indices;
 423   u32 *free_rx_node_indices;
 424   uword *device_index_by_id;
 425
 426   u8 *startup_config;
 427   u16 next_rx_queue_thread;
 428   u8 eth_port_rx_feature_arc_index;
 429 } vnet_dev_main_t;
 430
 431 extern vnet_dev_main_t vnet_dev_main;
 432
 433 typedef struct
 434 {
 435   struct
 436   {
 437     vnet_dev_port_attr_t attr;
 438     vnet_dev_port_ops_t ops;
 439     vnet_dev_arg_t *args;
 440     u16 data_size;
 441     void *initial_data;
 442   } port;
 443
 444   vnet_dev_node_t *rx_node;
 445   vnet_dev_node_t *tx_node;
 446
 447   struct
 448   {
 449     vnet_dev_queue_config_t config;
 450     vnet_dev_rx_queue_ops_t ops;
 451   } rx_queue;
 452
 453   struct
 454   {
 455     vnet_dev_queue_config_t config;
 456     vnet_dev_tx_queue_ops_t ops;
 457   } tx_queue;
 458 } vnet_dev_port_add_args_t;
 459
 460 typedef struct
 461 {
 462   union
 463   {
 464     struct
 465     {
 466       u8 link_speed : 1;
 467       u8 link_state : 1;
 468       u8 link_duplex : 1;
 469     };
 470     u8 any;
 471   } change;
 472   u8 link_state : 1;
 473   u8 full_duplex : 1;
 474   u32 link_speed;
 475 } vnet_dev_port_state_changes_t;
 476
 477 /* args.c */
 478 vnet_dev_rv_t vnet_dev_arg_parse (vlib_main_t *, vnet_dev_t *,
 479                                   vnet_dev_arg_t *, u8 *);
 480 void vnet_dev_arg_free (vnet_dev_arg_t **);
 481 void vnet_dev_arg_clear_value (vnet_dev_arg_t *);
 482 format_function_t format_vnet_dev_arg_type;
 483 format_function_t format_vnet_dev_arg_value;
 484 format_function_t format_vnet_dev_args;
 485
 486 /* dev.c */
 487 vnet_dev_t *vnet_dev_alloc (vlib_main_t *, vnet_dev_device_id_t,
 488                             vnet_dev_driver_t *);
 489 void vnet_dev_free (vlib_main_t *, vnet_dev_t *);
 490 vnet_dev_rv_t vnet_dev_init (vlib_main_t *, vnet_dev_t *);
 491 void vnet_dev_deinit (vlib_main_t *, vnet_dev_t *);
 492 vnet_dev_rv_t vnet_dev_reset (vlib_main_t *, vnet_dev_t *);
 493 void vnet_dev_detach (vlib_main_t *, vnet_dev_t *);
 494 vnet_dev_rv_t vnet_dev_port_add (vlib_main_t *, vnet_dev_t *,
 495                                  vnet_dev_port_id_t,
 496                                  vnet_dev_port_add_args_t *);
 497 vnet_dev_rv_t vnet_dev_dma_mem_alloc (vlib_main_t *, vnet_dev_t *, u32, u32,
 498                                       void **);
 499 void vnet_dev_dma_mem_free (vlib_main_t *, vnet_dev_t *, void *);
 500 vnet_dev_bus_t *vnet_dev_find_device_bus (vlib_main_t *, vnet_dev_device_id_t);
 501 void *vnet_dev_get_device_info (vlib_main_t *, vnet_dev_device_id_t);
 502
 503 /* error.c */
 504 clib_error_t *vnet_dev_port_err (vlib_main_t *, vnet_dev_port_t *,
 505                                  vnet_dev_rv_t, char *, ...);
 506
 507 /* handlers.c */
 508 clib_error_t *vnet_dev_port_set_max_frame_size (vnet_main_t *,
 509                                                 vnet_hw_interface_t *, u32);
 510 u32 vnet_dev_port_eth_flag_change (vnet_main_t *, vnet_hw_interface_t *, u32);
 511 clib_error_t *vnet_dev_port_mac_change (vnet_hw_interface_t *, const u8 *,
 512                                         const u8 *);
 513 clib_error_t *vnet_dev_add_del_mac_address (vnet_hw_interface_t *, const u8 *,
 514                                             u8);
 515 int vnet_dev_flow_ops_fn (vnet_main_t *, vnet_flow_dev_op_t, u32, u32,
 516                           uword *);
 517 clib_error_t *vnet_dev_interface_set_rss_queues (vnet_main_t *,
 518                                                  vnet_hw_interface_t *,
 519                                                  clib_bitmap_t *);
 520 void vnet_dev_clear_hw_interface_counters (u32);
 521 void vnet_dev_set_interface_next_node (vnet_main_t *, u32, u32);
 522
 523 /* port.c */
 524 vnet_dev_rv_t vnet_dev_port_start (vlib_main_t *, vnet_dev_port_t *);
 525 vnet_dev_rv_t vnet_dev_port_start_all_rx_queues (vlib_main_t *,
 526                                                  vnet_dev_port_t *);
 527 vnet_dev_rv_t vnet_dev_port_start_all_tx_queues (vlib_main_t *,
 528                                                  vnet_dev_port_t *);
 529 void vnet_dev_port_stop (vlib_main_t *, vnet_dev_port_t *);
 530 void vnet_dev_port_deinit (vlib_main_t *, vnet_dev_port_t *);
 531 void vnet_dev_port_free (vlib_main_t *, vnet_dev_port_t *);
 532 void vnet_dev_port_add_counters (vlib_main_t *, vnet_dev_port_t *,
 533                                  vnet_dev_counter_t *, u16);
 534 void vnet_dev_port_free_counters (vlib_main_t *, vnet_dev_port_t *);
 535 void vnet_dev_port_update_tx_node_runtime (vlib_main_t *, vnet_dev_port_t *);
 536 void vnet_dev_port_state_change (vlib_main_t *, vnet_dev_port_t *,
 537                                  vnet_dev_port_state_changes_t);
 538 void vnet_dev_port_clear_counters (vlib_main_t *, vnet_dev_port_t *);
 539 vnet_dev_rv_t
 540 vnet_dev_port_cfg_change_req_validate (vlib_main_t *, vnet_dev_port_t *,
 541                                        vnet_dev_port_cfg_change_req_t *);
 542 vnet_dev_rv_t vnet_dev_port_cfg_change (vlib_main_t *, vnet_dev_port_t *,
 543                                         vnet_dev_port_cfg_change_req_t *);
 544 vnet_dev_rv_t vnet_dev_port_if_create (vlib_main_t *, vnet_dev_port_t *);
 545 vnet_dev_rv_t vnet_dev_port_if_remove (vlib_main_t *, vnet_dev_port_t *);
 546
 547 /* queue.c */
 548 vnet_dev_rv_t vnet_dev_rx_queue_alloc (vlib_main_t *, vnet_dev_port_t *, u16);
 549 vnet_dev_rv_t vnet_dev_tx_queue_alloc (vlib_main_t *, vnet_dev_port_t *, u16);
 550 void vnet_dev_rx_queue_free (vlib_main_t *, vnet_dev_rx_queue_t *);
 551 void vnet_dev_tx_queue_free (vlib_main_t *, vnet_dev_tx_queue_t *);
 552 void vnet_dev_rx_queue_add_counters (vlib_main_t *, vnet_dev_rx_queue_t *,
 553                                      vnet_dev_counter_t *, u16);
 554 void vnet_dev_rx_queue_free_counters (vlib_main_t *, vnet_dev_rx_queue_t *);
 555 void vnet_dev_tx_queue_add_counters (vlib_main_t *, vnet_dev_tx_queue_t *,
 556                                      vnet_dev_counter_t *, u16);
 557 void vnet_dev_tx_queue_free_counters (vlib_main_t *, vnet_dev_tx_queue_t *);
 558 vnet_dev_rv_t vnet_dev_rx_queue_start (vlib_main_t *, vnet_dev_rx_queue_t *);
 559 vnet_dev_rv_t vnet_dev_tx_queue_start (vlib_main_t *, vnet_dev_tx_queue_t *);
 560 void vnet_dev_rx_queue_stop (vlib_main_t *, vnet_dev_rx_queue_t *);
 561 void vnet_dev_tx_queue_stop (vlib_main_t *, vnet_dev_tx_queue_t *);
 562
 563 /* process.c */
 564 vnet_dev_rv_t vnet_dev_process_create (vlib_main_t *, vnet_dev_t *);
 565 vnet_dev_rv_t vnet_dev_process_call_op (vlib_main_t *, vnet_dev_t *,
 566                                         vnet_dev_op_t *);
 567 vnet_dev_rv_t vnet_dev_process_call_op_no_rv (vlib_main_t *, vnet_dev_t *,
 568                                               vnet_dev_op_no_rv_t *);
 569 void vnet_dev_process_call_op_no_wait (vlib_main_t *, vnet_dev_t *,
 570                                        vnet_dev_op_no_rv_t *);
 571 vnet_dev_rv_t vnet_dev_process_call_port_op (vlib_main_t *, vnet_dev_port_t *,
 572                                              vnet_dev_port_op_t *);
 573 vnet_dev_rv_t vnet_dev_process_call_port_op_no_rv (vlib_main_t *vm,
 574                                                    vnet_dev_port_t *,
 575                                                    vnet_dev_port_op_no_rv_t *);
 576 void vnet_dev_process_call_port_op_no_wait (vlib_main_t *, vnet_dev_port_t *,
 577                                             vnet_dev_port_op_no_rv_t *);
 578 vnet_dev_rv_t
 579 vnet_dev_process_port_cfg_change_req (vlib_main_t *, vnet_dev_port_t *,
 580                                       vnet_dev_port_cfg_change_req_t *);
 581 void vnet_dev_process_quit (vlib_main_t *, vnet_dev_t *);
 582 void vnet_dev_poll_dev_add (vlib_main_t *, vnet_dev_t *, f64,
 583                             vnet_dev_op_no_rv_t *);
 584 void vnet_dev_poll_dev_remove (vlib_main_t *, vnet_dev_t *,
 585                                vnet_dev_op_no_rv_t *);
 586 void vnet_dev_poll_port_add (vlib_main_t *, vnet_dev_port_t *, f64,
 587                              vnet_dev_port_op_no_rv_t *);
 588 void vnet_dev_poll_port_remove (vlib_main_t *, vnet_dev_port_t *,
 589                                 vnet_dev_port_op_no_rv_t *);
 590
 591 typedef struct
 592 {
 593   u16 thread_index;
 594   u8 completed;
 595   u8 in_order;
 596   vnet_dev_port_t *port;
 597 } vnet_dev_rt_op_t;
 598
 599 vnet_dev_rv_t vnet_dev_rt_exec_ops (vlib_main_t *, vnet_dev_t *,
 600                                     vnet_dev_rt_op_t *, u32);
 601
 602 /* format.c */
 603 typedef struct
 604 {
 605   u8 counters : 1;
 606   u8 show_zero_counters : 1;
 607   u8 debug : 1;
 608 } vnet_dev_format_args_t;
 609
 610 format_function_t format_vnet_dev_addr;
 611 format_function_t format_vnet_dev_hw_addr;
 612 format_function_t format_vnet_dev_info;
 613 format_function_t format_vnet_dev_interface_info;
 614 format_function_t format_vnet_dev_interface_name;
 615 format_function_t format_vnet_dev_port_info;
 616 format_function_t format_vnet_dev_rv;
 617 format_function_t format_vnet_dev_rx_queue_info;
 618 format_function_t format_vnet_dev_tx_queue_info;
 619 format_function_t format_vnet_dev_flags;
 620 format_function_t format_vnet_dev_port_flags;
 621 format_function_t format_vnet_dev_log;
 622 unformat_function_t unformat_vnet_dev_flags;
 623 unformat_function_t unformat_vnet_dev_port_flags;
 624
 625 typedef struct
 626 {
 627   vnet_dev_rx_queue_t *first_rx_queue;
 628 } vnet_dev_rx_node_runtime_t;
 629
 630 STATIC_ASSERT (sizeof (vnet_dev_rx_node_runtime_t) <=
 631                  VLIB_NODE_RUNTIME_DATA_SIZE,
 632                "must fit into runtime data");
 633
 634 #define foreach_vnet_dev_port_rx_next                                         \
 635   _ (ETH_INPUT, "ethernet-input")                                             \
 636   _ (DROP, "error-drop")
 637
 638 typedef enum
 639 {
 640 #define _(n, s) VNET_DEV_ETH_RX_PORT_NEXT_##n,
 641   foreach_vnet_dev_port_rx_next
 642 #undef _
 643     VNET_DEV_ETH_RX_PORT_N_NEXTS
 644 } vnet_dev_eth_port_rx_next_t;
 645
 646 extern u16 vnet_dev_default_next_index_by_port_type[];
 647 extern vlib_node_registration_t port_rx_eth_node;
 648
 649 typedef vnet_interface_output_runtime_t vnet_dev_tx_node_runtime_t;
 650
 651 STATIC_ASSERT (sizeof (vnet_dev_tx_node_runtime_t) <=
 652                  VLIB_NODE_RUNTIME_DATA_SIZE,
 653                "must fit into runtime data");
 654
 655 #define VNET_DEV_REGISTER_BUS(x, ...)                                         \
 656   __VA_ARGS__ vnet_dev_bus_registration_t __vnet_dev_bus_registration_##x;    \
 657   static void __clib_constructor __vnet_dev_bus_registration_fn_##x (void)    \
 658   {                                                                           \
 659     vnet_dev_main_t *dm = &vnet_dev_main;                                     \
 660     __vnet_dev_bus_registration_##x.next_registration =                       \
 661       dm->bus_registrations;                                                  \
 662     dm->bus_registrations = &__vnet_dev_bus_registration_##x;                 \
 663   }                                                                           \
 664   __VA_ARGS__ vnet_dev_bus_registration_t __vnet_dev_bus_registration_##x
 665
 666 #define VNET_DEV_REGISTER_DRIVER(x, ...)                                      \
 667   __VA_ARGS__ vnet_dev_driver_registration_t                                  \
 668     __vnet_dev_driver_registration_##x;                                       \
 669   static void __clib_constructor __vnet_dev_driver_registration_fn_##x (void) \
 670   {                                                                           \
 671     vnet_dev_main_t *dm = &vnet_dev_main;                                     \
 672     __vnet_dev_driver_registration_##x.next_registration =                    \
 673       dm->driver_registrations;                                               \
 674     dm->driver_registrations = &__vnet_dev_driver_registration_##x;           \
 675   }                                                                           \
 676   __VA_ARGS__ vnet_dev_driver_registration_t __vnet_dev_driver_registration_##x
 677
 678 #define VNET_DEV_NODE_FN(node)                                                \
 679   uword CLIB_MARCH_SFX (node##_fn) (vlib_main_t *, vlib_node_runtime_t *,     \
 680                                     vlib_frame_t *);                          \
 681   static vlib_node_fn_registration_t CLIB_MARCH_SFX (                         \
 682     node##_fn_registration) = {                                               \
 683     .function = &CLIB_MARCH_SFX (node##_fn),                                  \
 684   };                                                                          \
 685                                                                               \
 686   static void __clib_constructor CLIB_MARCH_SFX (                             \
 687     node##_fn_multiarch_register) (void)                                      \
 688   {                                                                           \
 689     extern vnet_dev_node_t node;                                              \
 690     vlib_node_fn_registration_t *r;                                           \
 691     r = &CLIB_MARCH_SFX (node##_fn_registration);                             \
 692     r->march_variant = CLIB_MARCH_SFX (CLIB_MARCH_VARIANT_TYPE);              \
 693     r->next_registration = (node).registrations;                              \
 694     (node).registrations = r;                                                 \
 695   }                                                                           \
 696   uword CLIB_MARCH_SFX (node##_fn)
 697
 698 #define foreach_vnet_dev_port(p, d) pool_foreach_pointer (p, d->ports)
 699 #define foreach_vnet_dev_port_rx_queue(q, p)                                  \
 700   pool_foreach_pointer (q, p->rx_queues)
 701 #define foreach_vnet_dev_port_tx_queue(q, p)                                  \
 702   pool_foreach_pointer (q, p->tx_queues)
 703
 704 #include <vnet/dev/dev_funcs.h>
 705
 706 #endif /* _VNET_DEV_H_ */