lib/librte_pipeline/rte_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6 #include <stdio.h>
   7
   8 #include <rte_common.h>
   9 #include <rte_memory.h>
  10 #include <rte_cycles.h>
  11 #include <rte_prefetch.h>
  12 #include <rte_branch_prediction.h>
  13 #include <rte_mbuf.h>
  14 #include <rte_malloc.h>
  15 #include <rte_string_fns.h>
  16
  17 #include "rte_pipeline.h"
  18
  19 #define RTE_TABLE_INVALID                                 UINT32_MAX
  20
  21 #ifdef RTE_PIPELINE_STATS_COLLECT
  22
  23 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)                       \
  24         ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); })
  25
  26 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)                     \
  27         ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
  28
  29 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)                               \
  30         ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
  31
  32 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)                      \
  33 ({                                                                      \
  34         uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP];    \
  35         mask ^= (p)->pkts_drop_mask;                                    \
  36         (counter) += __builtin_popcountll(mask);                        \
  37 })
  38
  39 #else
  40
  41 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
  42 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
  43 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)
  44 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
  45
  46 #endif
  47
  48 struct rte_port_in {
  49         /* Input parameters */
  50         struct rte_port_in_ops ops;
  51         rte_pipeline_port_in_action_handler f_action;
  52         void *arg_ah;
  53         uint32_t burst_size;
  54
  55         /* The table to which this port is connected */
  56         uint32_t table_id;
  57
  58         /* Handle to low-level port */
  59         void *h_port;
  60
  61         /* List of enabled ports */
  62         struct rte_port_in *next;
  63
  64         /* Statistics */
  65         uint64_t n_pkts_dropped_by_ah;
  66 };
  67
  68 struct rte_port_out {
  69         /* Input parameters */
  70         struct rte_port_out_ops ops;
  71         rte_pipeline_port_out_action_handler f_action;
  72         void *arg_ah;
  73
  74         /* Handle to low-level port */
  75         void *h_port;
  76
  77         /* Statistics */
  78         uint64_t n_pkts_dropped_by_ah;
  79 };
  80
  81 struct rte_table {
  82         /* Input parameters */
  83         struct rte_table_ops ops;
  84         rte_pipeline_table_action_handler_hit f_action_hit;
  85         rte_pipeline_table_action_handler_miss f_action_miss;
  86         void *arg_ah;
  87         struct rte_pipeline_table_entry *default_entry;
  88         uint32_t entry_size;
  89
  90         uint32_t table_next_id;
  91         uint32_t table_next_id_valid;
  92
  93         /* Handle to the low-level table object */
  94         void *h_table;
  95
  96         /* Statistics */
  97         uint64_t n_pkts_dropped_by_lkp_hit_ah;
  98         uint64_t n_pkts_dropped_by_lkp_miss_ah;
  99         uint64_t n_pkts_dropped_lkp_hit;
 100         uint64_t n_pkts_dropped_lkp_miss;
 101 };
 102
 103 #define RTE_PIPELINE_MAX_NAME_SZ                           124
 104
 105 struct rte_pipeline {
 106         /* Input parameters */
 107         char name[RTE_PIPELINE_MAX_NAME_SZ];
 108         int socket_id;
 109         uint32_t offset_port_id;
 110
 111         /* Internal tables */
 112         struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
 113         struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
 114         struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
 115
 116         /* Occupancy of internal tables */
 117         uint32_t num_ports_in;
 118         uint32_t num_ports_out;
 119         uint32_t num_tables;
 120
 121         /* List of enabled ports */
 122         uint64_t enabled_port_in_mask;
 123         struct rte_port_in *port_in_next;
 124
 125         /* Pipeline run structures */
 126         struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
 127         struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
 128         uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
 129         uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
 130         uint64_t pkts_mask;
 131         uint64_t n_pkts_ah_drop;
 132         uint64_t pkts_drop_mask;
 133 } __rte_cache_aligned;
 134
 135 static inline uint32_t
 136 rte_mask_get_next(uint64_t mask, uint32_t pos)
 137 {
 138         uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
 139                         (mask >> ((pos + 1) & 0x3F));
 140         return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
 141 }
 142
 143 static inline uint32_t
 144 rte_mask_get_prev(uint64_t mask, uint32_t pos)
 145 {
 146         uint64_t mask_rot = (mask >> (pos & 0x3F)) |
 147                         (mask << ((64 - pos) & 0x3F));
 148         return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
 149 }
 150
 151 static void
 152 rte_pipeline_table_free(struct rte_table *table);
 153
 154 static void
 155 rte_pipeline_port_in_free(struct rte_port_in *port);
 156
 157 static void
 158 rte_pipeline_port_out_free(struct rte_port_out *port);
 159
 160 /*
 161  * Pipeline
 162  *
 163  */
 164 static int
 165 rte_pipeline_check_params(struct rte_pipeline_params *params)
 166 {
 167         if (params == NULL) {
 168                 RTE_LOG(ERR, PIPELINE,
 169                         "%s: Incorrect value for parameter params\n", __func__);
 170                 return -EINVAL;
 171         }
 172
 173         /* name */
 174         if (params->name == NULL) {
 175                 RTE_LOG(ERR, PIPELINE,
 176                         "%s: Incorrect value for parameter name\n", __func__);
 177                 return -EINVAL;
 178         }
 179
 180         /* socket */
 181         if ((params->socket_id < 0) ||
 182             (params->socket_id >= RTE_MAX_NUMA_NODES)) {
 183                 RTE_LOG(ERR, PIPELINE,
 184                         "%s: Incorrect value for parameter socket_id\n",
 185                         __func__);
 186                 return -EINVAL;
 187         }
 188
 189         return 0;
 190 }
 191
 192 struct rte_pipeline *
 193 rte_pipeline_create(struct rte_pipeline_params *params)
 194 {
 195         struct rte_pipeline *p;
 196         int status;
 197
 198         /* Check input parameters */
 199         status = rte_pipeline_check_params(params);
 200         if (status != 0) {
 201                 RTE_LOG(ERR, PIPELINE,
 202                         "%s: Pipeline params check failed (%d)\n",
 203                         __func__, status);
 204                 return NULL;
 205         }
 206
 207         /* Allocate memory for the pipeline on requested socket */
 208         p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
 209                         RTE_CACHE_LINE_SIZE, params->socket_id);
 210
 211         if (p == NULL) {
 212                 RTE_LOG(ERR, PIPELINE,
 213                         "%s: Pipeline memory allocation failed\n", __func__);
 214                 return NULL;
 215         }
 216
 217         /* Save input parameters */
 218         snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
 219         p->socket_id = params->socket_id;
 220         p->offset_port_id = params->offset_port_id;
 221
 222         /* Initialize pipeline internal data structure */
 223         p->num_ports_in = 0;
 224         p->num_ports_out = 0;
 225         p->num_tables = 0;
 226         p->enabled_port_in_mask = 0;
 227         p->port_in_next = NULL;
 228         p->pkts_mask = 0;
 229         p->n_pkts_ah_drop = 0;
 230
 231         return p;
 232 }
 233
 234 int
 235 rte_pipeline_free(struct rte_pipeline *p)
 236 {
 237         uint32_t i;
 238
 239         /* Check input parameters */
 240         if (p == NULL) {
 241                 RTE_LOG(ERR, PIPELINE,
 242                         "%s: rte_pipeline parameter is NULL\n", __func__);
 243                 return -EINVAL;
 244         }
 245
 246         /* Free input ports */
 247         for (i = 0; i < p->num_ports_in; i++) {
 248                 struct rte_port_in *port = &p->ports_in[i];
 249
 250                 rte_pipeline_port_in_free(port);
 251         }
 252
 253         /* Free tables */
 254         for (i = 0; i < p->num_tables; i++) {
 255                 struct rte_table *table = &p->tables[i];
 256
 257                 rte_pipeline_table_free(table);
 258         }
 259
 260         /* Free output ports */
 261         for (i = 0; i < p->num_ports_out; i++) {
 262                 struct rte_port_out *port = &p->ports_out[i];
 263
 264                 rte_pipeline_port_out_free(port);
 265         }
 266
 267         /* Free pipeline memory */
 268         rte_free(p);
 269
 270         return 0;
 271 }
 272
 273 /*
 274  * Table
 275  *
 276  */
 277 static int
 278 rte_table_check_params(struct rte_pipeline *p,
 279                 struct rte_pipeline_table_params *params,
 280                 uint32_t *table_id)
 281 {
 282         if (p == NULL) {
 283                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 284                         __func__);
 285                 return -EINVAL;
 286         }
 287         if (params == NULL) {
 288                 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
 289                         __func__);
 290                 return -EINVAL;
 291         }
 292         if (table_id == NULL) {
 293                 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
 294                         __func__);
 295                 return -EINVAL;
 296         }
 297
 298         /* ops */
 299         if (params->ops == NULL) {
 300                 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
 301                         __func__);
 302                 return -EINVAL;
 303         }
 304
 305         if (params->ops->f_create == NULL) {
 306                 RTE_LOG(ERR, PIPELINE,
 307                         "%s: f_create function pointer is NULL\n", __func__);
 308                 return -EINVAL;
 309         }
 310
 311         if (params->ops->f_lookup == NULL) {
 312                 RTE_LOG(ERR, PIPELINE,
 313                         "%s: f_lookup function pointer is NULL\n", __func__);
 314                 return -EINVAL;
 315         }
 316
 317         /* De we have room for one more table? */
 318         if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
 319                 RTE_LOG(ERR, PIPELINE,
 320                         "%s: Incorrect value for num_tables parameter\n",
 321                         __func__);
 322                 return -EINVAL;
 323         }
 324
 325         return 0;
 326 }
 327
 328 int
 329 rte_pipeline_table_create(struct rte_pipeline *p,
 330                 struct rte_pipeline_table_params *params,
 331                 uint32_t *table_id)
 332 {
 333         struct rte_table *table;
 334         struct rte_pipeline_table_entry *default_entry;
 335         void *h_table;
 336         uint32_t entry_size, id;
 337         int status;
 338
 339         /* Check input arguments */
 340         status = rte_table_check_params(p, params, table_id);
 341         if (status != 0)
 342                 return status;
 343
 344         id = p->num_tables;
 345         table = &p->tables[id];
 346
 347         /* Allocate space for the default table entry */
 348         entry_size = sizeof(struct rte_pipeline_table_entry) +
 349                 params->action_data_size;
 350         default_entry = rte_zmalloc_socket(
 351                 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
 352         if (default_entry == NULL) {
 353                 RTE_LOG(ERR, PIPELINE,
 354                         "%s: Failed to allocate default entry\n", __func__);
 355                 return -EINVAL;
 356         }
 357
 358         /* Create the table */
 359         h_table = params->ops->f_create(params->arg_create, p->socket_id,
 360                 entry_size);
 361         if (h_table == NULL) {
 362                 rte_free(default_entry);
 363                 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
 364                 return -EINVAL;
 365         }
 366
 367         /* Commit current table to the pipeline */
 368         p->num_tables++;
 369         *table_id = id;
 370
 371         /* Save input parameters */
 372         memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
 373         table->f_action_hit = params->f_action_hit;
 374         table->f_action_miss = params->f_action_miss;
 375         table->arg_ah = params->arg_ah;
 376         table->entry_size = entry_size;
 377
 378         /* Clear the lookup miss actions (to be set later through API) */
 379         table->default_entry = default_entry;
 380         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 381
 382         /* Initialize table internal data structure */
 383         table->h_table = h_table;
 384         table->table_next_id = 0;
 385         table->table_next_id_valid = 0;
 386
 387         return 0;
 388 }
 389
 390 void
 391 rte_pipeline_table_free(struct rte_table *table)
 392 {
 393         if (table->ops.f_free != NULL)
 394                 table->ops.f_free(table->h_table);
 395
 396         rte_free(table->default_entry);
 397 }
 398
 399 int
 400 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
 401         uint32_t table_id,
 402         struct rte_pipeline_table_entry *default_entry,
 403         struct rte_pipeline_table_entry **default_entry_ptr)
 404 {
 405         struct rte_table *table;
 406
 407         /* Check input arguments */
 408         if (p == NULL) {
 409                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 410                         __func__);
 411                 return -EINVAL;
 412         }
 413
 414         if (default_entry == NULL) {
 415                 RTE_LOG(ERR, PIPELINE,
 416                         "%s: default_entry parameter is NULL\n", __func__);
 417                 return -EINVAL;
 418         }
 419
 420         if (table_id >= p->num_tables) {
 421                 RTE_LOG(ERR, PIPELINE,
 422                         "%s: table_id %d out of range\n", __func__, table_id);
 423                 return -EINVAL;
 424         }
 425
 426         table = &p->tables[table_id];
 427
 428         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 429                 table->table_next_id_valid &&
 430                 (default_entry->table_id != table->table_next_id)) {
 431                 RTE_LOG(ERR, PIPELINE,
 432                         "%s: Tree-like topologies not allowed\n", __func__);
 433                 return -EINVAL;
 434         }
 435
 436         /* Set the lookup miss actions */
 437         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 438                 (table->table_next_id_valid == 0)) {
 439                 table->table_next_id = default_entry->table_id;
 440                 table->table_next_id_valid = 1;
 441         }
 442
 443         memcpy(table->default_entry, default_entry, table->entry_size);
 444
 445         *default_entry_ptr = table->default_entry;
 446         return 0;
 447 }
 448
 449 int
 450 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
 451                 uint32_t table_id,
 452                 struct rte_pipeline_table_entry *entry)
 453 {
 454         struct rte_table *table;
 455
 456         /* Check input arguments */
 457         if (p == NULL) {
 458                 RTE_LOG(ERR, PIPELINE,
 459                         "%s: pipeline parameter is NULL\n", __func__);
 460                 return -EINVAL;
 461         }
 462
 463         if (table_id >= p->num_tables) {
 464                 RTE_LOG(ERR, PIPELINE,
 465                         "%s: table_id %d out of range\n", __func__, table_id);
 466                 return -EINVAL;
 467         }
 468
 469         table = &p->tables[table_id];
 470
 471         /* Save the current contents of the default entry */
 472         if (entry)
 473                 memcpy(entry, table->default_entry, table->entry_size);
 474
 475         /* Clear the lookup miss actions */
 476         memset(table->default_entry, 0, table->entry_size);
 477         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 478
 479         return 0;
 480 }
 481
 482 int
 483 rte_pipeline_table_entry_add(struct rte_pipeline *p,
 484                 uint32_t table_id,
 485                 void *key,
 486                 struct rte_pipeline_table_entry *entry,
 487                 int *key_found,
 488                 struct rte_pipeline_table_entry **entry_ptr)
 489 {
 490         struct rte_table *table;
 491
 492         /* Check input arguments */
 493         if (p == NULL) {
 494                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 495                         __func__);
 496                 return -EINVAL;
 497         }
 498
 499         if (key == NULL) {
 500                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
 501                 return -EINVAL;
 502         }
 503
 504         if (entry == NULL) {
 505                 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
 506                         __func__);
 507                 return -EINVAL;
 508         }
 509
 510         if (table_id >= p->num_tables) {
 511                 RTE_LOG(ERR, PIPELINE,
 512                         "%s: table_id %d out of range\n", __func__, table_id);
 513                 return -EINVAL;
 514         }
 515
 516         table = &p->tables[table_id];
 517
 518         if (table->ops.f_add == NULL) {
 519                 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
 520                         __func__);
 521                 return -EINVAL;
 522         }
 523
 524         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 525                 table->table_next_id_valid &&
 526                 (entry->table_id != table->table_next_id)) {
 527                 RTE_LOG(ERR, PIPELINE,
 528                         "%s: Tree-like topologies not allowed\n", __func__);
 529                 return -EINVAL;
 530         }
 531
 532         /* Add entry */
 533         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 534                 (table->table_next_id_valid == 0)) {
 535                 table->table_next_id = entry->table_id;
 536                 table->table_next_id_valid = 1;
 537         }
 538
 539         return (table->ops.f_add)(table->h_table, key, (void *) entry,
 540                 key_found, (void **) entry_ptr);
 541 }
 542
 543 int
 544 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
 545                 uint32_t table_id,
 546                 void *key,
 547                 int *key_found,
 548                 struct rte_pipeline_table_entry *entry)
 549 {
 550         struct rte_table *table;
 551
 552         /* Check input arguments */
 553         if (p == NULL) {
 554                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 555                         __func__);
 556                 return -EINVAL;
 557         }
 558
 559         if (key == NULL) {
 560                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 561                         __func__);
 562                 return -EINVAL;
 563         }
 564
 565         if (table_id >= p->num_tables) {
 566                 RTE_LOG(ERR, PIPELINE,
 567                         "%s: table_id %d out of range\n", __func__, table_id);
 568                 return -EINVAL;
 569         }
 570
 571         table = &p->tables[table_id];
 572
 573         if (table->ops.f_delete == NULL) {
 574                 RTE_LOG(ERR, PIPELINE,
 575                         "%s: f_delete function pointer NULL\n", __func__);
 576                 return -EINVAL;
 577         }
 578
 579         return (table->ops.f_delete)(table->h_table, key, key_found, entry);
 580 }
 581
 582 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
 583         uint32_t table_id,
 584         void **keys,
 585         struct rte_pipeline_table_entry **entries,
 586         uint32_t n_keys,
 587         int *key_found,
 588         struct rte_pipeline_table_entry **entries_ptr)
 589 {
 590         struct rte_table *table;
 591         uint32_t i;
 592
 593         /* Check input arguments */
 594         if (p == NULL) {
 595                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 596                         __func__);
 597                 return -EINVAL;
 598         }
 599
 600         if (keys == NULL) {
 601                 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
 602                 return -EINVAL;
 603         }
 604
 605         if (entries == NULL) {
 606                 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
 607                         __func__);
 608                 return -EINVAL;
 609         }
 610
 611         if (table_id >= p->num_tables) {
 612                 RTE_LOG(ERR, PIPELINE,
 613                         "%s: table_id %d out of range\n", __func__, table_id);
 614                 return -EINVAL;
 615         }
 616
 617         table = &p->tables[table_id];
 618
 619         if (table->ops.f_add_bulk == NULL) {
 620                 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
 621                         __func__);
 622                 return -EINVAL;
 623         }
 624
 625         for (i = 0; i < n_keys; i++) {
 626                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 627                         table->table_next_id_valid &&
 628                         (entries[i]->table_id != table->table_next_id)) {
 629                         RTE_LOG(ERR, PIPELINE,
 630                                 "%s: Tree-like topologies not allowed\n", __func__);
 631                         return -EINVAL;
 632                 }
 633         }
 634
 635         /* Add entry */
 636         for (i = 0; i < n_keys; i++) {
 637                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 638                         (table->table_next_id_valid == 0)) {
 639                         table->table_next_id = entries[i]->table_id;
 640                         table->table_next_id_valid = 1;
 641                 }
 642         }
 643
 644         return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
 645                 n_keys, key_found, (void **) entries_ptr);
 646 }
 647
 648 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
 649         uint32_t table_id,
 650         void **keys,
 651         uint32_t n_keys,
 652         int *key_found,
 653         struct rte_pipeline_table_entry **entries)
 654 {
 655         struct rte_table *table;
 656
 657         /* Check input arguments */
 658         if (p == NULL) {
 659                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 660                         __func__);
 661                 return -EINVAL;
 662         }
 663
 664         if (keys == NULL) {
 665                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 666                         __func__);
 667                 return -EINVAL;
 668         }
 669
 670         if (table_id >= p->num_tables) {
 671                 RTE_LOG(ERR, PIPELINE,
 672                         "%s: table_id %d out of range\n", __func__, table_id);
 673                 return -EINVAL;
 674         }
 675
 676         table = &p->tables[table_id];
 677
 678         if (table->ops.f_delete_bulk == NULL) {
 679                 RTE_LOG(ERR, PIPELINE,
 680                         "%s: f_delete function pointer NULL\n", __func__);
 681                 return -EINVAL;
 682         }
 683
 684         return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
 685                         (void **) entries);
 686 }
 687
 688 /*
 689  * Port
 690  *
 691  */
 692 static int
 693 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
 694                 struct rte_pipeline_port_in_params *params,
 695                 uint32_t *port_id)
 696 {
 697         if (p == NULL) {
 698                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 699                         __func__);
 700                 return -EINVAL;
 701         }
 702         if (params == NULL) {
 703                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 704                 return -EINVAL;
 705         }
 706         if (port_id == NULL) {
 707                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 708                         __func__);
 709                 return -EINVAL;
 710         }
 711
 712         /* ops */
 713         if (params->ops == NULL) {
 714                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 715                         __func__);
 716                 return -EINVAL;
 717         }
 718
 719         if (params->ops->f_create == NULL) {
 720                 RTE_LOG(ERR, PIPELINE,
 721                         "%s: f_create function pointer NULL\n", __func__);
 722                 return -EINVAL;
 723         }
 724
 725         if (params->ops->f_rx == NULL) {
 726                 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
 727                         __func__);
 728                 return -EINVAL;
 729         }
 730
 731         /* burst_size */
 732         if ((params->burst_size == 0) ||
 733                 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
 734                 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
 735                         __func__);
 736                 return -EINVAL;
 737         }
 738
 739         /* Do we have room for one more port? */
 740         if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
 741                 RTE_LOG(ERR, PIPELINE,
 742                         "%s: invalid value for num_ports_in\n", __func__);
 743                 return -EINVAL;
 744         }
 745
 746         return 0;
 747 }
 748
 749 static int
 750 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
 751                 struct rte_pipeline_port_out_params *params,
 752                 uint32_t *port_id)
 753 {
 754         if (p == NULL) {
 755                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 756                         __func__);
 757                 return -EINVAL;
 758         }
 759
 760         if (params == NULL) {
 761                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 762                 return -EINVAL;
 763         }
 764
 765         if (port_id == NULL) {
 766                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 767                         __func__);
 768                 return -EINVAL;
 769         }
 770
 771         /* ops */
 772         if (params->ops == NULL) {
 773                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 774                         __func__);
 775                 return -EINVAL;
 776         }
 777
 778         if (params->ops->f_create == NULL) {
 779                 RTE_LOG(ERR, PIPELINE,
 780                         "%s: f_create function pointer NULL\n", __func__);
 781                 return -EINVAL;
 782         }
 783
 784         if (params->ops->f_tx == NULL) {
 785                 RTE_LOG(ERR, PIPELINE,
 786                         "%s: f_tx function pointer NULL\n", __func__);
 787                 return -EINVAL;
 788         }
 789
 790         if (params->ops->f_tx_bulk == NULL) {
 791                 RTE_LOG(ERR, PIPELINE,
 792                         "%s: f_tx_bulk function pointer NULL\n", __func__);
 793                 return -EINVAL;
 794         }
 795
 796         /* Do we have room for one more port? */
 797         if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
 798                 RTE_LOG(ERR, PIPELINE,
 799                         "%s: invalid value for num_ports_out\n", __func__);
 800                 return -EINVAL;
 801         }
 802
 803         return 0;
 804 }
 805
 806 int
 807 rte_pipeline_port_in_create(struct rte_pipeline *p,
 808                 struct rte_pipeline_port_in_params *params,
 809                 uint32_t *port_id)
 810 {
 811         struct rte_port_in *port;
 812         void *h_port;
 813         uint32_t id;
 814         int status;
 815
 816         /* Check input arguments */
 817         status = rte_pipeline_port_in_check_params(p, params, port_id);
 818         if (status != 0)
 819                 return status;
 820
 821         id = p->num_ports_in;
 822         port = &p->ports_in[id];
 823
 824         /* Create the port */
 825         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 826         if (h_port == NULL) {
 827                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 828                 return -EINVAL;
 829         }
 830
 831         /* Commit current table to the pipeline */
 832         p->num_ports_in++;
 833         *port_id = id;
 834
 835         /* Save input parameters */
 836         memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
 837         port->f_action = params->f_action;
 838         port->arg_ah = params->arg_ah;
 839         port->burst_size = params->burst_size;
 840
 841         /* Initialize port internal data structure */
 842         port->table_id = RTE_TABLE_INVALID;
 843         port->h_port = h_port;
 844         port->next = NULL;
 845
 846         return 0;
 847 }
 848
 849 void
 850 rte_pipeline_port_in_free(struct rte_port_in *port)
 851 {
 852         if (port->ops.f_free != NULL)
 853                 port->ops.f_free(port->h_port);
 854 }
 855
 856 int
 857 rte_pipeline_port_out_create(struct rte_pipeline *p,
 858                 struct rte_pipeline_port_out_params *params,
 859                 uint32_t *port_id)
 860 {
 861         struct rte_port_out *port;
 862         void *h_port;
 863         uint32_t id;
 864         int status;
 865
 866         /* Check input arguments */
 867         status = rte_pipeline_port_out_check_params(p, params, port_id);
 868         if (status != 0)
 869                 return status;
 870
 871         id = p->num_ports_out;
 872         port = &p->ports_out[id];
 873
 874         /* Create the port */
 875         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 876         if (h_port == NULL) {
 877                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 878                 return -EINVAL;
 879         }
 880
 881         /* Commit current table to the pipeline */
 882         p->num_ports_out++;
 883         *port_id = id;
 884
 885         /* Save input parameters */
 886         memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
 887         port->f_action = params->f_action;
 888         port->arg_ah = params->arg_ah;
 889
 890         /* Initialize port internal data structure */
 891         port->h_port = h_port;
 892
 893         return 0;
 894 }
 895
 896 void
 897 rte_pipeline_port_out_free(struct rte_port_out *port)
 898 {
 899         if (port->ops.f_free != NULL)
 900                 port->ops.f_free(port->h_port);
 901 }
 902
 903 int
 904 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
 905                 uint32_t port_id,
 906                 uint32_t table_id)
 907 {
 908         struct rte_port_in *port;
 909
 910         /* Check input arguments */
 911         if (p == NULL) {
 912                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 913                         __func__);
 914                 return -EINVAL;
 915         }
 916
 917         if (port_id >= p->num_ports_in) {
 918                 RTE_LOG(ERR, PIPELINE,
 919                         "%s: port IN ID %u is out of range\n",
 920                         __func__, port_id);
 921                 return -EINVAL;
 922         }
 923
 924         if (table_id >= p->num_tables) {
 925                 RTE_LOG(ERR, PIPELINE,
 926                         "%s: Table ID %u is out of range\n",
 927                         __func__, table_id);
 928                 return -EINVAL;
 929         }
 930
 931         port = &p->ports_in[port_id];
 932         port->table_id = table_id;
 933
 934         return 0;
 935 }
 936
 937 int
 938 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
 939 {
 940         struct rte_port_in *port, *port_prev, *port_next;
 941         uint64_t port_mask;
 942         uint32_t port_prev_id, port_next_id;
 943
 944         /* Check input arguments */
 945         if (p == NULL) {
 946                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 947                         __func__);
 948                 return -EINVAL;
 949         }
 950
 951         if (port_id >= p->num_ports_in) {
 952                 RTE_LOG(ERR, PIPELINE,
 953                         "%s: port IN ID %u is out of range\n",
 954                         __func__, port_id);
 955                 return -EINVAL;
 956         }
 957
 958         port = &p->ports_in[port_id];
 959
 960         /* Return if current input port is already enabled */
 961         port_mask = 1LLU << port_id;
 962         if (p->enabled_port_in_mask & port_mask)
 963                 return 0;
 964
 965         p->enabled_port_in_mask |= port_mask;
 966
 967         /* Add current input port to the pipeline chain of enabled ports */
 968         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 969         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
 970
 971         port_prev = &p->ports_in[port_prev_id];
 972         port_next = &p->ports_in[port_next_id];
 973
 974         port_prev->next = port;
 975         port->next = port_next;
 976
 977         /* Check if list of enabled ports was previously empty */
 978         if (p->enabled_port_in_mask == port_mask)
 979                 p->port_in_next = port;
 980
 981         return 0;
 982 }
 983
 984 int
 985 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
 986 {
 987         struct rte_port_in *port, *port_prev, *port_next;
 988         uint64_t port_mask;
 989         uint32_t port_prev_id, port_next_id;
 990
 991         /* Check input arguments */
 992         if (p == NULL) {
 993                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 994                 __func__);
 995                 return -EINVAL;
 996         }
 997
 998         if (port_id >= p->num_ports_in) {
 999                 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
1000                         __func__, port_id);
1001                 return -EINVAL;
1002         }
1003
1004         port = &p->ports_in[port_id];
1005
1006         /* Return if current input port is already disabled */
1007         port_mask = 1LLU << port_id;
1008         if ((p->enabled_port_in_mask & port_mask) == 0)
1009                 return 0;
1010
1011         p->enabled_port_in_mask &= ~port_mask;
1012
1013         /* Return if no other enabled ports */
1014         if (p->enabled_port_in_mask == 0) {
1015                 p->port_in_next = NULL;
1016
1017                 return 0;
1018         }
1019
1020         /* Add current input port to the pipeline chain of enabled ports */
1021         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1022         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1023
1024         port_prev = &p->ports_in[port_prev_id];
1025         port_next = &p->ports_in[port_next_id];
1026
1027         port_prev->next = port_next;
1028
1029         /* Check if the port which has just been disabled is next to serve */
1030         if (port == p->port_in_next)
1031                 p->port_in_next = port_next;
1032
1033         return 0;
1034 }
1035
1036 /*
1037  * Pipeline run-time
1038  *
1039  */
1040 int
1041 rte_pipeline_check(struct rte_pipeline *p)
1042 {
1043         uint32_t port_in_id;
1044
1045         /* Check input arguments */
1046         if (p == NULL) {
1047                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1048                         __func__);
1049                 return -EINVAL;
1050         }
1051
1052         /* Check that pipeline has at least one input port, one table and one
1053         output port */
1054         if (p->num_ports_in == 0) {
1055                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1056                         __func__);
1057                 return -EINVAL;
1058         }
1059         if (p->num_tables == 0) {
1060                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1061                         __func__);
1062                 return -EINVAL;
1063         }
1064         if (p->num_ports_out == 0) {
1065                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1066                         __func__);
1067                 return -EINVAL;
1068         }
1069
1070         /* Check that all input ports are connected */
1071         for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1072                 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1073
1074                 if (port_in->table_id == RTE_TABLE_INVALID) {
1075                         RTE_LOG(ERR, PIPELINE,
1076                                 "%s: Port IN ID %u is not connected\n",
1077                                 __func__, port_in_id);
1078                         return -EINVAL;
1079                 }
1080         }
1081
1082         return 0;
1083 }
1084
1085 static inline void
1086 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1087 {
1088         p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1089         p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1090         p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1091         p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1092
1093         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1094                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1095                 uint32_t i;
1096
1097                 for (i = 0; i < n_pkts; i++) {
1098                         uint64_t pkt_mask = 1LLU << i;
1099                         uint32_t pos = p->entries[i]->action;
1100
1101                         p->action_mask1[pos] |= pkt_mask;
1102                 }
1103         } else {
1104                 uint32_t i;
1105
1106                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1107                         uint64_t pkt_mask = 1LLU << i;
1108                         uint32_t pos;
1109
1110                         if ((pkt_mask & pkts_mask) == 0)
1111                                 continue;
1112
1113                         pos = p->entries[i]->action;
1114                         p->action_mask1[pos] |= pkt_mask;
1115                 }
1116         }
1117 }
1118
1119 static inline void
1120 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1121         uint64_t pkts_mask, uint32_t port_id)
1122 {
1123         struct rte_port_out *port_out = &p->ports_out[port_id];
1124
1125         p->pkts_mask = pkts_mask;
1126
1127         /* Output port user actions */
1128         if (port_out->f_action != NULL) {
1129                 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1130
1131                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1132                         port_out->n_pkts_dropped_by_ah);
1133         }
1134
1135         /* Output port TX */
1136         if (p->pkts_mask != 0)
1137                 port_out->ops.f_tx_bulk(port_out->h_port,
1138                         p->pkts,
1139                         p->pkts_mask);
1140 }
1141
1142 static inline void
1143 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1144 {
1145         p->pkts_mask = pkts_mask;
1146
1147         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1148                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1149                 uint32_t i;
1150
1151                 for (i = 0; i < n_pkts; i++) {
1152                         struct rte_mbuf *pkt = p->pkts[i];
1153                         uint32_t port_out_id = p->entries[i]->port_id;
1154                         struct rte_port_out *port_out =
1155                                 &p->ports_out[port_out_id];
1156
1157                         /* Output port user actions */
1158                         if (port_out->f_action == NULL) /* Output port TX */
1159                                 port_out->ops.f_tx(port_out->h_port, pkt);
1160                         else {
1161                                 uint64_t pkt_mask = 1LLU << i;
1162
1163                                 port_out->f_action(p,
1164                                         p->pkts,
1165                                         pkt_mask,
1166                                         port_out->arg_ah);
1167
1168                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1169                                         port_out->n_pkts_dropped_by_ah);
1170
1171                                 /* Output port TX */
1172                                 if (pkt_mask & p->pkts_mask)
1173                                         port_out->ops.f_tx(port_out->h_port,
1174                                                 pkt);
1175                         }
1176                 }
1177         } else {
1178                 uint32_t i;
1179
1180                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1181                         uint64_t pkt_mask = 1LLU << i;
1182                         struct rte_mbuf *pkt;
1183                         struct rte_port_out *port_out;
1184                         uint32_t port_out_id;
1185
1186                         if ((pkt_mask & pkts_mask) == 0)
1187                                 continue;
1188
1189                         pkt = p->pkts[i];
1190                         port_out_id = p->entries[i]->port_id;
1191                         port_out = &p->ports_out[port_out_id];
1192
1193                         /* Output port user actions */
1194                         if (port_out->f_action == NULL) /* Output port TX */
1195                                 port_out->ops.f_tx(port_out->h_port, pkt);
1196                         else {
1197                                 port_out->f_action(p,
1198                                         p->pkts,
1199                                         pkt_mask,
1200                                         port_out->arg_ah);
1201
1202                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1203                                         port_out->n_pkts_dropped_by_ah);
1204
1205                                 /* Output port TX */
1206                                 if (pkt_mask & p->pkts_mask)
1207                                         port_out->ops.f_tx(port_out->h_port,
1208                                                 pkt);
1209                         }
1210                 }
1211         }
1212 }
1213
1214 static inline void
1215 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1216         uint64_t pkts_mask)
1217 {
1218         p->pkts_mask = pkts_mask;
1219
1220         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1221                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1222                 uint32_t i;
1223
1224                 for (i = 0; i < n_pkts; i++) {
1225                         struct rte_mbuf *pkt = p->pkts[i];
1226                         uint32_t port_out_id =
1227                                 RTE_MBUF_METADATA_UINT32(pkt,
1228                                         p->offset_port_id);
1229                         struct rte_port_out *port_out = &p->ports_out[
1230                                 port_out_id];
1231
1232                         /* Output port user actions */
1233                         if (port_out->f_action == NULL) /* Output port TX */
1234                                 port_out->ops.f_tx(port_out->h_port, pkt);
1235                         else {
1236                                 uint64_t pkt_mask = 1LLU << i;
1237
1238                                 port_out->f_action(p,
1239                                         p->pkts,
1240                                         pkt_mask,
1241                                         port_out->arg_ah);
1242
1243                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1244                                         port_out->n_pkts_dropped_by_ah);
1245
1246                                 /* Output port TX */
1247                                 if (pkt_mask & p->pkts_mask)
1248                                         port_out->ops.f_tx(port_out->h_port,
1249                                                 pkt);
1250                         }
1251                 }
1252         } else {
1253                 uint32_t i;
1254
1255                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1256                         uint64_t pkt_mask = 1LLU << i;
1257                         struct rte_mbuf *pkt;
1258                         struct rte_port_out *port_out;
1259                         uint32_t port_out_id;
1260
1261                         if ((pkt_mask & pkts_mask) == 0)
1262                                 continue;
1263
1264                         pkt = p->pkts[i];
1265                         port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1266                                 p->offset_port_id);
1267                         port_out = &p->ports_out[port_out_id];
1268
1269                         /* Output port user actions */
1270                         if (port_out->f_action == NULL) /* Output port TX */
1271                                 port_out->ops.f_tx(port_out->h_port, pkt);
1272                         else {
1273                                 port_out->f_action(p,
1274                                         p->pkts,
1275                                         pkt_mask,
1276                                         port_out->arg_ah);
1277
1278                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1279                                         port_out->n_pkts_dropped_by_ah);
1280
1281                                 /* Output port TX */
1282                                 if (pkt_mask & p->pkts_mask)
1283                                         port_out->ops.f_tx(port_out->h_port,
1284                                                 pkt);
1285                         }
1286                 }
1287         }
1288 }
1289
1290 static inline void
1291 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1292 {
1293         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1294                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1295                 uint32_t i;
1296
1297                 for (i = 0; i < n_pkts; i++)
1298                         rte_pktmbuf_free(p->pkts[i]);
1299         } else {
1300                 uint32_t i;
1301
1302                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1303                         uint64_t pkt_mask = 1LLU << i;
1304
1305                         if ((pkt_mask & pkts_mask) == 0)
1306                                 continue;
1307
1308                         rte_pktmbuf_free(p->pkts[i]);
1309                 }
1310         }
1311 }
1312
1313 int
1314 rte_pipeline_run(struct rte_pipeline *p)
1315 {
1316         struct rte_port_in *port_in = p->port_in_next;
1317         uint32_t n_pkts, table_id;
1318
1319         if (port_in == NULL)
1320                 return 0;
1321
1322         /* Input port RX */
1323         n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1324                 port_in->burst_size);
1325         if (n_pkts == 0) {
1326                 p->port_in_next = port_in->next;
1327                 return 0;
1328         }
1329
1330         p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1331         p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1332         p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1333         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1334         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1335
1336         /* Input port user actions */
1337         if (port_in->f_action != NULL) {
1338                 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1339
1340                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1341                         port_in->n_pkts_dropped_by_ah);
1342         }
1343
1344         /* Table */
1345         for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1346                 struct rte_table *table;
1347                 uint64_t lookup_hit_mask, lookup_miss_mask;
1348
1349                 /* Lookup */
1350                 table = &p->tables[table_id];
1351                 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1352                         &lookup_hit_mask, (void **) p->entries);
1353                 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1354
1355                 /* Lookup miss */
1356                 if (lookup_miss_mask != 0) {
1357                         struct rte_pipeline_table_entry *default_entry =
1358                                 table->default_entry;
1359
1360                         p->pkts_mask = lookup_miss_mask;
1361
1362                         /* Table user actions */
1363                         if (table->f_action_miss != NULL) {
1364                                 table->f_action_miss(p,
1365                                         p->pkts,
1366                                         lookup_miss_mask,
1367                                         default_entry,
1368                                         table->arg_ah);
1369
1370                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1371                                         table->n_pkts_dropped_by_lkp_miss_ah);
1372                         }
1373
1374                         /* Table reserved actions */
1375                         if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1376                                 (p->pkts_mask != 0))
1377                                 rte_pipeline_action_handler_port_bulk(p,
1378                                         p->pkts_mask,
1379                                         default_entry->port_id);
1380                         else {
1381                                 uint32_t pos = default_entry->action;
1382
1383                                 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1384
1385                                 p->action_mask0[pos] |= p->pkts_mask;
1386
1387                                 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1388                                         table->n_pkts_dropped_lkp_miss);
1389                         }
1390                 }
1391
1392                 /* Lookup hit */
1393                 if (lookup_hit_mask != 0) {
1394                         p->pkts_mask = lookup_hit_mask;
1395
1396                         /* Table user actions */
1397                         if (table->f_action_hit != NULL) {
1398                                 table->f_action_hit(p,
1399                                         p->pkts,
1400                                         lookup_hit_mask,
1401                                         p->entries,
1402                                         table->arg_ah);
1403
1404                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1405                                         table->n_pkts_dropped_by_lkp_hit_ah);
1406                         }
1407
1408                         /* Table reserved actions */
1409                         RTE_PIPELINE_STATS_TABLE_DROP0(p);
1410                         rte_pipeline_compute_masks(p, p->pkts_mask);
1411                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1412                                 p->action_mask1[
1413                                         RTE_PIPELINE_ACTION_DROP];
1414                         p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1415                                 p->action_mask1[
1416                                         RTE_PIPELINE_ACTION_PORT];
1417                         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1418                                 p->action_mask1[
1419                                         RTE_PIPELINE_ACTION_PORT_META];
1420                         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1421                                 p->action_mask1[
1422                                         RTE_PIPELINE_ACTION_TABLE];
1423
1424                         RTE_PIPELINE_STATS_TABLE_DROP1(p,
1425                                 table->n_pkts_dropped_lkp_hit);
1426                 }
1427
1428                 /* Prepare for next iteration */
1429                 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1430                 table_id = table->table_next_id;
1431                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1432         }
1433
1434         /* Table reserved action PORT */
1435         rte_pipeline_action_handler_port(p,
1436                 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1437
1438         /* Table reserved action PORT META */
1439         rte_pipeline_action_handler_port_meta(p,
1440                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1441
1442         /* Table reserved action DROP */
1443         rte_pipeline_action_handler_drop(p,
1444                 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1445
1446         /* Pick candidate for next port IN to serve */
1447         p->port_in_next = port_in->next;
1448
1449         return (int) n_pkts;
1450 }
1451
1452 int
1453 rte_pipeline_flush(struct rte_pipeline *p)
1454 {
1455         uint32_t port_id;
1456
1457         /* Check input arguments */
1458         if (p == NULL) {
1459                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1460                         __func__);
1461                 return -EINVAL;
1462         }
1463
1464         for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1465                 struct rte_port_out *port = &p->ports_out[port_id];
1466
1467                 if (port->ops.f_flush != NULL)
1468                         port->ops.f_flush(port->h_port);
1469         }
1470
1471         return 0;
1472 }
1473
1474 int
1475 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1476         uint32_t port_id, struct rte_mbuf *pkt)
1477 {
1478         struct rte_port_out *port_out = &p->ports_out[port_id];
1479
1480         port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1481
1482         return 0;
1483 }
1484
1485 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1486         uint64_t pkts_mask)
1487 {
1488         pkts_mask &= p->pkts_mask;
1489         p->pkts_mask &= ~pkts_mask;
1490
1491         return 0;
1492 }
1493
1494 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1495         uint64_t pkts_mask)
1496 {
1497         pkts_mask &= p->pkts_mask;
1498         p->pkts_mask &= ~pkts_mask;
1499         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1500
1501         RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1502         return 0;
1503 }
1504
1505 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1506         struct rte_pipeline_port_in_stats *stats, int clear)
1507 {
1508         struct rte_port_in *port;
1509         int retval;
1510
1511         if (p == NULL) {
1512                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1513                         __func__);
1514                 return -EINVAL;
1515         }
1516
1517         if (port_id >= p->num_ports_in) {
1518                 RTE_LOG(ERR, PIPELINE,
1519                         "%s: port IN ID %u is out of range\n",
1520                         __func__, port_id);
1521                 return -EINVAL;
1522         }
1523
1524         port = &p->ports_in[port_id];
1525
1526         if (port->ops.f_stats != NULL) {
1527                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1528                 if (retval)
1529                         return retval;
1530         } else if (stats != NULL)
1531                 memset(&stats->stats, 0, sizeof(stats->stats));
1532
1533         if (stats != NULL)
1534                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1535
1536         if (clear != 0)
1537                 port->n_pkts_dropped_by_ah = 0;
1538
1539         return 0;
1540 }
1541
1542 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1543         struct rte_pipeline_port_out_stats *stats, int clear)
1544 {
1545         struct rte_port_out *port;
1546         int retval;
1547
1548         if (p == NULL) {
1549                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1550                 return -EINVAL;
1551         }
1552
1553         if (port_id >= p->num_ports_out) {
1554                 RTE_LOG(ERR, PIPELINE,
1555                         "%s: port OUT ID %u is out of range\n", __func__, port_id);
1556                 return -EINVAL;
1557         }
1558
1559         port = &p->ports_out[port_id];
1560         if (port->ops.f_stats != NULL) {
1561                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1562                 if (retval != 0)
1563                         return retval;
1564         } else if (stats != NULL)
1565                 memset(&stats->stats, 0, sizeof(stats->stats));
1566
1567         if (stats != NULL)
1568                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1569
1570         if (clear != 0)
1571                 port->n_pkts_dropped_by_ah = 0;
1572
1573         return 0;
1574 }
1575
1576 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1577         struct rte_pipeline_table_stats *stats, int clear)
1578 {
1579         struct rte_table *table;
1580         int retval;
1581
1582         if (p == NULL) {
1583                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1584                         __func__);
1585                 return -EINVAL;
1586         }
1587
1588         if (table_id >= p->num_tables) {
1589                 RTE_LOG(ERR, PIPELINE,
1590                                 "%s: table %u is out of range\n", __func__, table_id);
1591                 return -EINVAL;
1592         }
1593
1594         table = &p->tables[table_id];
1595         if (table->ops.f_stats != NULL) {
1596                 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1597                 if (retval != 0)
1598                         return retval;
1599         } else if (stats != NULL)
1600                 memset(&stats->stats, 0, sizeof(stats->stats));
1601
1602         if (stats != NULL) {
1603                 stats->n_pkts_dropped_by_lkp_hit_ah =
1604                         table->n_pkts_dropped_by_lkp_hit_ah;
1605                 stats->n_pkts_dropped_by_lkp_miss_ah =
1606                         table->n_pkts_dropped_by_lkp_miss_ah;
1607                 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1608                 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1609         }
1610
1611         if (clear != 0) {
1612                 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1613                 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1614                 table->n_pkts_dropped_lkp_hit = 0;
1615                 table->n_pkts_dropped_lkp_miss = 0;
1616         }
1617
1618         return 0;
1619 }