lib/librte_pipeline/rte_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6 #include <stdio.h>
   7
   8 #include <rte_common.h>
   9 #include <rte_memory.h>
  10 #include <rte_cycles.h>
  11 #include <rte_prefetch.h>
  12 #include <rte_branch_prediction.h>
  13 #include <rte_mbuf.h>
  14 #include <rte_malloc.h>
  15 #include <rte_string_fns.h>
  16
  17 #include "rte_pipeline.h"
  18
  19 #define RTE_TABLE_INVALID                                 UINT32_MAX
  20
  21 #ifdef RTE_PIPELINE_STATS_COLLECT
  22
  23 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)                       \
  24         ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); })
  25
  26 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)                     \
  27         ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
  28
  29 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)                               \
  30         ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
  31
  32 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)                      \
  33 ({                                                                      \
  34         uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP];    \
  35         mask ^= (p)->pkts_drop_mask;                                    \
  36         (counter) += __builtin_popcountll(mask);                        \
  37 })
  38
  39 #else
  40
  41 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
  42 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
  43 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)
  44 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
  45
  46 #endif
  47
  48 struct rte_port_in {
  49         /* Input parameters */
  50         struct rte_port_in_ops ops;
  51         rte_pipeline_port_in_action_handler f_action;
  52         void *arg_ah;
  53         uint32_t burst_size;
  54
  55         /* The table to which this port is connected */
  56         uint32_t table_id;
  57
  58         /* Handle to low-level port */
  59         void *h_port;
  60
  61         /* List of enabled ports */
  62         struct rte_port_in *next;
  63
  64         /* Statistics */
  65         uint64_t n_pkts_dropped_by_ah;
  66 };
  67
  68 struct rte_port_out {
  69         /* Input parameters */
  70         struct rte_port_out_ops ops;
  71         rte_pipeline_port_out_action_handler f_action;
  72         void *arg_ah;
  73
  74         /* Handle to low-level port */
  75         void *h_port;
  76
  77         /* Statistics */
  78         uint64_t n_pkts_dropped_by_ah;
  79 };
  80
  81 struct rte_table {
  82         /* Input parameters */
  83         struct rte_table_ops ops;
  84         rte_pipeline_table_action_handler_hit f_action_hit;
  85         rte_pipeline_table_action_handler_miss f_action_miss;
  86         void *arg_ah;
  87         struct rte_pipeline_table_entry *default_entry;
  88         uint32_t entry_size;
  89
  90         uint32_t table_next_id;
  91         uint32_t table_next_id_valid;
  92
  93         /* Handle to the low-level table object */
  94         void *h_table;
  95
  96         /* Statistics */
  97         uint64_t n_pkts_dropped_by_lkp_hit_ah;
  98         uint64_t n_pkts_dropped_by_lkp_miss_ah;
  99         uint64_t n_pkts_dropped_lkp_hit;
 100         uint64_t n_pkts_dropped_lkp_miss;
 101 };
 102
 103 #define RTE_PIPELINE_MAX_NAME_SZ                           124
 104
 105 struct rte_pipeline {
 106         /* Input parameters */
 107         char name[RTE_PIPELINE_MAX_NAME_SZ];
 108         int socket_id;
 109         uint32_t offset_port_id;
 110
 111         /* Internal tables */
 112         struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
 113         struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
 114         struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
 115
 116         /* Occupancy of internal tables */
 117         uint32_t num_ports_in;
 118         uint32_t num_ports_out;
 119         uint32_t num_tables;
 120
 121         /* List of enabled ports */
 122         uint64_t enabled_port_in_mask;
 123         struct rte_port_in *port_in_next;
 124
 125         /* Pipeline run structures */
 126         struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
 127         struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
 128         uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
 129         uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
 130         uint64_t pkts_mask;
 131         uint64_t n_pkts_ah_drop;
 132         uint64_t pkts_drop_mask;
 133 } __rte_cache_aligned;
 134
 135 static inline uint32_t
 136 rte_mask_get_next(uint64_t mask, uint32_t pos)
 137 {
 138         uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
 139                         (mask >> ((pos + 1) & 0x3F));
 140         return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
 141 }
 142
 143 static inline uint32_t
 144 rte_mask_get_prev(uint64_t mask, uint32_t pos)
 145 {
 146         uint64_t mask_rot = (mask >> (pos & 0x3F)) |
 147                         (mask << ((64 - pos) & 0x3F));
 148         return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
 149 }
 150
 151 static void
 152 rte_pipeline_table_free(struct rte_table *table);
 153
 154 static void
 155 rte_pipeline_port_in_free(struct rte_port_in *port);
 156
 157 static void
 158 rte_pipeline_port_out_free(struct rte_port_out *port);
 159
 160 /*
 161  * Pipeline
 162  *
 163  */
 164 static int
 165 rte_pipeline_check_params(struct rte_pipeline_params *params)
 166 {
 167         if (params == NULL) {
 168                 RTE_LOG(ERR, PIPELINE,
 169                         "%s: Incorrect value for parameter params\n", __func__);
 170                 return -EINVAL;
 171         }
 172
 173         /* name */
 174         if (params->name == NULL) {
 175                 RTE_LOG(ERR, PIPELINE,
 176                         "%s: Incorrect value for parameter name\n", __func__);
 177                 return -EINVAL;
 178         }
 179
 180         /* socket */
 181         if (params->socket_id < 0) {
 182                 RTE_LOG(ERR, PIPELINE,
 183                         "%s: Incorrect value for parameter socket_id\n",
 184                         __func__);
 185                 return -EINVAL;
 186         }
 187
 188         return 0;
 189 }
 190
 191 struct rte_pipeline *
 192 rte_pipeline_create(struct rte_pipeline_params *params)
 193 {
 194         struct rte_pipeline *p;
 195         int status;
 196
 197         /* Check input parameters */
 198         status = rte_pipeline_check_params(params);
 199         if (status != 0) {
 200                 RTE_LOG(ERR, PIPELINE,
 201                         "%s: Pipeline params check failed (%d)\n",
 202                         __func__, status);
 203                 return NULL;
 204         }
 205
 206         /* Allocate memory for the pipeline on requested socket */
 207         p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
 208                         RTE_CACHE_LINE_SIZE, params->socket_id);
 209
 210         if (p == NULL) {
 211                 RTE_LOG(ERR, PIPELINE,
 212                         "%s: Pipeline memory allocation failed\n", __func__);
 213                 return NULL;
 214         }
 215
 216         /* Save input parameters */
 217         snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
 218         p->socket_id = params->socket_id;
 219         p->offset_port_id = params->offset_port_id;
 220
 221         /* Initialize pipeline internal data structure */
 222         p->num_ports_in = 0;
 223         p->num_ports_out = 0;
 224         p->num_tables = 0;
 225         p->enabled_port_in_mask = 0;
 226         p->port_in_next = NULL;
 227         p->pkts_mask = 0;
 228         p->n_pkts_ah_drop = 0;
 229
 230         return p;
 231 }
 232
 233 int
 234 rte_pipeline_free(struct rte_pipeline *p)
 235 {
 236         uint32_t i;
 237
 238         /* Check input parameters */
 239         if (p == NULL) {
 240                 RTE_LOG(ERR, PIPELINE,
 241                         "%s: rte_pipeline parameter is NULL\n", __func__);
 242                 return -EINVAL;
 243         }
 244
 245         /* Free input ports */
 246         for (i = 0; i < p->num_ports_in; i++) {
 247                 struct rte_port_in *port = &p->ports_in[i];
 248
 249                 rte_pipeline_port_in_free(port);
 250         }
 251
 252         /* Free tables */
 253         for (i = 0; i < p->num_tables; i++) {
 254                 struct rte_table *table = &p->tables[i];
 255
 256                 rte_pipeline_table_free(table);
 257         }
 258
 259         /* Free output ports */
 260         for (i = 0; i < p->num_ports_out; i++) {
 261                 struct rte_port_out *port = &p->ports_out[i];
 262
 263                 rte_pipeline_port_out_free(port);
 264         }
 265
 266         /* Free pipeline memory */
 267         rte_free(p);
 268
 269         return 0;
 270 }
 271
 272 /*
 273  * Table
 274  *
 275  */
 276 static int
 277 rte_table_check_params(struct rte_pipeline *p,
 278                 struct rte_pipeline_table_params *params,
 279                 uint32_t *table_id)
 280 {
 281         if (p == NULL) {
 282                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 283                         __func__);
 284                 return -EINVAL;
 285         }
 286         if (params == NULL) {
 287                 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
 288                         __func__);
 289                 return -EINVAL;
 290         }
 291         if (table_id == NULL) {
 292                 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
 293                         __func__);
 294                 return -EINVAL;
 295         }
 296
 297         /* ops */
 298         if (params->ops == NULL) {
 299                 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
 300                         __func__);
 301                 return -EINVAL;
 302         }
 303
 304         if (params->ops->f_create == NULL) {
 305                 RTE_LOG(ERR, PIPELINE,
 306                         "%s: f_create function pointer is NULL\n", __func__);
 307                 return -EINVAL;
 308         }
 309
 310         if (params->ops->f_lookup == NULL) {
 311                 RTE_LOG(ERR, PIPELINE,
 312                         "%s: f_lookup function pointer is NULL\n", __func__);
 313                 return -EINVAL;
 314         }
 315
 316         /* De we have room for one more table? */
 317         if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
 318                 RTE_LOG(ERR, PIPELINE,
 319                         "%s: Incorrect value for num_tables parameter\n",
 320                         __func__);
 321                 return -EINVAL;
 322         }
 323
 324         return 0;
 325 }
 326
 327 int
 328 rte_pipeline_table_create(struct rte_pipeline *p,
 329                 struct rte_pipeline_table_params *params,
 330                 uint32_t *table_id)
 331 {
 332         struct rte_table *table;
 333         struct rte_pipeline_table_entry *default_entry;
 334         void *h_table;
 335         uint32_t entry_size, id;
 336         int status;
 337
 338         /* Check input arguments */
 339         status = rte_table_check_params(p, params, table_id);
 340         if (status != 0)
 341                 return status;
 342
 343         id = p->num_tables;
 344         table = &p->tables[id];
 345
 346         /* Allocate space for the default table entry */
 347         entry_size = sizeof(struct rte_pipeline_table_entry) +
 348                 params->action_data_size;
 349         default_entry = rte_zmalloc_socket(
 350                 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
 351         if (default_entry == NULL) {
 352                 RTE_LOG(ERR, PIPELINE,
 353                         "%s: Failed to allocate default entry\n", __func__);
 354                 return -EINVAL;
 355         }
 356
 357         /* Create the table */
 358         h_table = params->ops->f_create(params->arg_create, p->socket_id,
 359                 entry_size);
 360         if (h_table == NULL) {
 361                 rte_free(default_entry);
 362                 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
 363                 return -EINVAL;
 364         }
 365
 366         /* Commit current table to the pipeline */
 367         p->num_tables++;
 368         *table_id = id;
 369
 370         /* Save input parameters */
 371         memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
 372         table->f_action_hit = params->f_action_hit;
 373         table->f_action_miss = params->f_action_miss;
 374         table->arg_ah = params->arg_ah;
 375         table->entry_size = entry_size;
 376
 377         /* Clear the lookup miss actions (to be set later through API) */
 378         table->default_entry = default_entry;
 379         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 380
 381         /* Initialize table internal data structure */
 382         table->h_table = h_table;
 383         table->table_next_id = 0;
 384         table->table_next_id_valid = 0;
 385
 386         return 0;
 387 }
 388
 389 void
 390 rte_pipeline_table_free(struct rte_table *table)
 391 {
 392         if (table->ops.f_free != NULL)
 393                 table->ops.f_free(table->h_table);
 394
 395         rte_free(table->default_entry);
 396 }
 397
 398 int
 399 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
 400         uint32_t table_id,
 401         struct rte_pipeline_table_entry *default_entry,
 402         struct rte_pipeline_table_entry **default_entry_ptr)
 403 {
 404         struct rte_table *table;
 405
 406         /* Check input arguments */
 407         if (p == NULL) {
 408                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 409                         __func__);
 410                 return -EINVAL;
 411         }
 412
 413         if (default_entry == NULL) {
 414                 RTE_LOG(ERR, PIPELINE,
 415                         "%s: default_entry parameter is NULL\n", __func__);
 416                 return -EINVAL;
 417         }
 418
 419         if (table_id >= p->num_tables) {
 420                 RTE_LOG(ERR, PIPELINE,
 421                         "%s: table_id %d out of range\n", __func__, table_id);
 422                 return -EINVAL;
 423         }
 424
 425         table = &p->tables[table_id];
 426
 427         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 428                 table->table_next_id_valid &&
 429                 (default_entry->table_id != table->table_next_id)) {
 430                 RTE_LOG(ERR, PIPELINE,
 431                         "%s: Tree-like topologies not allowed\n", __func__);
 432                 return -EINVAL;
 433         }
 434
 435         /* Set the lookup miss actions */
 436         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 437                 (table->table_next_id_valid == 0)) {
 438                 table->table_next_id = default_entry->table_id;
 439                 table->table_next_id_valid = 1;
 440         }
 441
 442         memcpy(table->default_entry, default_entry, table->entry_size);
 443
 444         *default_entry_ptr = table->default_entry;
 445         return 0;
 446 }
 447
 448 int
 449 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
 450                 uint32_t table_id,
 451                 struct rte_pipeline_table_entry *entry)
 452 {
 453         struct rte_table *table;
 454
 455         /* Check input arguments */
 456         if (p == NULL) {
 457                 RTE_LOG(ERR, PIPELINE,
 458                         "%s: pipeline parameter is NULL\n", __func__);
 459                 return -EINVAL;
 460         }
 461
 462         if (table_id >= p->num_tables) {
 463                 RTE_LOG(ERR, PIPELINE,
 464                         "%s: table_id %d out of range\n", __func__, table_id);
 465                 return -EINVAL;
 466         }
 467
 468         table = &p->tables[table_id];
 469
 470         /* Save the current contents of the default entry */
 471         if (entry)
 472                 memcpy(entry, table->default_entry, table->entry_size);
 473
 474         /* Clear the lookup miss actions */
 475         memset(table->default_entry, 0, table->entry_size);
 476         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 477
 478         return 0;
 479 }
 480
 481 int
 482 rte_pipeline_table_entry_add(struct rte_pipeline *p,
 483                 uint32_t table_id,
 484                 void *key,
 485                 struct rte_pipeline_table_entry *entry,
 486                 int *key_found,
 487                 struct rte_pipeline_table_entry **entry_ptr)
 488 {
 489         struct rte_table *table;
 490
 491         /* Check input arguments */
 492         if (p == NULL) {
 493                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 494                         __func__);
 495                 return -EINVAL;
 496         }
 497
 498         if (key == NULL) {
 499                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
 500                 return -EINVAL;
 501         }
 502
 503         if (entry == NULL) {
 504                 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
 505                         __func__);
 506                 return -EINVAL;
 507         }
 508
 509         if (table_id >= p->num_tables) {
 510                 RTE_LOG(ERR, PIPELINE,
 511                         "%s: table_id %d out of range\n", __func__, table_id);
 512                 return -EINVAL;
 513         }
 514
 515         table = &p->tables[table_id];
 516
 517         if (table->ops.f_add == NULL) {
 518                 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
 519                         __func__);
 520                 return -EINVAL;
 521         }
 522
 523         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 524                 table->table_next_id_valid &&
 525                 (entry->table_id != table->table_next_id)) {
 526                 RTE_LOG(ERR, PIPELINE,
 527                         "%s: Tree-like topologies not allowed\n", __func__);
 528                 return -EINVAL;
 529         }
 530
 531         /* Add entry */
 532         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 533                 (table->table_next_id_valid == 0)) {
 534                 table->table_next_id = entry->table_id;
 535                 table->table_next_id_valid = 1;
 536         }
 537
 538         return (table->ops.f_add)(table->h_table, key, (void *) entry,
 539                 key_found, (void **) entry_ptr);
 540 }
 541
 542 int
 543 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
 544                 uint32_t table_id,
 545                 void *key,
 546                 int *key_found,
 547                 struct rte_pipeline_table_entry *entry)
 548 {
 549         struct rte_table *table;
 550
 551         /* Check input arguments */
 552         if (p == NULL) {
 553                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 554                         __func__);
 555                 return -EINVAL;
 556         }
 557
 558         if (key == NULL) {
 559                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 560                         __func__);
 561                 return -EINVAL;
 562         }
 563
 564         if (table_id >= p->num_tables) {
 565                 RTE_LOG(ERR, PIPELINE,
 566                         "%s: table_id %d out of range\n", __func__, table_id);
 567                 return -EINVAL;
 568         }
 569
 570         table = &p->tables[table_id];
 571
 572         if (table->ops.f_delete == NULL) {
 573                 RTE_LOG(ERR, PIPELINE,
 574                         "%s: f_delete function pointer NULL\n", __func__);
 575                 return -EINVAL;
 576         }
 577
 578         return (table->ops.f_delete)(table->h_table, key, key_found, entry);
 579 }
 580
 581 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
 582         uint32_t table_id,
 583         void **keys,
 584         struct rte_pipeline_table_entry **entries,
 585         uint32_t n_keys,
 586         int *key_found,
 587         struct rte_pipeline_table_entry **entries_ptr)
 588 {
 589         struct rte_table *table;
 590         uint32_t i;
 591
 592         /* Check input arguments */
 593         if (p == NULL) {
 594                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 595                         __func__);
 596                 return -EINVAL;
 597         }
 598
 599         if (keys == NULL) {
 600                 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
 601                 return -EINVAL;
 602         }
 603
 604         if (entries == NULL) {
 605                 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
 606                         __func__);
 607                 return -EINVAL;
 608         }
 609
 610         if (table_id >= p->num_tables) {
 611                 RTE_LOG(ERR, PIPELINE,
 612                         "%s: table_id %d out of range\n", __func__, table_id);
 613                 return -EINVAL;
 614         }
 615
 616         table = &p->tables[table_id];
 617
 618         if (table->ops.f_add_bulk == NULL) {
 619                 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
 620                         __func__);
 621                 return -EINVAL;
 622         }
 623
 624         for (i = 0; i < n_keys; i++) {
 625                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 626                         table->table_next_id_valid &&
 627                         (entries[i]->table_id != table->table_next_id)) {
 628                         RTE_LOG(ERR, PIPELINE,
 629                                 "%s: Tree-like topologies not allowed\n", __func__);
 630                         return -EINVAL;
 631                 }
 632         }
 633
 634         /* Add entry */
 635         for (i = 0; i < n_keys; i++) {
 636                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 637                         (table->table_next_id_valid == 0)) {
 638                         table->table_next_id = entries[i]->table_id;
 639                         table->table_next_id_valid = 1;
 640                 }
 641         }
 642
 643         return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
 644                 n_keys, key_found, (void **) entries_ptr);
 645 }
 646
 647 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
 648         uint32_t table_id,
 649         void **keys,
 650         uint32_t n_keys,
 651         int *key_found,
 652         struct rte_pipeline_table_entry **entries)
 653 {
 654         struct rte_table *table;
 655
 656         /* Check input arguments */
 657         if (p == NULL) {
 658                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 659                         __func__);
 660                 return -EINVAL;
 661         }
 662
 663         if (keys == NULL) {
 664                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 665                         __func__);
 666                 return -EINVAL;
 667         }
 668
 669         if (table_id >= p->num_tables) {
 670                 RTE_LOG(ERR, PIPELINE,
 671                         "%s: table_id %d out of range\n", __func__, table_id);
 672                 return -EINVAL;
 673         }
 674
 675         table = &p->tables[table_id];
 676
 677         if (table->ops.f_delete_bulk == NULL) {
 678                 RTE_LOG(ERR, PIPELINE,
 679                         "%s: f_delete function pointer NULL\n", __func__);
 680                 return -EINVAL;
 681         }
 682
 683         return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
 684                         (void **) entries);
 685 }
 686
 687 /*
 688  * Port
 689  *
 690  */
 691 static int
 692 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
 693                 struct rte_pipeline_port_in_params *params,
 694                 uint32_t *port_id)
 695 {
 696         if (p == NULL) {
 697                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 698                         __func__);
 699                 return -EINVAL;
 700         }
 701         if (params == NULL) {
 702                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 703                 return -EINVAL;
 704         }
 705         if (port_id == NULL) {
 706                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 707                         __func__);
 708                 return -EINVAL;
 709         }
 710
 711         /* ops */
 712         if (params->ops == NULL) {
 713                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 714                         __func__);
 715                 return -EINVAL;
 716         }
 717
 718         if (params->ops->f_create == NULL) {
 719                 RTE_LOG(ERR, PIPELINE,
 720                         "%s: f_create function pointer NULL\n", __func__);
 721                 return -EINVAL;
 722         }
 723
 724         if (params->ops->f_rx == NULL) {
 725                 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
 726                         __func__);
 727                 return -EINVAL;
 728         }
 729
 730         /* burst_size */
 731         if ((params->burst_size == 0) ||
 732                 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
 733                 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
 734                         __func__);
 735                 return -EINVAL;
 736         }
 737
 738         /* Do we have room for one more port? */
 739         if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
 740                 RTE_LOG(ERR, PIPELINE,
 741                         "%s: invalid value for num_ports_in\n", __func__);
 742                 return -EINVAL;
 743         }
 744
 745         return 0;
 746 }
 747
 748 static int
 749 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
 750                 struct rte_pipeline_port_out_params *params,
 751                 uint32_t *port_id)
 752 {
 753         if (p == NULL) {
 754                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 755                         __func__);
 756                 return -EINVAL;
 757         }
 758
 759         if (params == NULL) {
 760                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 761                 return -EINVAL;
 762         }
 763
 764         if (port_id == NULL) {
 765                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 766                         __func__);
 767                 return -EINVAL;
 768         }
 769
 770         /* ops */
 771         if (params->ops == NULL) {
 772                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 773                         __func__);
 774                 return -EINVAL;
 775         }
 776
 777         if (params->ops->f_create == NULL) {
 778                 RTE_LOG(ERR, PIPELINE,
 779                         "%s: f_create function pointer NULL\n", __func__);
 780                 return -EINVAL;
 781         }
 782
 783         if (params->ops->f_tx == NULL) {
 784                 RTE_LOG(ERR, PIPELINE,
 785                         "%s: f_tx function pointer NULL\n", __func__);
 786                 return -EINVAL;
 787         }
 788
 789         if (params->ops->f_tx_bulk == NULL) {
 790                 RTE_LOG(ERR, PIPELINE,
 791                         "%s: f_tx_bulk function pointer NULL\n", __func__);
 792                 return -EINVAL;
 793         }
 794
 795         /* Do we have room for one more port? */
 796         if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
 797                 RTE_LOG(ERR, PIPELINE,
 798                         "%s: invalid value for num_ports_out\n", __func__);
 799                 return -EINVAL;
 800         }
 801
 802         return 0;
 803 }
 804
 805 int
 806 rte_pipeline_port_in_create(struct rte_pipeline *p,
 807                 struct rte_pipeline_port_in_params *params,
 808                 uint32_t *port_id)
 809 {
 810         struct rte_port_in *port;
 811         void *h_port;
 812         uint32_t id;
 813         int status;
 814
 815         /* Check input arguments */
 816         status = rte_pipeline_port_in_check_params(p, params, port_id);
 817         if (status != 0)
 818                 return status;
 819
 820         id = p->num_ports_in;
 821         port = &p->ports_in[id];
 822
 823         /* Create the port */
 824         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 825         if (h_port == NULL) {
 826                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 827                 return -EINVAL;
 828         }
 829
 830         /* Commit current table to the pipeline */
 831         p->num_ports_in++;
 832         *port_id = id;
 833
 834         /* Save input parameters */
 835         memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
 836         port->f_action = params->f_action;
 837         port->arg_ah = params->arg_ah;
 838         port->burst_size = params->burst_size;
 839
 840         /* Initialize port internal data structure */
 841         port->table_id = RTE_TABLE_INVALID;
 842         port->h_port = h_port;
 843         port->next = NULL;
 844
 845         return 0;
 846 }
 847
 848 void
 849 rte_pipeline_port_in_free(struct rte_port_in *port)
 850 {
 851         if (port->ops.f_free != NULL)
 852                 port->ops.f_free(port->h_port);
 853 }
 854
 855 int
 856 rte_pipeline_port_out_create(struct rte_pipeline *p,
 857                 struct rte_pipeline_port_out_params *params,
 858                 uint32_t *port_id)
 859 {
 860         struct rte_port_out *port;
 861         void *h_port;
 862         uint32_t id;
 863         int status;
 864
 865         /* Check input arguments */
 866         status = rte_pipeline_port_out_check_params(p, params, port_id);
 867         if (status != 0)
 868                 return status;
 869
 870         id = p->num_ports_out;
 871         port = &p->ports_out[id];
 872
 873         /* Create the port */
 874         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 875         if (h_port == NULL) {
 876                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 877                 return -EINVAL;
 878         }
 879
 880         /* Commit current table to the pipeline */
 881         p->num_ports_out++;
 882         *port_id = id;
 883
 884         /* Save input parameters */
 885         memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
 886         port->f_action = params->f_action;
 887         port->arg_ah = params->arg_ah;
 888
 889         /* Initialize port internal data structure */
 890         port->h_port = h_port;
 891
 892         return 0;
 893 }
 894
 895 void
 896 rte_pipeline_port_out_free(struct rte_port_out *port)
 897 {
 898         if (port->ops.f_free != NULL)
 899                 port->ops.f_free(port->h_port);
 900 }
 901
 902 int
 903 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
 904                 uint32_t port_id,
 905                 uint32_t table_id)
 906 {
 907         struct rte_port_in *port;
 908
 909         /* Check input arguments */
 910         if (p == NULL) {
 911                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 912                         __func__);
 913                 return -EINVAL;
 914         }
 915
 916         if (port_id >= p->num_ports_in) {
 917                 RTE_LOG(ERR, PIPELINE,
 918                         "%s: port IN ID %u is out of range\n",
 919                         __func__, port_id);
 920                 return -EINVAL;
 921         }
 922
 923         if (table_id >= p->num_tables) {
 924                 RTE_LOG(ERR, PIPELINE,
 925                         "%s: Table ID %u is out of range\n",
 926                         __func__, table_id);
 927                 return -EINVAL;
 928         }
 929
 930         port = &p->ports_in[port_id];
 931         port->table_id = table_id;
 932
 933         return 0;
 934 }
 935
 936 int
 937 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
 938 {
 939         struct rte_port_in *port, *port_prev, *port_next;
 940         uint64_t port_mask;
 941         uint32_t port_prev_id, port_next_id;
 942
 943         /* Check input arguments */
 944         if (p == NULL) {
 945                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 946                         __func__);
 947                 return -EINVAL;
 948         }
 949
 950         if (port_id >= p->num_ports_in) {
 951                 RTE_LOG(ERR, PIPELINE,
 952                         "%s: port IN ID %u is out of range\n",
 953                         __func__, port_id);
 954                 return -EINVAL;
 955         }
 956
 957         port = &p->ports_in[port_id];
 958
 959         /* Return if current input port is already enabled */
 960         port_mask = 1LLU << port_id;
 961         if (p->enabled_port_in_mask & port_mask)
 962                 return 0;
 963
 964         p->enabled_port_in_mask |= port_mask;
 965
 966         /* Add current input port to the pipeline chain of enabled ports */
 967         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 968         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
 969
 970         port_prev = &p->ports_in[port_prev_id];
 971         port_next = &p->ports_in[port_next_id];
 972
 973         port_prev->next = port;
 974         port->next = port_next;
 975
 976         /* Check if list of enabled ports was previously empty */
 977         if (p->enabled_port_in_mask == port_mask)
 978                 p->port_in_next = port;
 979
 980         return 0;
 981 }
 982
 983 int
 984 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
 985 {
 986         struct rte_port_in *port, *port_prev, *port_next;
 987         uint64_t port_mask;
 988         uint32_t port_prev_id, port_next_id;
 989
 990         /* Check input arguments */
 991         if (p == NULL) {
 992                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 993                 __func__);
 994                 return -EINVAL;
 995         }
 996
 997         if (port_id >= p->num_ports_in) {
 998                 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
 999                         __func__, port_id);
1000                 return -EINVAL;
1001         }
1002
1003         port = &p->ports_in[port_id];
1004
1005         /* Return if current input port is already disabled */
1006         port_mask = 1LLU << port_id;
1007         if ((p->enabled_port_in_mask & port_mask) == 0)
1008                 return 0;
1009
1010         p->enabled_port_in_mask &= ~port_mask;
1011
1012         /* Return if no other enabled ports */
1013         if (p->enabled_port_in_mask == 0) {
1014                 p->port_in_next = NULL;
1015
1016                 return 0;
1017         }
1018
1019         /* Add current input port to the pipeline chain of enabled ports */
1020         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1021         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1022
1023         port_prev = &p->ports_in[port_prev_id];
1024         port_next = &p->ports_in[port_next_id];
1025
1026         port_prev->next = port_next;
1027
1028         /* Check if the port which has just been disabled is next to serve */
1029         if (port == p->port_in_next)
1030                 p->port_in_next = port_next;
1031
1032         return 0;
1033 }
1034
1035 /*
1036  * Pipeline run-time
1037  *
1038  */
1039 int
1040 rte_pipeline_check(struct rte_pipeline *p)
1041 {
1042         uint32_t port_in_id;
1043
1044         /* Check input arguments */
1045         if (p == NULL) {
1046                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1047                         __func__);
1048                 return -EINVAL;
1049         }
1050
1051         /* Check that pipeline has at least one input port, one table and one
1052         output port */
1053         if (p->num_ports_in == 0) {
1054                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1055                         __func__);
1056                 return -EINVAL;
1057         }
1058         if (p->num_tables == 0) {
1059                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1060                         __func__);
1061                 return -EINVAL;
1062         }
1063         if (p->num_ports_out == 0) {
1064                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1065                         __func__);
1066                 return -EINVAL;
1067         }
1068
1069         /* Check that all input ports are connected */
1070         for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1071                 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1072
1073                 if (port_in->table_id == RTE_TABLE_INVALID) {
1074                         RTE_LOG(ERR, PIPELINE,
1075                                 "%s: Port IN ID %u is not connected\n",
1076                                 __func__, port_in_id);
1077                         return -EINVAL;
1078                 }
1079         }
1080
1081         return 0;
1082 }
1083
1084 static inline void
1085 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1086 {
1087         p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1088         p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1089         p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1090         p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1091
1092         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1093                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1094                 uint32_t i;
1095
1096                 for (i = 0; i < n_pkts; i++) {
1097                         uint64_t pkt_mask = 1LLU << i;
1098                         uint32_t pos = p->entries[i]->action;
1099
1100                         p->action_mask1[pos] |= pkt_mask;
1101                 }
1102         } else {
1103                 uint32_t i;
1104
1105                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1106                         uint64_t pkt_mask = 1LLU << i;
1107                         uint32_t pos;
1108
1109                         if ((pkt_mask & pkts_mask) == 0)
1110                                 continue;
1111
1112                         pos = p->entries[i]->action;
1113                         p->action_mask1[pos] |= pkt_mask;
1114                 }
1115         }
1116 }
1117
1118 static inline void
1119 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1120         uint64_t pkts_mask, uint32_t port_id)
1121 {
1122         struct rte_port_out *port_out = &p->ports_out[port_id];
1123
1124         p->pkts_mask = pkts_mask;
1125
1126         /* Output port user actions */
1127         if (port_out->f_action != NULL) {
1128                 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1129
1130                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1131                         port_out->n_pkts_dropped_by_ah);
1132         }
1133
1134         /* Output port TX */
1135         if (p->pkts_mask != 0)
1136                 port_out->ops.f_tx_bulk(port_out->h_port,
1137                         p->pkts,
1138                         p->pkts_mask);
1139 }
1140
1141 static inline void
1142 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1143 {
1144         p->pkts_mask = pkts_mask;
1145
1146         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1147                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1148                 uint32_t i;
1149
1150                 for (i = 0; i < n_pkts; i++) {
1151                         struct rte_mbuf *pkt = p->pkts[i];
1152                         uint32_t port_out_id = p->entries[i]->port_id;
1153                         struct rte_port_out *port_out =
1154                                 &p->ports_out[port_out_id];
1155
1156                         /* Output port user actions */
1157                         if (port_out->f_action == NULL) /* Output port TX */
1158                                 port_out->ops.f_tx(port_out->h_port, pkt);
1159                         else {
1160                                 uint64_t pkt_mask = 1LLU << i;
1161
1162                                 port_out->f_action(p,
1163                                         p->pkts,
1164                                         pkt_mask,
1165                                         port_out->arg_ah);
1166
1167                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1168                                         port_out->n_pkts_dropped_by_ah);
1169
1170                                 /* Output port TX */
1171                                 if (pkt_mask & p->pkts_mask)
1172                                         port_out->ops.f_tx(port_out->h_port,
1173                                                 pkt);
1174                         }
1175                 }
1176         } else {
1177                 uint32_t i;
1178
1179                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1180                         uint64_t pkt_mask = 1LLU << i;
1181                         struct rte_mbuf *pkt;
1182                         struct rte_port_out *port_out;
1183                         uint32_t port_out_id;
1184
1185                         if ((pkt_mask & pkts_mask) == 0)
1186                                 continue;
1187
1188                         pkt = p->pkts[i];
1189                         port_out_id = p->entries[i]->port_id;
1190                         port_out = &p->ports_out[port_out_id];
1191
1192                         /* Output port user actions */
1193                         if (port_out->f_action == NULL) /* Output port TX */
1194                                 port_out->ops.f_tx(port_out->h_port, pkt);
1195                         else {
1196                                 port_out->f_action(p,
1197                                         p->pkts,
1198                                         pkt_mask,
1199                                         port_out->arg_ah);
1200
1201                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1202                                         port_out->n_pkts_dropped_by_ah);
1203
1204                                 /* Output port TX */
1205                                 if (pkt_mask & p->pkts_mask)
1206                                         port_out->ops.f_tx(port_out->h_port,
1207                                                 pkt);
1208                         }
1209                 }
1210         }
1211 }
1212
1213 static inline void
1214 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1215         uint64_t pkts_mask)
1216 {
1217         p->pkts_mask = pkts_mask;
1218
1219         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1220                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1221                 uint32_t i;
1222
1223                 for (i = 0; i < n_pkts; i++) {
1224                         struct rte_mbuf *pkt = p->pkts[i];
1225                         uint32_t port_out_id =
1226                                 RTE_MBUF_METADATA_UINT32(pkt,
1227                                         p->offset_port_id);
1228                         struct rte_port_out *port_out = &p->ports_out[
1229                                 port_out_id];
1230
1231                         /* Output port user actions */
1232                         if (port_out->f_action == NULL) /* Output port TX */
1233                                 port_out->ops.f_tx(port_out->h_port, pkt);
1234                         else {
1235                                 uint64_t pkt_mask = 1LLU << i;
1236
1237                                 port_out->f_action(p,
1238                                         p->pkts,
1239                                         pkt_mask,
1240                                         port_out->arg_ah);
1241
1242                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1243                                         port_out->n_pkts_dropped_by_ah);
1244
1245                                 /* Output port TX */
1246                                 if (pkt_mask & p->pkts_mask)
1247                                         port_out->ops.f_tx(port_out->h_port,
1248                                                 pkt);
1249                         }
1250                 }
1251         } else {
1252                 uint32_t i;
1253
1254                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1255                         uint64_t pkt_mask = 1LLU << i;
1256                         struct rte_mbuf *pkt;
1257                         struct rte_port_out *port_out;
1258                         uint32_t port_out_id;
1259
1260                         if ((pkt_mask & pkts_mask) == 0)
1261                                 continue;
1262
1263                         pkt = p->pkts[i];
1264                         port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1265                                 p->offset_port_id);
1266                         port_out = &p->ports_out[port_out_id];
1267
1268                         /* Output port user actions */
1269                         if (port_out->f_action == NULL) /* Output port TX */
1270                                 port_out->ops.f_tx(port_out->h_port, pkt);
1271                         else {
1272                                 port_out->f_action(p,
1273                                         p->pkts,
1274                                         pkt_mask,
1275                                         port_out->arg_ah);
1276
1277                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1278                                         port_out->n_pkts_dropped_by_ah);
1279
1280                                 /* Output port TX */
1281                                 if (pkt_mask & p->pkts_mask)
1282                                         port_out->ops.f_tx(port_out->h_port,
1283                                                 pkt);
1284                         }
1285                 }
1286         }
1287 }
1288
1289 static inline void
1290 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1291 {
1292         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1293                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1294                 uint32_t i;
1295
1296                 for (i = 0; i < n_pkts; i++)
1297                         rte_pktmbuf_free(p->pkts[i]);
1298         } else {
1299                 uint32_t i;
1300
1301                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1302                         uint64_t pkt_mask = 1LLU << i;
1303
1304                         if ((pkt_mask & pkts_mask) == 0)
1305                                 continue;
1306
1307                         rte_pktmbuf_free(p->pkts[i]);
1308                 }
1309         }
1310 }
1311
1312 int
1313 rte_pipeline_run(struct rte_pipeline *p)
1314 {
1315         struct rte_port_in *port_in = p->port_in_next;
1316         uint32_t n_pkts, table_id;
1317
1318         if (port_in == NULL)
1319                 return 0;
1320
1321         /* Input port RX */
1322         n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1323                 port_in->burst_size);
1324         if (n_pkts == 0) {
1325                 p->port_in_next = port_in->next;
1326                 return 0;
1327         }
1328
1329         p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1330         p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1331         p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1332         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1333         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1334
1335         /* Input port user actions */
1336         if (port_in->f_action != NULL) {
1337                 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1338
1339                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1340                         port_in->n_pkts_dropped_by_ah);
1341         }
1342
1343         /* Table */
1344         for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1345                 struct rte_table *table;
1346                 uint64_t lookup_hit_mask, lookup_miss_mask;
1347
1348                 /* Lookup */
1349                 table = &p->tables[table_id];
1350                 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1351                         &lookup_hit_mask, (void **) p->entries);
1352                 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1353
1354                 /* Lookup miss */
1355                 if (lookup_miss_mask != 0) {
1356                         struct rte_pipeline_table_entry *default_entry =
1357                                 table->default_entry;
1358
1359                         p->pkts_mask = lookup_miss_mask;
1360
1361                         /* Table user actions */
1362                         if (table->f_action_miss != NULL) {
1363                                 table->f_action_miss(p,
1364                                         p->pkts,
1365                                         lookup_miss_mask,
1366                                         default_entry,
1367                                         table->arg_ah);
1368
1369                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1370                                         table->n_pkts_dropped_by_lkp_miss_ah);
1371                         }
1372
1373                         /* Table reserved actions */
1374                         if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1375                                 (p->pkts_mask != 0))
1376                                 rte_pipeline_action_handler_port_bulk(p,
1377                                         p->pkts_mask,
1378                                         default_entry->port_id);
1379                         else {
1380                                 uint32_t pos = default_entry->action;
1381
1382                                 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1383
1384                                 p->action_mask0[pos] |= p->pkts_mask;
1385
1386                                 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1387                                         table->n_pkts_dropped_lkp_miss);
1388                         }
1389                 }
1390
1391                 /* Lookup hit */
1392                 if (lookup_hit_mask != 0) {
1393                         p->pkts_mask = lookup_hit_mask;
1394
1395                         /* Table user actions */
1396                         if (table->f_action_hit != NULL) {
1397                                 table->f_action_hit(p,
1398                                         p->pkts,
1399                                         lookup_hit_mask,
1400                                         p->entries,
1401                                         table->arg_ah);
1402
1403                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1404                                         table->n_pkts_dropped_by_lkp_hit_ah);
1405                         }
1406
1407                         /* Table reserved actions */
1408                         RTE_PIPELINE_STATS_TABLE_DROP0(p);
1409                         rte_pipeline_compute_masks(p, p->pkts_mask);
1410                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1411                                 p->action_mask1[
1412                                         RTE_PIPELINE_ACTION_DROP];
1413                         p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1414                                 p->action_mask1[
1415                                         RTE_PIPELINE_ACTION_PORT];
1416                         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1417                                 p->action_mask1[
1418                                         RTE_PIPELINE_ACTION_PORT_META];
1419                         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1420                                 p->action_mask1[
1421                                         RTE_PIPELINE_ACTION_TABLE];
1422
1423                         RTE_PIPELINE_STATS_TABLE_DROP1(p,
1424                                 table->n_pkts_dropped_lkp_hit);
1425                 }
1426
1427                 /* Prepare for next iteration */
1428                 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1429                 table_id = table->table_next_id;
1430                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1431         }
1432
1433         /* Table reserved action PORT */
1434         rte_pipeline_action_handler_port(p,
1435                 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1436
1437         /* Table reserved action PORT META */
1438         rte_pipeline_action_handler_port_meta(p,
1439                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1440
1441         /* Table reserved action DROP */
1442         rte_pipeline_action_handler_drop(p,
1443                 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1444
1445         /* Pick candidate for next port IN to serve */
1446         p->port_in_next = port_in->next;
1447
1448         return (int) n_pkts;
1449 }
1450
1451 int
1452 rte_pipeline_flush(struct rte_pipeline *p)
1453 {
1454         uint32_t port_id;
1455
1456         /* Check input arguments */
1457         if (p == NULL) {
1458                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1459                         __func__);
1460                 return -EINVAL;
1461         }
1462
1463         for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1464                 struct rte_port_out *port = &p->ports_out[port_id];
1465
1466                 if (port->ops.f_flush != NULL)
1467                         port->ops.f_flush(port->h_port);
1468         }
1469
1470         return 0;
1471 }
1472
1473 int
1474 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1475         uint32_t port_id, struct rte_mbuf *pkt)
1476 {
1477         struct rte_port_out *port_out = &p->ports_out[port_id];
1478
1479         port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1480
1481         return 0;
1482 }
1483
1484 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1485         uint64_t pkts_mask)
1486 {
1487         pkts_mask &= p->pkts_mask;
1488         p->pkts_mask &= ~pkts_mask;
1489
1490         return 0;
1491 }
1492
1493 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1494         uint64_t pkts_mask)
1495 {
1496         pkts_mask &= p->pkts_mask;
1497         p->pkts_mask &= ~pkts_mask;
1498         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1499
1500         RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1501         return 0;
1502 }
1503
1504 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1505         struct rte_pipeline_port_in_stats *stats, int clear)
1506 {
1507         struct rte_port_in *port;
1508         int retval;
1509
1510         if (p == NULL) {
1511                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1512                         __func__);
1513                 return -EINVAL;
1514         }
1515
1516         if (port_id >= p->num_ports_in) {
1517                 RTE_LOG(ERR, PIPELINE,
1518                         "%s: port IN ID %u is out of range\n",
1519                         __func__, port_id);
1520                 return -EINVAL;
1521         }
1522
1523         port = &p->ports_in[port_id];
1524
1525         if (port->ops.f_stats != NULL) {
1526                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1527                 if (retval)
1528                         return retval;
1529         } else if (stats != NULL)
1530                 memset(&stats->stats, 0, sizeof(stats->stats));
1531
1532         if (stats != NULL)
1533                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1534
1535         if (clear != 0)
1536                 port->n_pkts_dropped_by_ah = 0;
1537
1538         return 0;
1539 }
1540
1541 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1542         struct rte_pipeline_port_out_stats *stats, int clear)
1543 {
1544         struct rte_port_out *port;
1545         int retval;
1546
1547         if (p == NULL) {
1548                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1549                 return -EINVAL;
1550         }
1551
1552         if (port_id >= p->num_ports_out) {
1553                 RTE_LOG(ERR, PIPELINE,
1554                         "%s: port OUT ID %u is out of range\n", __func__, port_id);
1555                 return -EINVAL;
1556         }
1557
1558         port = &p->ports_out[port_id];
1559         if (port->ops.f_stats != NULL) {
1560                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1561                 if (retval != 0)
1562                         return retval;
1563         } else if (stats != NULL)
1564                 memset(&stats->stats, 0, sizeof(stats->stats));
1565
1566         if (stats != NULL)
1567                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1568
1569         if (clear != 0)
1570                 port->n_pkts_dropped_by_ah = 0;
1571
1572         return 0;
1573 }
1574
1575 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1576         struct rte_pipeline_table_stats *stats, int clear)
1577 {
1578         struct rte_table *table;
1579         int retval;
1580
1581         if (p == NULL) {
1582                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1583                         __func__);
1584                 return -EINVAL;
1585         }
1586
1587         if (table_id >= p->num_tables) {
1588                 RTE_LOG(ERR, PIPELINE,
1589                                 "%s: table %u is out of range\n", __func__, table_id);
1590                 return -EINVAL;
1591         }
1592
1593         table = &p->tables[table_id];
1594         if (table->ops.f_stats != NULL) {
1595                 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1596                 if (retval != 0)
1597                         return retval;
1598         } else if (stats != NULL)
1599                 memset(&stats->stats, 0, sizeof(stats->stats));
1600
1601         if (stats != NULL) {
1602                 stats->n_pkts_dropped_by_lkp_hit_ah =
1603                         table->n_pkts_dropped_by_lkp_hit_ah;
1604                 stats->n_pkts_dropped_by_lkp_miss_ah =
1605                         table->n_pkts_dropped_by_lkp_miss_ah;
1606                 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1607                 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1608         }
1609
1610         if (clear != 0) {
1611                 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1612                 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1613                 table->n_pkts_dropped_lkp_hit = 0;
1614                 table->n_pkts_dropped_lkp_miss = 0;
1615         }
1616
1617         return 0;
1618 }