examples/ip_pipeline/pipeline/pipeline_passthrough_be.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <string.h>
  36
  37 #include <rte_common.h>
  38 #include <rte_malloc.h>
  39 #include <rte_byteorder.h>
  40 #include <rte_table_stub.h>
  41 #include <rte_table_hash.h>
  42 #include <rte_pipeline.h>
  43
  44 #include "pipeline_passthrough_be.h"
  45 #include "pipeline_actions_common.h"
  46 #include "parser.h"
  47 #include "hash_func.h"
  48
  49 #define SWAP_DIM (PIPELINE_PASSTHROUGH_SWAP_N_FIELDS_MAX * \
  50         (PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX / sizeof(uint64_t)))
  51
  52 struct pipeline_passthrough {
  53         struct pipeline p;
  54         struct pipeline_passthrough_params params;
  55         rte_table_hash_op_hash f_hash;
  56         uint32_t swap_field0_offset[SWAP_DIM];
  57         uint32_t swap_field1_offset[SWAP_DIM];
  58         uint64_t swap_field_mask[SWAP_DIM];
  59         uint32_t swap_n_fields;
  60 } __rte_cache_aligned;
  61
  62 static pipeline_msg_req_handler handlers[] = {
  63         [PIPELINE_MSG_REQ_PING] =
  64                 pipeline_msg_req_ping_handler,
  65         [PIPELINE_MSG_REQ_STATS_PORT_IN] =
  66                 pipeline_msg_req_stats_port_in_handler,
  67         [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
  68                 pipeline_msg_req_stats_port_out_handler,
  69         [PIPELINE_MSG_REQ_STATS_TABLE] =
  70                 pipeline_msg_req_stats_table_handler,
  71         [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
  72                 pipeline_msg_req_port_in_enable_handler,
  73         [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
  74                 pipeline_msg_req_port_in_disable_handler,
  75         [PIPELINE_MSG_REQ_CUSTOM] =
  76                 pipeline_msg_req_invalid_handler,
  77 };
  78
  79 static inline __attribute__((always_inline)) void
  80 pkt_work_dma(
  81         struct rte_mbuf *pkt,
  82         void *arg,
  83         uint32_t dma_size,
  84         uint32_t hash_enabled,
  85         uint32_t lb_hash,
  86         uint32_t port_out_pow2)
  87 {
  88         struct pipeline_passthrough *p = arg;
  89
  90         uint64_t *dma_dst = RTE_MBUF_METADATA_UINT64_PTR(pkt,
  91                 p->params.dma_dst_offset);
  92         uint64_t *dma_src = RTE_MBUF_METADATA_UINT64_PTR(pkt,
  93                 p->params.dma_src_offset);
  94         uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
  95         uint32_t *dma_hash = RTE_MBUF_METADATA_UINT32_PTR(pkt,
  96                 p->params.dma_hash_offset);
  97         uint32_t i;
  98
  99         /* Read (dma_src), compute (dma_dst), write (dma_dst) */
 100         for (i = 0; i < (dma_size / 8); i++)
 101                 dma_dst[i] = dma_src[i] & dma_mask[i];
 102
 103         /* Read (dma_dst), compute (hash), write (hash) */
 104         if (hash_enabled) {
 105                 uint32_t hash = p->f_hash(dma_dst, dma_size, 0);
 106                 *dma_hash = hash;
 107
 108                 if (lb_hash) {
 109                         uint32_t port_out;
 110
 111                         if (port_out_pow2)
 112                                 port_out
 113                                         = hash & (p->p.n_ports_out - 1);
 114                         else
 115                                 port_out
 116                                         = hash % p->p.n_ports_out;
 117
 118                         rte_pipeline_port_out_packet_insert(p->p.p,
 119                                 port_out, pkt);
 120                 }
 121         }
 122 }
 123
 124 static inline __attribute__((always_inline)) void
 125 pkt4_work_dma(
 126         struct rte_mbuf **pkts,
 127         void *arg,
 128         uint32_t dma_size,
 129         uint32_t hash_enabled,
 130         uint32_t lb_hash,
 131         uint32_t port_out_pow2)
 132 {
 133         struct pipeline_passthrough *p = arg;
 134
 135         uint64_t *dma_dst0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 136                 p->params.dma_dst_offset);
 137         uint64_t *dma_dst1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 138                 p->params.dma_dst_offset);
 139         uint64_t *dma_dst2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 140                 p->params.dma_dst_offset);
 141         uint64_t *dma_dst3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 142                 p->params.dma_dst_offset);
 143
 144         uint64_t *dma_src0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 145                 p->params.dma_src_offset);
 146         uint64_t *dma_src1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 147                 p->params.dma_src_offset);
 148         uint64_t *dma_src2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 149                 p->params.dma_src_offset);
 150         uint64_t *dma_src3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 151                 p->params.dma_src_offset);
 152
 153         uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
 154
 155         uint32_t *dma_hash0 = RTE_MBUF_METADATA_UINT32_PTR(pkts[0],
 156                 p->params.dma_hash_offset);
 157         uint32_t *dma_hash1 = RTE_MBUF_METADATA_UINT32_PTR(pkts[1],
 158                 p->params.dma_hash_offset);
 159         uint32_t *dma_hash2 = RTE_MBUF_METADATA_UINT32_PTR(pkts[2],
 160                 p->params.dma_hash_offset);
 161         uint32_t *dma_hash3 = RTE_MBUF_METADATA_UINT32_PTR(pkts[3],
 162                 p->params.dma_hash_offset);
 163
 164         uint32_t i;
 165
 166         /* Read (dma_src), compute (dma_dst), write (dma_dst) */
 167         for (i = 0; i < (dma_size / 8); i++) {
 168                 dma_dst0[i] = dma_src0[i] & dma_mask[i];
 169                 dma_dst1[i] = dma_src1[i] & dma_mask[i];
 170                 dma_dst2[i] = dma_src2[i] & dma_mask[i];
 171                 dma_dst3[i] = dma_src3[i] & dma_mask[i];
 172         }
 173
 174         /* Read (dma_dst), compute (hash), write (hash) */
 175         if (hash_enabled) {
 176                 uint32_t hash0 = p->f_hash(dma_dst0, dma_size, 0);
 177                 uint32_t hash1 = p->f_hash(dma_dst1, dma_size, 0);
 178                 uint32_t hash2 = p->f_hash(dma_dst2, dma_size, 0);
 179                 uint32_t hash3 = p->f_hash(dma_dst3, dma_size, 0);
 180
 181                 *dma_hash0 = hash0;
 182                 *dma_hash1 = hash1;
 183                 *dma_hash2 = hash2;
 184                 *dma_hash3 = hash3;
 185
 186                 if (lb_hash) {
 187                         uint32_t port_out0, port_out1, port_out2, port_out3;
 188
 189                         if (port_out_pow2) {
 190                                 port_out0
 191                                         = hash0 & (p->p.n_ports_out - 1);
 192                                 port_out1
 193                                         = hash1 & (p->p.n_ports_out - 1);
 194                                 port_out2
 195                                         = hash2 & (p->p.n_ports_out - 1);
 196                                 port_out3
 197                                         = hash3 & (p->p.n_ports_out - 1);
 198                         } else {
 199                                 port_out0
 200                                         = hash0 % p->p.n_ports_out;
 201                                 port_out1
 202                                         = hash1 % p->p.n_ports_out;
 203                                 port_out2
 204                                         = hash2 % p->p.n_ports_out;
 205                                 port_out3
 206                                         = hash3 % p->p.n_ports_out;
 207                         }
 208                         rte_pipeline_port_out_packet_insert(p->p.p,
 209                                 port_out0, pkts[0]);
 210                         rte_pipeline_port_out_packet_insert(p->p.p,
 211                                 port_out1, pkts[1]);
 212                         rte_pipeline_port_out_packet_insert(p->p.p,
 213                                 port_out2, pkts[2]);
 214                         rte_pipeline_port_out_packet_insert(p->p.p,
 215                                 port_out3, pkts[3]);
 216                 }
 217         }
 218 }
 219
 220 static inline __attribute__((always_inline)) void
 221 pkt_work_swap(
 222         struct rte_mbuf *pkt,
 223         void *arg)
 224 {
 225         struct pipeline_passthrough *p = arg;
 226         uint32_t i;
 227
 228         /* Read(field0, field1), compute(field0, field1), write(field0, field1) */
 229         for (i = 0; i < p->swap_n_fields; i++) {
 230                 uint64_t *field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt,
 231                         p->swap_field0_offset[i]);
 232                 uint64_t *field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt,
 233                         p->swap_field1_offset[i]);
 234                 uint64_t mask = p->swap_field_mask[i];
 235
 236                 uint64_t field0 = *field0_ptr;
 237                 uint64_t field1 = *field1_ptr;
 238
 239                 *field0_ptr = (field0 & (~mask)) + (field1 & mask);
 240                 *field1_ptr = (field0 & mask) + (field1 & (~mask));
 241         }
 242 }
 243
 244 static inline __attribute__((always_inline)) void
 245 pkt4_work_swap(
 246         struct rte_mbuf **pkts,
 247         void *arg)
 248 {
 249         struct pipeline_passthrough *p = arg;
 250         uint32_t i;
 251
 252         /* Read(field0, field1), compute(field0, field1), write(field0, field1) */
 253         for (i = 0; i < p->swap_n_fields; i++) {
 254                 uint64_t *pkt0_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 255                         p->swap_field0_offset[i]);
 256                 uint64_t *pkt1_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 257                         p->swap_field0_offset[i]);
 258                 uint64_t *pkt2_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 259                         p->swap_field0_offset[i]);
 260                 uint64_t *pkt3_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 261                         p->swap_field0_offset[i]);
 262
 263                 uint64_t *pkt0_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 264                         p->swap_field1_offset[i]);
 265                 uint64_t *pkt1_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 266                         p->swap_field1_offset[i]);
 267                 uint64_t *pkt2_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 268                         p->swap_field1_offset[i]);
 269                 uint64_t *pkt3_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 270                         p->swap_field1_offset[i]);
 271
 272                 uint64_t mask = p->swap_field_mask[i];
 273
 274                 uint64_t pkt0_field0 = *pkt0_field0_ptr;
 275                 uint64_t pkt1_field0 = *pkt1_field0_ptr;
 276                 uint64_t pkt2_field0 = *pkt2_field0_ptr;
 277                 uint64_t pkt3_field0 = *pkt3_field0_ptr;
 278
 279                 uint64_t pkt0_field1 = *pkt0_field1_ptr;
 280                 uint64_t pkt1_field1 = *pkt1_field1_ptr;
 281                 uint64_t pkt2_field1 = *pkt2_field1_ptr;
 282                 uint64_t pkt3_field1 = *pkt3_field1_ptr;
 283
 284                 *pkt0_field0_ptr = (pkt0_field0 & (~mask)) + (pkt0_field1 & mask);
 285                 *pkt1_field0_ptr = (pkt1_field0 & (~mask)) + (pkt1_field1 & mask);
 286                 *pkt2_field0_ptr = (pkt2_field0 & (~mask)) + (pkt2_field1 & mask);
 287                 *pkt3_field0_ptr = (pkt3_field0 & (~mask)) + (pkt3_field1 & mask);
 288
 289                 *pkt0_field1_ptr = (pkt0_field0 & mask) + (pkt0_field1 & (~mask));
 290                 *pkt1_field1_ptr = (pkt1_field0 & mask) + (pkt1_field1 & (~mask));
 291                 *pkt2_field1_ptr = (pkt2_field0 & mask) + (pkt2_field1 & (~mask));
 292                 *pkt3_field1_ptr = (pkt3_field0 & mask) + (pkt3_field1 & (~mask));
 293         }
 294 }
 295
 296 #define PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)        \
 297 static inline void                                              \
 298 pkt_work_dma_size##dma_size##_hash##hash_enabled                \
 299         ##_lb##lb_hash##_pw##port_pow2(                 \
 300         struct rte_mbuf *pkt,                                   \
 301         void *arg)                                              \
 302 {                                                               \
 303         pkt_work_dma(pkt, arg, dma_size, hash_enabled, lb_hash, port_pow2);     \
 304 }
 305
 306 #define PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)       \
 307 static inline void                                              \
 308 pkt4_work_dma_size##dma_size##_hash##hash_enabled                       \
 309         ##_lb##lb_hash##_pw##port_pow2(                 \
 310         struct rte_mbuf **pkts,                                 \
 311         void *arg)                                              \
 312 {                                                               \
 313         pkt4_work_dma(pkts, arg, dma_size, hash_enabled, lb_hash, port_pow2); \
 314 }
 315
 316 #define port_in_ah_dma(dma_size, hash_enabled, lb_hash, port_pow2)      \
 317 PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)                        \
 318 PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)                       \
 319 PIPELINE_PORT_IN_AH(port_in_ah_dma_size##dma_size##_hash        \
 320         ##hash_enabled##_lb##lb_hash##_pw##port_pow2,           \
 321         pkt_work_dma_size##dma_size##_hash##hash_enabled                \
 322         ##_lb##lb_hash##_pw##port_pow2,                 \
 323         pkt4_work_dma_size##dma_size##_hash##hash_enabled               \
 324         ##_lb##lb_hash##_pw##port_pow2)
 325
 326
 327 #define port_in_ah_lb(dma_size, hash_enabled, lb_hash, port_pow2) \
 328 PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)                \
 329 PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)       \
 330 PIPELINE_PORT_IN_AH_HIJACK_ALL(                                         \
 331         port_in_ah_lb_size##dma_size##_hash##hash_enabled               \
 332         ##_lb##lb_hash##_pw##port_pow2,                 \
 333         pkt_work_dma_size##dma_size##_hash##hash_enabled                \
 334         ##_lb##lb_hash##_pw##port_pow2, \
 335         pkt4_work_dma_size##dma_size##_hash##hash_enabled               \
 336         ##_lb##lb_hash##_pw##port_pow2)
 337
 338 PIPELINE_PORT_IN_AH(port_in_ah_swap, pkt_work_swap,     pkt4_work_swap)
 339
 340
 341 /* Port in AH DMA(dma_size, hash_enabled, lb_hash, port_pow2) */
 342
 343 port_in_ah_dma(8, 0, 0, 0)
 344 port_in_ah_dma(8, 1, 0, 0)
 345 port_in_ah_lb(8, 1, 1, 0)
 346 port_in_ah_lb(8, 1, 1, 1)
 347
 348 port_in_ah_dma(16, 0, 0, 0)
 349 port_in_ah_dma(16, 1, 0, 0)
 350 port_in_ah_lb(16, 1, 1, 0)
 351 port_in_ah_lb(16, 1, 1, 1)
 352
 353 port_in_ah_dma(24, 0, 0, 0)
 354 port_in_ah_dma(24, 1, 0, 0)
 355 port_in_ah_lb(24, 1, 1, 0)
 356 port_in_ah_lb(24, 1, 1, 1)
 357
 358 port_in_ah_dma(32, 0, 0, 0)
 359 port_in_ah_dma(32, 1, 0, 0)
 360 port_in_ah_lb(32, 1, 1, 0)
 361 port_in_ah_lb(32, 1, 1, 1)
 362
 363 port_in_ah_dma(40, 0, 0, 0)
 364 port_in_ah_dma(40, 1, 0, 0)
 365 port_in_ah_lb(40, 1, 1, 0)
 366 port_in_ah_lb(40, 1, 1, 1)
 367
 368 port_in_ah_dma(48, 0, 0, 0)
 369 port_in_ah_dma(48, 1, 0, 0)
 370 port_in_ah_lb(48, 1, 1, 0)
 371 port_in_ah_lb(48, 1, 1, 1)
 372
 373 port_in_ah_dma(56, 0, 0, 0)
 374 port_in_ah_dma(56, 1, 0, 0)
 375 port_in_ah_lb(56, 1, 1, 0)
 376 port_in_ah_lb(56, 1, 1, 1)
 377
 378 port_in_ah_dma(64, 0, 0, 0)
 379 port_in_ah_dma(64, 1, 0, 0)
 380 port_in_ah_lb(64, 1, 1, 0)
 381 port_in_ah_lb(64, 1, 1, 1)
 382
 383 static rte_pipeline_port_in_action_handler
 384 get_port_in_ah(struct pipeline_passthrough *p)
 385 {
 386         if ((p->params.dma_enabled == 0) &&
 387                 (p->params.swap_enabled == 0))
 388                 return NULL;
 389
 390         if (p->params.swap_enabled)
 391                 return port_in_ah_swap;
 392
 393         if (p->params.dma_hash_enabled) {
 394                 if (p->params.dma_hash_lb_enabled) {
 395                         if (rte_is_power_of_2(p->p.n_ports_out))
 396                                 switch (p->params.dma_size) {
 397
 398                                 case 8: return port_in_ah_lb_size8_hash1_lb1_pw1;
 399                                 case 16: return port_in_ah_lb_size16_hash1_lb1_pw1;
 400                                 case 24: return port_in_ah_lb_size24_hash1_lb1_pw1;
 401                                 case 32: return port_in_ah_lb_size32_hash1_lb1_pw1;
 402                                 case 40: return port_in_ah_lb_size40_hash1_lb1_pw1;
 403                                 case 48: return port_in_ah_lb_size48_hash1_lb1_pw1;
 404                                 case 56: return port_in_ah_lb_size56_hash1_lb1_pw1;
 405                                 case 64: return port_in_ah_lb_size64_hash1_lb1_pw1;
 406                                 default: return NULL;
 407                                 }
 408                         else
 409                                 switch (p->params.dma_size) {
 410
 411                                 case 8: return port_in_ah_lb_size8_hash1_lb1_pw0;
 412                                 case 16: return port_in_ah_lb_size16_hash1_lb1_pw0;
 413                                 case 24: return port_in_ah_lb_size24_hash1_lb1_pw0;
 414                                 case 32: return port_in_ah_lb_size32_hash1_lb1_pw0;
 415                                 case 40: return port_in_ah_lb_size40_hash1_lb1_pw0;
 416                                 case 48: return port_in_ah_lb_size48_hash1_lb1_pw0;
 417                                 case 56: return port_in_ah_lb_size56_hash1_lb1_pw0;
 418                                 case 64: return port_in_ah_lb_size64_hash1_lb1_pw0;
 419                                 default: return NULL;
 420                         }
 421                 } else
 422                         switch (p->params.dma_size) {
 423
 424                         case 8: return port_in_ah_dma_size8_hash1_lb0_pw0;
 425                         case 16: return port_in_ah_dma_size16_hash1_lb0_pw0;
 426                         case 24: return port_in_ah_dma_size24_hash1_lb0_pw0;
 427                         case 32: return port_in_ah_dma_size32_hash1_lb0_pw0;
 428                         case 40: return port_in_ah_dma_size40_hash1_lb0_pw0;
 429                         case 48: return port_in_ah_dma_size48_hash1_lb0_pw0;
 430                         case 56: return port_in_ah_dma_size56_hash1_lb0_pw0;
 431                         case 64: return port_in_ah_dma_size64_hash1_lb0_pw0;
 432                         default: return NULL;
 433                 }
 434         } else
 435                 switch (p->params.dma_size) {
 436
 437                 case 8: return port_in_ah_dma_size8_hash0_lb0_pw0;
 438                 case 16: return port_in_ah_dma_size16_hash0_lb0_pw0;
 439                 case 24: return port_in_ah_dma_size24_hash0_lb0_pw0;
 440                 case 32: return port_in_ah_dma_size32_hash0_lb0_pw0;
 441                 case 40: return port_in_ah_dma_size40_hash0_lb0_pw0;
 442                 case 48: return port_in_ah_dma_size48_hash0_lb0_pw0;
 443                 case 56: return port_in_ah_dma_size56_hash0_lb0_pw0;
 444                 case 64: return port_in_ah_dma_size64_hash0_lb0_pw0;
 445                 default: return NULL;
 446                 }
 447 }
 448
 449 int
 450 pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p,
 451         struct pipeline_params *params)
 452 {
 453         uint32_t dma_dst_offset_present = 0;
 454         uint32_t dma_src_offset_present = 0;
 455         uint32_t dma_src_mask_present = 0;
 456         char dma_mask_str[PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2 + 1];
 457         uint32_t dma_size_present = 0;
 458         uint32_t dma_hash_offset_present = 0;
 459         uint32_t dma_hash_lb_present = 0;
 460         uint32_t i;
 461
 462         /* default values */
 463         p->dma_enabled = 0;
 464         p->dma_hash_enabled = 0;
 465         p->dma_hash_lb_enabled = 0;
 466         memset(p->dma_src_mask, 0xFF, sizeof(p->dma_src_mask));
 467         p->swap_enabled = 0;
 468         p->swap_n_fields = 0;
 469
 470         for (i = 0; i < params->n_args; i++) {
 471                 char *arg_name = params->args_name[i];
 472                 char *arg_value = params->args_value[i];
 473
 474                 /* dma_dst_offset */
 475                 if (strcmp(arg_name, "dma_dst_offset") == 0) {
 476                         int status;
 477
 478                         PIPELINE_PARSE_ERR_DUPLICATE(
 479                                 dma_dst_offset_present == 0, params->name,
 480                                 arg_name);
 481                         dma_dst_offset_present = 1;
 482
 483                         status = parser_read_uint32(&p->dma_dst_offset,
 484                                 arg_value);
 485                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
 486                                 params->name, arg_name, arg_value);
 487                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 488                                 params->name, arg_name, arg_value);
 489
 490                         p->dma_enabled = 1;
 491
 492                         continue;
 493                 }
 494
 495                 /* dma_src_offset */
 496                 if (strcmp(arg_name, "dma_src_offset") == 0) {
 497                         int status;
 498
 499                         PIPELINE_PARSE_ERR_DUPLICATE(
 500                                 dma_src_offset_present == 0, params->name,
 501                                 arg_name);
 502                         dma_src_offset_present = 1;
 503
 504                         status = parser_read_uint32(&p->dma_src_offset,
 505                                 arg_value);
 506                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
 507                                 params->name, arg_name, arg_value);
 508                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 509                                 params->name, arg_name, arg_value);
 510
 511                         p->dma_enabled = 1;
 512
 513                         continue;
 514                 }
 515
 516                 /* dma_size */
 517                 if (strcmp(arg_name, "dma_size") == 0) {
 518                         int status;
 519
 520                         PIPELINE_PARSE_ERR_DUPLICATE(
 521                                 dma_size_present == 0, params->name,
 522                                 arg_name);
 523                         dma_size_present = 1;
 524
 525                         status = parser_read_uint32(&p->dma_size,
 526                                 arg_value);
 527                         PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
 528                                 (p->dma_size != 0) &&
 529                                 ((p->dma_size % 8) == 0)),
 530                                 params->name, arg_name, arg_value);
 531                         PIPELINE_PARSE_ERR_OUT_RNG(((status != -ERANGE) &&
 532                                 (p->dma_size <=
 533                                 PIPELINE_PASSTHROUGH_DMA_SIZE_MAX)),
 534                                 params->name, arg_name, arg_value);
 535
 536                         p->dma_enabled = 1;
 537
 538                         continue;
 539                 }
 540
 541                 /* dma_src_mask */
 542                 if (strcmp(arg_name, "dma_src_mask") == 0) {
 543                         int mask_str_len = strlen(arg_value);
 544
 545                         PIPELINE_PARSE_ERR_DUPLICATE(
 546                                 dma_src_mask_present == 0,
 547                                 params->name, arg_name);
 548                         dma_src_mask_present = 1;
 549
 550                         PIPELINE_ARG_CHECK((mask_str_len <=
 551                                 (PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2)),
 552                                 "Parse error in section \"%s\": entry "
 553                                 "\"%s\" too long", params->name,
 554                                 arg_name);
 555
 556                         snprintf(dma_mask_str, mask_str_len + 1,
 557                                 "%s", arg_value);
 558
 559                         p->dma_enabled = 1;
 560
 561                         continue;
 562                 }
 563
 564                 /* dma_hash_offset */
 565                 if (strcmp(arg_name, "dma_hash_offset") == 0) {
 566                         int status;
 567
 568                         PIPELINE_PARSE_ERR_DUPLICATE(
 569                                 dma_hash_offset_present == 0,
 570                                 params->name, arg_name);
 571                         dma_hash_offset_present = 1;
 572
 573                         status = parser_read_uint32(&p->dma_hash_offset,
 574                                 arg_value);
 575                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
 576                                 params->name, arg_name, arg_value);
 577                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 578                                 params->name, arg_name, arg_value);
 579
 580                         p->dma_hash_enabled = 1;
 581
 582                         continue;
 583                 }
 584
 585                 /* load_balance mode */
 586                 if (strcmp(arg_name, "lb") == 0) {
 587                         PIPELINE_PARSE_ERR_DUPLICATE(
 588                                 dma_hash_lb_present == 0,
 589                                 params->name, arg_name);
 590                         dma_hash_lb_present = 1;
 591
 592                         if (strcmp(arg_value, "hash") &&
 593                                 strcmp(arg_value, "HASH"))
 594
 595                                 PIPELINE_PARSE_ERR_INV_VAL(0,
 596                                         params->name,
 597                                         arg_name,
 598                                         arg_value);
 599
 600                         p->dma_hash_lb_enabled = 1;
 601
 602                         continue;
 603                 }
 604
 605                 /* swap */
 606                 if (strcmp(arg_name, "swap") == 0) {
 607                         uint32_t a, b, n_args;
 608                         int len;
 609
 610                         n_args = sscanf(arg_value, "%" SCNu32 " %" SCNu32 "%n",
 611                                 &a, &b, &len);
 612                         PIPELINE_PARSE_ERR_INV_VAL(((n_args == 2) &&
 613                                 ((size_t) len == strlen(arg_value))),
 614                                 params->name, arg_name, arg_value);
 615
 616                         p->swap_field0_offset[p->swap_n_fields] = a;
 617                         p->swap_field1_offset[p->swap_n_fields] = b;
 618                         p->swap_n_fields++;
 619                         p->swap_enabled = 1;
 620
 621                         continue;
 622                 }
 623
 624                 /* any other */
 625                 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
 626         }
 627
 628         /* Check correlations between arguments */
 629         PIPELINE_ARG_CHECK((p->dma_enabled + p->swap_enabled < 2),
 630                 "Parse error in section \"%s\": DMA and SWAP actions are both enabled",
 631                 params->name);
 632         PIPELINE_ARG_CHECK((dma_dst_offset_present == p->dma_enabled),
 633                 "Parse error in section \"%s\": missing entry "
 634                 "\"dma_dst_offset\"", params->name);
 635         PIPELINE_ARG_CHECK((dma_src_offset_present == p->dma_enabled),
 636                 "Parse error in section \"%s\": missing entry "
 637                 "\"dma_src_offset\"", params->name);
 638         PIPELINE_ARG_CHECK((dma_size_present == p->dma_enabled),
 639                 "Parse error in section \"%s\": missing entry "
 640                 "\"dma_size\"", params->name);
 641         PIPELINE_ARG_CHECK((p->dma_hash_enabled <= p->dma_enabled),
 642                 "Parse error in section \"%s\": missing all DMA entries",
 643                 params->name);
 644         PIPELINE_ARG_CHECK((p->dma_hash_lb_enabled <= p->dma_hash_enabled),
 645                 "Parse error in section \"%s\": missing all DMA hash entries ",
 646                 params->name);
 647
 648         if (dma_src_mask_present) {
 649                 uint32_t dma_size = p->dma_size;
 650                 int status;
 651
 652                 PIPELINE_ARG_CHECK((strlen(dma_mask_str) ==
 653                         (dma_size * 2)), "Parse error in section "
 654                         "\"%s\": dma_src_mask should have exactly %u hex "
 655                         "digits", params->name, (dma_size * 2));
 656
 657                 status = parse_hex_string(dma_mask_str, p->dma_src_mask,
 658                         &p->dma_size);
 659
 660                 PIPELINE_PARSE_ERR_INV_VAL(((status == 0) &&
 661                         (dma_size == p->dma_size)), params->name,
 662                         "dma_src_mask", dma_mask_str);
 663         }
 664
 665         if (p->dma_hash_lb_enabled)
 666                 PIPELINE_ARG_CHECK((params->n_ports_out > 1),
 667                         "Parse error in section \"%s\": entry \"lb\" not "
 668                         "allowed for single output port pipeline",
 669                         params->name);
 670         else
 671                 PIPELINE_ARG_CHECK(((params->n_ports_in >= params->n_ports_out)
 672                         && ((params->n_ports_in % params->n_ports_out) == 0)),
 673                         "Parse error in section \"%s\": n_ports_in needs to be "
 674                         "a multiple of n_ports_out (lb mode disabled)",
 675                         params->name);
 676
 677         return 0;
 678 }
 679
 680 static rte_table_hash_op_hash
 681 get_hash_function(struct pipeline_passthrough *p)
 682 {
 683         switch (p->params.dma_size) {
 684
 685         case 8: return hash_default_key8;
 686         case 16: return hash_default_key16;
 687         case 24: return hash_default_key24;
 688         case 32: return hash_default_key32;
 689         case 40: return hash_default_key40;
 690         case 48: return hash_default_key48;
 691         case 56: return hash_default_key56;
 692         case 64: return hash_default_key64;
 693         default: return NULL;
 694         }
 695 }
 696
 697 static int
 698 pipeline_passthrough_swap_convert(struct pipeline_passthrough *p)
 699 {
 700         uint32_t i;
 701
 702         p->swap_n_fields = 0;
 703
 704         for (i = 0; i < p->params.swap_n_fields; i++) {
 705                 uint32_t offset0 = p->params.swap_field0_offset[i];
 706                 uint32_t offset1 = p->params.swap_field1_offset[i];
 707                 uint32_t size = offset1 - offset0;
 708                 uint32_t j;
 709
 710                 /* Check */
 711                 if ((offset0 >= offset1) ||
 712                         (size > PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX) ||
 713                         (p->swap_n_fields >= SWAP_DIM))
 714                         return -1;
 715
 716                 for (j = 0; j < (size / sizeof(uint64_t)); j++) {
 717                         p->swap_field0_offset[p->swap_n_fields] = offset0;
 718                         p->swap_field1_offset[p->swap_n_fields] = offset1;
 719                         p->swap_field_mask[p->swap_n_fields] = UINT64_MAX;
 720                         p->swap_n_fields++;
 721                         offset0 += sizeof(uint64_t);
 722                         offset1 += sizeof(uint64_t);
 723                 }
 724                 if (size % sizeof(uint64_t)) {
 725                         uint32_t n_bits = (size % sizeof(uint64_t)) * 8;
 726
 727                         p->swap_field0_offset[p->swap_n_fields] = offset0;
 728                         p->swap_field1_offset[p->swap_n_fields] = offset1;
 729                         p->swap_field_mask[p->swap_n_fields] =
 730                                 RTE_LEN2MASK(n_bits, uint64_t);
 731                         p->swap_n_fields++;
 732                 }
 733         }
 734
 735         return 0;
 736 }
 737
 738 static void*
 739 pipeline_passthrough_init(struct pipeline_params *params,
 740         __rte_unused void *arg)
 741 {
 742         struct pipeline *p;
 743         struct pipeline_passthrough *p_pt;
 744         uint32_t size, i;
 745
 746         /* Check input arguments */
 747         if ((params == NULL) ||
 748                 (params->n_ports_in == 0) ||
 749                 (params->n_ports_out == 0))
 750                 return NULL;
 751
 752         /* Memory allocation */
 753         size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_passthrough));
 754         p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
 755         p_pt = (struct pipeline_passthrough *) p;
 756         if (p == NULL)
 757                 return NULL;
 758
 759         strcpy(p->name, params->name);
 760         p->log_level = params->log_level;
 761
 762         PLOG(p, HIGH, "Pass-through");
 763
 764         /* Parse arguments */
 765         if (pipeline_passthrough_parse_args(&p_pt->params, params))
 766                 return NULL;
 767         if (pipeline_passthrough_swap_convert(p_pt))
 768                 return NULL;
 769         p_pt->f_hash = get_hash_function(p_pt);
 770
 771         /* Pipeline */
 772         {
 773                 struct rte_pipeline_params pipeline_params = {
 774                         .name = "PASS-THROUGH",
 775                         .socket_id = params->socket_id,
 776                         .offset_port_id = 0,
 777                 };
 778
 779                 p->p = rte_pipeline_create(&pipeline_params);
 780                 if (p->p == NULL) {
 781                         rte_free(p);
 782                         return NULL;
 783                 }
 784         }
 785
 786         p->n_ports_in = params->n_ports_in;
 787         p->n_ports_out = params->n_ports_out;
 788         p->n_tables = p->n_ports_in;
 789
 790         /*Input ports*/
 791         for (i = 0; i < p->n_ports_in; i++) {
 792                 struct rte_pipeline_port_in_params port_params = {
 793                         .ops = pipeline_port_in_params_get_ops(
 794                                 &params->port_in[i]),
 795                         .arg_create = pipeline_port_in_params_convert(
 796                                 &params->port_in[i]),
 797                         .f_action = get_port_in_ah(p_pt),
 798                         .arg_ah = p_pt,
 799                         .burst_size = params->port_in[i].burst_size,
 800                 };
 801
 802                 int status = rte_pipeline_port_in_create(p->p,
 803                         &port_params,
 804                         &p->port_in_id[i]);
 805
 806                 if (status) {
 807                         rte_pipeline_free(p->p);
 808                         rte_free(p);
 809                         return NULL;
 810                 }
 811         }
 812
 813         /* Output ports */
 814         for (i = 0; i < p->n_ports_out; i++) {
 815                 struct rte_pipeline_port_out_params port_params = {
 816                         .ops = pipeline_port_out_params_get_ops(
 817                                 &params->port_out[i]),
 818                         .arg_create = pipeline_port_out_params_convert(
 819                                 &params->port_out[i]),
 820                         .f_action = NULL,
 821                         .arg_ah = NULL,
 822                 };
 823
 824                 int status = rte_pipeline_port_out_create(p->p,
 825                         &port_params,
 826                         &p->port_out_id[i]);
 827
 828                 if (status) {
 829                         rte_pipeline_free(p->p);
 830                         rte_free(p);
 831                         return NULL;
 832                 }
 833         }
 834
 835         /* Tables */
 836         for (i = 0; i < p->n_ports_in; i++) {
 837                 struct rte_pipeline_table_params table_params = {
 838                         .ops = &rte_table_stub_ops,
 839                         .arg_create = NULL,
 840                         .f_action_hit = NULL,
 841                         .f_action_miss = NULL,
 842                         .arg_ah = NULL,
 843                         .action_data_size = 0,
 844                 };
 845
 846                 int status = rte_pipeline_table_create(p->p,
 847                         &table_params,
 848                         &p->table_id[i]);
 849
 850                 if (status) {
 851                         rte_pipeline_free(p->p);
 852                         rte_free(p);
 853                         return NULL;
 854                 }
 855         }
 856
 857         /* Connecting input ports to tables */
 858         for (i = 0; i < p->n_ports_in; i++) {
 859                 int status = rte_pipeline_port_in_connect_to_table(p->p,
 860                         p->port_in_id[i],
 861                         p->table_id[i]);
 862
 863                 if (status) {
 864                         rte_pipeline_free(p->p);
 865                         rte_free(p);
 866                         return NULL;
 867                 }
 868         }
 869
 870         /* Add entries to tables */
 871         for (i = 0; i < p->n_ports_in; i++) {
 872                 uint32_t port_out_id = (p_pt->params.dma_hash_lb_enabled == 0) ?
 873                         (i / (p->n_ports_in / p->n_ports_out)) :
 874                         0;
 875
 876                 struct rte_pipeline_table_entry default_entry = {
 877                         .action = RTE_PIPELINE_ACTION_PORT,
 878                         {.port_id = p->port_out_id[port_out_id]},
 879                 };
 880
 881                 struct rte_pipeline_table_entry *default_entry_ptr;
 882
 883                 int status = rte_pipeline_table_default_entry_add(p->p,
 884                         p->table_id[i],
 885                         &default_entry,
 886                         &default_entry_ptr);
 887
 888                 if (status) {
 889                         rte_pipeline_free(p->p);
 890                         rte_free(p);
 891                         return NULL;
 892                 }
 893         }
 894
 895         /* Enable input ports */
 896         for (i = 0; i < p->n_ports_in; i++) {
 897                 int status = rte_pipeline_port_in_enable(p->p,
 898                         p->port_in_id[i]);
 899
 900                 if (status) {
 901                         rte_pipeline_free(p->p);
 902                         rte_free(p);
 903                         return NULL;
 904                 }
 905         }
 906
 907         /* Check pipeline consistency */
 908         if (rte_pipeline_check(p->p) < 0) {
 909                 rte_pipeline_free(p->p);
 910                 rte_free(p);
 911                 return NULL;
 912         }
 913
 914         /* Message queues */
 915         p->n_msgq = params->n_msgq;
 916         for (i = 0; i < p->n_msgq; i++)
 917                 p->msgq_in[i] = params->msgq_in[i];
 918         for (i = 0; i < p->n_msgq; i++)
 919                 p->msgq_out[i] = params->msgq_out[i];
 920
 921         /* Message handlers */
 922         memcpy(p->handlers, handlers, sizeof(p->handlers));
 923
 924         return p;
 925 }
 926
 927 static int
 928 pipeline_passthrough_free(void *pipeline)
 929 {
 930         struct pipeline *p = (struct pipeline *) pipeline;
 931
 932         /* Check input arguments */
 933         if (p == NULL)
 934                 return -1;
 935
 936         /* Free resources */
 937         rte_pipeline_free(p->p);
 938         rte_free(p);
 939         return 0;
 940 }
 941
 942 static int
 943 pipeline_passthrough_timer(void *pipeline)
 944 {
 945         struct pipeline *p = (struct pipeline *) pipeline;
 946
 947         pipeline_msg_req_handle(p);
 948         rte_pipeline_flush(p->p);
 949
 950         return 0;
 951 }
 952
 953 struct pipeline_be_ops pipeline_passthrough_be_ops = {
 954         .f_init = pipeline_passthrough_init,
 955         .f_free = pipeline_passthrough_free,
 956         .f_run = NULL,
 957         .f_timer = pipeline_passthrough_timer,
 958 };