examples/ip_pipeline/pipeline/pipeline_passthrough_be.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <string.h>
   7
   8 #include <rte_common.h>
   9 #include <rte_malloc.h>
  10 #include <rte_byteorder.h>
  11 #include <rte_table_stub.h>
  12 #include <rte_table_hash.h>
  13 #include <rte_pipeline.h>
  14
  15 #include "pipeline_passthrough_be.h"
  16 #include "pipeline_actions_common.h"
  17 #include "parser.h"
  18 #include "hash_func.h"
  19
  20 #define SWAP_DIM (PIPELINE_PASSTHROUGH_SWAP_N_FIELDS_MAX * \
  21         (PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX / sizeof(uint64_t)))
  22
  23 struct pipeline_passthrough {
  24         struct pipeline p;
  25         struct pipeline_passthrough_params params;
  26         rte_table_hash_op_hash f_hash;
  27         uint32_t swap_field0_offset[SWAP_DIM];
  28         uint32_t swap_field1_offset[SWAP_DIM];
  29         uint64_t swap_field_mask[SWAP_DIM];
  30         uint32_t swap_n_fields;
  31 } __rte_cache_aligned;
  32
  33 static pipeline_msg_req_handler handlers[] = {
  34         [PIPELINE_MSG_REQ_PING] =
  35                 pipeline_msg_req_ping_handler,
  36         [PIPELINE_MSG_REQ_STATS_PORT_IN] =
  37                 pipeline_msg_req_stats_port_in_handler,
  38         [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
  39                 pipeline_msg_req_stats_port_out_handler,
  40         [PIPELINE_MSG_REQ_STATS_TABLE] =
  41                 pipeline_msg_req_stats_table_handler,
  42         [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
  43                 pipeline_msg_req_port_in_enable_handler,
  44         [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
  45                 pipeline_msg_req_port_in_disable_handler,
  46         [PIPELINE_MSG_REQ_CUSTOM] =
  47                 pipeline_msg_req_invalid_handler,
  48 };
  49
  50 static __rte_always_inline void
  51 pkt_work_dma(
  52         struct rte_mbuf *pkt,
  53         void *arg,
  54         uint32_t dma_size,
  55         uint32_t hash_enabled,
  56         uint32_t lb_hash,
  57         uint32_t port_out_pow2)
  58 {
  59         struct pipeline_passthrough *p = arg;
  60
  61         uint64_t *dma_dst = RTE_MBUF_METADATA_UINT64_PTR(pkt,
  62                 p->params.dma_dst_offset);
  63         uint64_t *dma_src = RTE_MBUF_METADATA_UINT64_PTR(pkt,
  64                 p->params.dma_src_offset);
  65         uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
  66         uint32_t *dma_hash = RTE_MBUF_METADATA_UINT32_PTR(pkt,
  67                 p->params.dma_hash_offset);
  68         uint32_t i;
  69
  70         /* Read (dma_src), compute (dma_dst), write (dma_dst) */
  71         for (i = 0; i < (dma_size / 8); i++)
  72                 dma_dst[i] = dma_src[i] & dma_mask[i];
  73
  74         /* Read (dma_dst), compute (hash), write (hash) */
  75         if (hash_enabled) {
  76                 uint32_t hash = p->f_hash(dma_src, dma_mask, dma_size, 0);
  77                 *dma_hash = hash;
  78
  79                 if (lb_hash) {
  80                         uint32_t port_out;
  81
  82                         if (port_out_pow2)
  83                                 port_out
  84                                         = hash & (p->p.n_ports_out - 1);
  85                         else
  86                                 port_out
  87                                         = hash % p->p.n_ports_out;
  88
  89                         rte_pipeline_port_out_packet_insert(p->p.p,
  90                                 port_out, pkt);
  91                 }
  92         }
  93 }
  94
  95 static __rte_always_inline void
  96 pkt4_work_dma(
  97         struct rte_mbuf **pkts,
  98         void *arg,
  99         uint32_t dma_size,
 100         uint32_t hash_enabled,
 101         uint32_t lb_hash,
 102         uint32_t port_out_pow2)
 103 {
 104         struct pipeline_passthrough *p = arg;
 105
 106         uint64_t *dma_dst0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 107                 p->params.dma_dst_offset);
 108         uint64_t *dma_dst1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 109                 p->params.dma_dst_offset);
 110         uint64_t *dma_dst2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 111                 p->params.dma_dst_offset);
 112         uint64_t *dma_dst3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 113                 p->params.dma_dst_offset);
 114
 115         uint64_t *dma_src0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 116                 p->params.dma_src_offset);
 117         uint64_t *dma_src1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 118                 p->params.dma_src_offset);
 119         uint64_t *dma_src2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 120                 p->params.dma_src_offset);
 121         uint64_t *dma_src3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 122                 p->params.dma_src_offset);
 123
 124         uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
 125
 126         uint32_t *dma_hash0 = RTE_MBUF_METADATA_UINT32_PTR(pkts[0],
 127                 p->params.dma_hash_offset);
 128         uint32_t *dma_hash1 = RTE_MBUF_METADATA_UINT32_PTR(pkts[1],
 129                 p->params.dma_hash_offset);
 130         uint32_t *dma_hash2 = RTE_MBUF_METADATA_UINT32_PTR(pkts[2],
 131                 p->params.dma_hash_offset);
 132         uint32_t *dma_hash3 = RTE_MBUF_METADATA_UINT32_PTR(pkts[3],
 133                 p->params.dma_hash_offset);
 134
 135         uint32_t i;
 136
 137         /* Read (dma_src), compute (dma_dst), write (dma_dst) */
 138         for (i = 0; i < (dma_size / 8); i++) {
 139                 dma_dst0[i] = dma_src0[i] & dma_mask[i];
 140                 dma_dst1[i] = dma_src1[i] & dma_mask[i];
 141                 dma_dst2[i] = dma_src2[i] & dma_mask[i];
 142                 dma_dst3[i] = dma_src3[i] & dma_mask[i];
 143         }
 144
 145         /* Read (dma_dst), compute (hash), write (hash) */
 146         if (hash_enabled) {
 147                 uint32_t hash0 = p->f_hash(dma_src0, dma_mask, dma_size, 0);
 148                 uint32_t hash1 = p->f_hash(dma_src1, dma_mask, dma_size, 0);
 149                 uint32_t hash2 = p->f_hash(dma_src2, dma_mask, dma_size, 0);
 150                 uint32_t hash3 = p->f_hash(dma_src3, dma_mask, dma_size, 0);
 151
 152                 *dma_hash0 = hash0;
 153                 *dma_hash1 = hash1;
 154                 *dma_hash2 = hash2;
 155                 *dma_hash3 = hash3;
 156
 157                 if (lb_hash) {
 158                         uint32_t port_out0, port_out1, port_out2, port_out3;
 159
 160                         if (port_out_pow2) {
 161                                 port_out0
 162                                         = hash0 & (p->p.n_ports_out - 1);
 163                                 port_out1
 164                                         = hash1 & (p->p.n_ports_out - 1);
 165                                 port_out2
 166                                         = hash2 & (p->p.n_ports_out - 1);
 167                                 port_out3
 168                                         = hash3 & (p->p.n_ports_out - 1);
 169                         } else {
 170                                 port_out0
 171                                         = hash0 % p->p.n_ports_out;
 172                                 port_out1
 173                                         = hash1 % p->p.n_ports_out;
 174                                 port_out2
 175                                         = hash2 % p->p.n_ports_out;
 176                                 port_out3
 177                                         = hash3 % p->p.n_ports_out;
 178                         }
 179                         rte_pipeline_port_out_packet_insert(p->p.p,
 180                                 port_out0, pkts[0]);
 181                         rte_pipeline_port_out_packet_insert(p->p.p,
 182                                 port_out1, pkts[1]);
 183                         rte_pipeline_port_out_packet_insert(p->p.p,
 184                                 port_out2, pkts[2]);
 185                         rte_pipeline_port_out_packet_insert(p->p.p,
 186                                 port_out3, pkts[3]);
 187                 }
 188         }
 189 }
 190
 191 static __rte_always_inline void
 192 pkt_work_swap(
 193         struct rte_mbuf *pkt,
 194         void *arg)
 195 {
 196         struct pipeline_passthrough *p = arg;
 197         uint32_t i;
 198
 199         /* Read(field0, field1), compute(field0, field1), write(field0, field1) */
 200         for (i = 0; i < p->swap_n_fields; i++) {
 201                 uint64_t *field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt,
 202                         p->swap_field0_offset[i]);
 203                 uint64_t *field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt,
 204                         p->swap_field1_offset[i]);
 205                 uint64_t mask = p->swap_field_mask[i];
 206
 207                 uint64_t field0 = *field0_ptr;
 208                 uint64_t field1 = *field1_ptr;
 209
 210                 *field0_ptr = (field0 & (~mask)) + (field1 & mask);
 211                 *field1_ptr = (field0 & mask) + (field1 & (~mask));
 212         }
 213 }
 214
 215 static __rte_always_inline void
 216 pkt4_work_swap(
 217         struct rte_mbuf **pkts,
 218         void *arg)
 219 {
 220         struct pipeline_passthrough *p = arg;
 221         uint32_t i;
 222
 223         /* Read(field0, field1), compute(field0, field1), write(field0, field1) */
 224         for (i = 0; i < p->swap_n_fields; i++) {
 225                 uint64_t *pkt0_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 226                         p->swap_field0_offset[i]);
 227                 uint64_t *pkt1_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 228                         p->swap_field0_offset[i]);
 229                 uint64_t *pkt2_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 230                         p->swap_field0_offset[i]);
 231                 uint64_t *pkt3_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 232                         p->swap_field0_offset[i]);
 233
 234                 uint64_t *pkt0_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 235                         p->swap_field1_offset[i]);
 236                 uint64_t *pkt1_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 237                         p->swap_field1_offset[i]);
 238                 uint64_t *pkt2_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 239                         p->swap_field1_offset[i]);
 240                 uint64_t *pkt3_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 241                         p->swap_field1_offset[i]);
 242
 243                 uint64_t mask = p->swap_field_mask[i];
 244
 245                 uint64_t pkt0_field0 = *pkt0_field0_ptr;
 246                 uint64_t pkt1_field0 = *pkt1_field0_ptr;
 247                 uint64_t pkt2_field0 = *pkt2_field0_ptr;
 248                 uint64_t pkt3_field0 = *pkt3_field0_ptr;
 249
 250                 uint64_t pkt0_field1 = *pkt0_field1_ptr;
 251                 uint64_t pkt1_field1 = *pkt1_field1_ptr;
 252                 uint64_t pkt2_field1 = *pkt2_field1_ptr;
 253                 uint64_t pkt3_field1 = *pkt3_field1_ptr;
 254
 255                 *pkt0_field0_ptr = (pkt0_field0 & (~mask)) + (pkt0_field1 & mask);
 256                 *pkt1_field0_ptr = (pkt1_field0 & (~mask)) + (pkt1_field1 & mask);
 257                 *pkt2_field0_ptr = (pkt2_field0 & (~mask)) + (pkt2_field1 & mask);
 258                 *pkt3_field0_ptr = (pkt3_field0 & (~mask)) + (pkt3_field1 & mask);
 259
 260                 *pkt0_field1_ptr = (pkt0_field0 & mask) + (pkt0_field1 & (~mask));
 261                 *pkt1_field1_ptr = (pkt1_field0 & mask) + (pkt1_field1 & (~mask));
 262                 *pkt2_field1_ptr = (pkt2_field0 & mask) + (pkt2_field1 & (~mask));
 263                 *pkt3_field1_ptr = (pkt3_field0 & mask) + (pkt3_field1 & (~mask));
 264         }
 265 }
 266
 267 #define PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)        \
 268 static inline void                                              \
 269 pkt_work_dma_size##dma_size##_hash##hash_enabled                \
 270         ##_lb##lb_hash##_pw##port_pow2(                 \
 271         struct rte_mbuf *pkt,                                   \
 272         void *arg)                                              \
 273 {                                                               \
 274         pkt_work_dma(pkt, arg, dma_size, hash_enabled, lb_hash, port_pow2);     \
 275 }
 276
 277 #define PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)       \
 278 static inline void                                              \
 279 pkt4_work_dma_size##dma_size##_hash##hash_enabled                       \
 280         ##_lb##lb_hash##_pw##port_pow2(                 \
 281         struct rte_mbuf **pkts,                                 \
 282         void *arg)                                              \
 283 {                                                               \
 284         pkt4_work_dma(pkts, arg, dma_size, hash_enabled, lb_hash, port_pow2); \
 285 }
 286
 287 #define port_in_ah_dma(dma_size, hash_enabled, lb_hash, port_pow2)      \
 288 PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)                        \
 289 PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)                       \
 290 PIPELINE_PORT_IN_AH(port_in_ah_dma_size##dma_size##_hash        \
 291         ##hash_enabled##_lb##lb_hash##_pw##port_pow2,           \
 292         pkt_work_dma_size##dma_size##_hash##hash_enabled                \
 293         ##_lb##lb_hash##_pw##port_pow2,                 \
 294         pkt4_work_dma_size##dma_size##_hash##hash_enabled               \
 295         ##_lb##lb_hash##_pw##port_pow2)
 296
 297
 298 #define port_in_ah_lb(dma_size, hash_enabled, lb_hash, port_pow2) \
 299 PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)                \
 300 PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)       \
 301 PIPELINE_PORT_IN_AH_HIJACK_ALL(                                         \
 302         port_in_ah_lb_size##dma_size##_hash##hash_enabled               \
 303         ##_lb##lb_hash##_pw##port_pow2,                 \
 304         pkt_work_dma_size##dma_size##_hash##hash_enabled                \
 305         ##_lb##lb_hash##_pw##port_pow2, \
 306         pkt4_work_dma_size##dma_size##_hash##hash_enabled               \
 307         ##_lb##lb_hash##_pw##port_pow2)
 308
 309 PIPELINE_PORT_IN_AH(port_in_ah_swap, pkt_work_swap,     pkt4_work_swap)
 310
 311
 312 /* Port in AH DMA(dma_size, hash_enabled, lb_hash, port_pow2) */
 313
 314 port_in_ah_dma(8, 0, 0, 0)
 315 port_in_ah_dma(8, 1, 0, 0)
 316 port_in_ah_lb(8, 1, 1, 0)
 317 port_in_ah_lb(8, 1, 1, 1)
 318
 319 port_in_ah_dma(16, 0, 0, 0)
 320 port_in_ah_dma(16, 1, 0, 0)
 321 port_in_ah_lb(16, 1, 1, 0)
 322 port_in_ah_lb(16, 1, 1, 1)
 323
 324 port_in_ah_dma(24, 0, 0, 0)
 325 port_in_ah_dma(24, 1, 0, 0)
 326 port_in_ah_lb(24, 1, 1, 0)
 327 port_in_ah_lb(24, 1, 1, 1)
 328
 329 port_in_ah_dma(32, 0, 0, 0)
 330 port_in_ah_dma(32, 1, 0, 0)
 331 port_in_ah_lb(32, 1, 1, 0)
 332 port_in_ah_lb(32, 1, 1, 1)
 333
 334 port_in_ah_dma(40, 0, 0, 0)
 335 port_in_ah_dma(40, 1, 0, 0)
 336 port_in_ah_lb(40, 1, 1, 0)
 337 port_in_ah_lb(40, 1, 1, 1)
 338
 339 port_in_ah_dma(48, 0, 0, 0)
 340 port_in_ah_dma(48, 1, 0, 0)
 341 port_in_ah_lb(48, 1, 1, 0)
 342 port_in_ah_lb(48, 1, 1, 1)
 343
 344 port_in_ah_dma(56, 0, 0, 0)
 345 port_in_ah_dma(56, 1, 0, 0)
 346 port_in_ah_lb(56, 1, 1, 0)
 347 port_in_ah_lb(56, 1, 1, 1)
 348
 349 port_in_ah_dma(64, 0, 0, 0)
 350 port_in_ah_dma(64, 1, 0, 0)
 351 port_in_ah_lb(64, 1, 1, 0)
 352 port_in_ah_lb(64, 1, 1, 1)
 353
 354 static rte_pipeline_port_in_action_handler
 355 get_port_in_ah(struct pipeline_passthrough *p)
 356 {
 357         if ((p->params.dma_enabled == 0) &&
 358                 (p->params.swap_enabled == 0))
 359                 return NULL;
 360
 361         if (p->params.swap_enabled)
 362                 return port_in_ah_swap;
 363
 364         if (p->params.dma_hash_enabled) {
 365                 if (p->params.dma_hash_lb_enabled) {
 366                         if (rte_is_power_of_2(p->p.n_ports_out))
 367                                 switch (p->params.dma_size) {
 368
 369                                 case 8: return port_in_ah_lb_size8_hash1_lb1_pw1;
 370                                 case 16: return port_in_ah_lb_size16_hash1_lb1_pw1;
 371                                 case 24: return port_in_ah_lb_size24_hash1_lb1_pw1;
 372                                 case 32: return port_in_ah_lb_size32_hash1_lb1_pw1;
 373                                 case 40: return port_in_ah_lb_size40_hash1_lb1_pw1;
 374                                 case 48: return port_in_ah_lb_size48_hash1_lb1_pw1;
 375                                 case 56: return port_in_ah_lb_size56_hash1_lb1_pw1;
 376                                 case 64: return port_in_ah_lb_size64_hash1_lb1_pw1;
 377                                 default: return NULL;
 378                                 }
 379                         else
 380                                 switch (p->params.dma_size) {
 381
 382                                 case 8: return port_in_ah_lb_size8_hash1_lb1_pw0;
 383                                 case 16: return port_in_ah_lb_size16_hash1_lb1_pw0;
 384                                 case 24: return port_in_ah_lb_size24_hash1_lb1_pw0;
 385                                 case 32: return port_in_ah_lb_size32_hash1_lb1_pw0;
 386                                 case 40: return port_in_ah_lb_size40_hash1_lb1_pw0;
 387                                 case 48: return port_in_ah_lb_size48_hash1_lb1_pw0;
 388                                 case 56: return port_in_ah_lb_size56_hash1_lb1_pw0;
 389                                 case 64: return port_in_ah_lb_size64_hash1_lb1_pw0;
 390                                 default: return NULL;
 391                         }
 392                 } else
 393                         switch (p->params.dma_size) {
 394
 395                         case 8: return port_in_ah_dma_size8_hash1_lb0_pw0;
 396                         case 16: return port_in_ah_dma_size16_hash1_lb0_pw0;
 397                         case 24: return port_in_ah_dma_size24_hash1_lb0_pw0;
 398                         case 32: return port_in_ah_dma_size32_hash1_lb0_pw0;
 399                         case 40: return port_in_ah_dma_size40_hash1_lb0_pw0;
 400                         case 48: return port_in_ah_dma_size48_hash1_lb0_pw0;
 401                         case 56: return port_in_ah_dma_size56_hash1_lb0_pw0;
 402                         case 64: return port_in_ah_dma_size64_hash1_lb0_pw0;
 403                         default: return NULL;
 404                 }
 405         } else
 406                 switch (p->params.dma_size) {
 407
 408                 case 8: return port_in_ah_dma_size8_hash0_lb0_pw0;
 409                 case 16: return port_in_ah_dma_size16_hash0_lb0_pw0;
 410                 case 24: return port_in_ah_dma_size24_hash0_lb0_pw0;
 411                 case 32: return port_in_ah_dma_size32_hash0_lb0_pw0;
 412                 case 40: return port_in_ah_dma_size40_hash0_lb0_pw0;
 413                 case 48: return port_in_ah_dma_size48_hash0_lb0_pw0;
 414                 case 56: return port_in_ah_dma_size56_hash0_lb0_pw0;
 415                 case 64: return port_in_ah_dma_size64_hash0_lb0_pw0;
 416                 default: return NULL;
 417                 }
 418 }
 419
 420 int
 421 pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p,
 422         struct pipeline_params *params)
 423 {
 424         uint32_t dma_dst_offset_present = 0;
 425         uint32_t dma_src_offset_present = 0;
 426         uint32_t dma_src_mask_present = 0;
 427         char dma_mask_str[PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2 + 1];
 428         uint32_t dma_size_present = 0;
 429         uint32_t dma_hash_offset_present = 0;
 430         uint32_t dma_hash_lb_present = 0;
 431         uint32_t i;
 432
 433         /* default values */
 434         p->dma_enabled = 0;
 435         p->dma_hash_enabled = 0;
 436         p->dma_hash_lb_enabled = 0;
 437         memset(p->dma_src_mask, 0xFF, sizeof(p->dma_src_mask));
 438         p->swap_enabled = 0;
 439         p->swap_n_fields = 0;
 440
 441         for (i = 0; i < params->n_args; i++) {
 442                 char *arg_name = params->args_name[i];
 443                 char *arg_value = params->args_value[i];
 444
 445                 /* dma_dst_offset */
 446                 if (strcmp(arg_name, "dma_dst_offset") == 0) {
 447                         int status;
 448
 449                         PIPELINE_PARSE_ERR_DUPLICATE(
 450                                 dma_dst_offset_present == 0, params->name,
 451                                 arg_name);
 452                         dma_dst_offset_present = 1;
 453
 454                         status = parser_read_uint32(&p->dma_dst_offset,
 455                                 arg_value);
 456                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
 457                                 params->name, arg_name, arg_value);
 458                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 459                                 params->name, arg_name, arg_value);
 460
 461                         p->dma_enabled = 1;
 462
 463                         continue;
 464                 }
 465
 466                 /* dma_src_offset */
 467                 if (strcmp(arg_name, "dma_src_offset") == 0) {
 468                         int status;
 469
 470                         PIPELINE_PARSE_ERR_DUPLICATE(
 471                                 dma_src_offset_present == 0, params->name,
 472                                 arg_name);
 473                         dma_src_offset_present = 1;
 474
 475                         status = parser_read_uint32(&p->dma_src_offset,
 476                                 arg_value);
 477                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
 478                                 params->name, arg_name, arg_value);
 479                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 480                                 params->name, arg_name, arg_value);
 481
 482                         p->dma_enabled = 1;
 483
 484                         continue;
 485                 }
 486
 487                 /* dma_size */
 488                 if (strcmp(arg_name, "dma_size") == 0) {
 489                         int status;
 490
 491                         PIPELINE_PARSE_ERR_DUPLICATE(
 492                                 dma_size_present == 0, params->name,
 493                                 arg_name);
 494                         dma_size_present = 1;
 495
 496                         status = parser_read_uint32(&p->dma_size,
 497                                 arg_value);
 498                         PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
 499                                 (p->dma_size != 0) &&
 500                                 ((p->dma_size % 8) == 0)),
 501                                 params->name, arg_name, arg_value);
 502                         PIPELINE_PARSE_ERR_OUT_RNG(((status != -ERANGE) &&
 503                                 (p->dma_size <=
 504                                 PIPELINE_PASSTHROUGH_DMA_SIZE_MAX)),
 505                                 params->name, arg_name, arg_value);
 506
 507                         p->dma_enabled = 1;
 508
 509                         continue;
 510                 }
 511
 512                 /* dma_src_mask */
 513                 if (strcmp(arg_name, "dma_src_mask") == 0) {
 514                         int mask_str_len = strlen(arg_value);
 515
 516                         PIPELINE_PARSE_ERR_DUPLICATE(
 517                                 dma_src_mask_present == 0,
 518                                 params->name, arg_name);
 519                         dma_src_mask_present = 1;
 520
 521                         PIPELINE_ARG_CHECK((mask_str_len <=
 522                                 (PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2)),
 523                                 "Parse error in section \"%s\": entry "
 524                                 "\"%s\" too long", params->name,
 525                                 arg_name);
 526
 527                         snprintf(dma_mask_str, mask_str_len + 1,
 528                                 "%s", arg_value);
 529
 530                         p->dma_enabled = 1;
 531
 532                         continue;
 533                 }
 534
 535                 /* dma_hash_offset */
 536                 if (strcmp(arg_name, "dma_hash_offset") == 0) {
 537                         int status;
 538
 539                         PIPELINE_PARSE_ERR_DUPLICATE(
 540                                 dma_hash_offset_present == 0,
 541                                 params->name, arg_name);
 542                         dma_hash_offset_present = 1;
 543
 544                         status = parser_read_uint32(&p->dma_hash_offset,
 545                                 arg_value);
 546                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
 547                                 params->name, arg_name, arg_value);
 548                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 549                                 params->name, arg_name, arg_value);
 550
 551                         p->dma_hash_enabled = 1;
 552
 553                         continue;
 554                 }
 555
 556                 /* load_balance mode */
 557                 if (strcmp(arg_name, "lb") == 0) {
 558                         PIPELINE_PARSE_ERR_DUPLICATE(
 559                                 dma_hash_lb_present == 0,
 560                                 params->name, arg_name);
 561                         dma_hash_lb_present = 1;
 562
 563                         if (strcmp(arg_value, "hash") &&
 564                                 strcmp(arg_value, "HASH"))
 565
 566                                 PIPELINE_PARSE_ERR_INV_VAL(0,
 567                                         params->name,
 568                                         arg_name,
 569                                         arg_value);
 570
 571                         p->dma_hash_lb_enabled = 1;
 572
 573                         continue;
 574                 }
 575
 576                 /* swap */
 577                 if (strcmp(arg_name, "swap") == 0) {
 578                         uint32_t a, b, n_args;
 579                         int len;
 580
 581                         n_args = sscanf(arg_value, "%" SCNu32 " %" SCNu32 "%n",
 582                                 &a, &b, &len);
 583                         PIPELINE_PARSE_ERR_INV_VAL(((n_args == 2) &&
 584                                 ((size_t) len == strlen(arg_value))),
 585                                 params->name, arg_name, arg_value);
 586
 587                         p->swap_field0_offset[p->swap_n_fields] = a;
 588                         p->swap_field1_offset[p->swap_n_fields] = b;
 589                         p->swap_n_fields++;
 590                         p->swap_enabled = 1;
 591
 592                         continue;
 593                 }
 594
 595                 /* any other */
 596                 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
 597         }
 598
 599         /* Check correlations between arguments */
 600         PIPELINE_ARG_CHECK((p->dma_enabled + p->swap_enabled < 2),
 601                 "Parse error in section \"%s\": DMA and SWAP actions are both enabled",
 602                 params->name);
 603         PIPELINE_ARG_CHECK((dma_dst_offset_present == p->dma_enabled),
 604                 "Parse error in section \"%s\": missing entry "
 605                 "\"dma_dst_offset\"", params->name);
 606         PIPELINE_ARG_CHECK((dma_src_offset_present == p->dma_enabled),
 607                 "Parse error in section \"%s\": missing entry "
 608                 "\"dma_src_offset\"", params->name);
 609         PIPELINE_ARG_CHECK((dma_size_present == p->dma_enabled),
 610                 "Parse error in section \"%s\": missing entry "
 611                 "\"dma_size\"", params->name);
 612         PIPELINE_ARG_CHECK((p->dma_hash_enabled <= p->dma_enabled),
 613                 "Parse error in section \"%s\": missing all DMA entries",
 614                 params->name);
 615         PIPELINE_ARG_CHECK((p->dma_hash_lb_enabled <= p->dma_hash_enabled),
 616                 "Parse error in section \"%s\": missing all DMA hash entries ",
 617                 params->name);
 618
 619         if (dma_src_mask_present) {
 620                 uint32_t dma_size = p->dma_size;
 621                 int status;
 622
 623                 PIPELINE_ARG_CHECK((strlen(dma_mask_str) ==
 624                         (dma_size * 2)), "Parse error in section "
 625                         "\"%s\": dma_src_mask should have exactly %u hex "
 626                         "digits", params->name, (dma_size * 2));
 627
 628                 status = parse_hex_string(dma_mask_str, p->dma_src_mask,
 629                         &p->dma_size);
 630
 631                 PIPELINE_PARSE_ERR_INV_VAL(((status == 0) &&
 632                         (dma_size == p->dma_size)), params->name,
 633                         "dma_src_mask", dma_mask_str);
 634         }
 635
 636         if (p->dma_hash_lb_enabled)
 637                 PIPELINE_ARG_CHECK((params->n_ports_out > 1),
 638                         "Parse error in section \"%s\": entry \"lb\" not "
 639                         "allowed for single output port pipeline",
 640                         params->name);
 641         else
 642                 PIPELINE_ARG_CHECK(((params->n_ports_in >= params->n_ports_out)
 643                         && ((params->n_ports_in % params->n_ports_out) == 0)),
 644                         "Parse error in section \"%s\": n_ports_in needs to be "
 645                         "a multiple of n_ports_out (lb mode disabled)",
 646                         params->name);
 647
 648         return 0;
 649 }
 650
 651 static rte_table_hash_op_hash
 652 get_hash_function(struct pipeline_passthrough *p)
 653 {
 654         switch (p->params.dma_size) {
 655
 656         case 8: return hash_default_key8;
 657         case 16: return hash_default_key16;
 658         case 24: return hash_default_key24;
 659         case 32: return hash_default_key32;
 660         case 40: return hash_default_key40;
 661         case 48: return hash_default_key48;
 662         case 56: return hash_default_key56;
 663         case 64: return hash_default_key64;
 664         default: return NULL;
 665         }
 666 }
 667
 668 static int
 669 pipeline_passthrough_swap_convert(struct pipeline_passthrough *p)
 670 {
 671         uint32_t i;
 672
 673         p->swap_n_fields = 0;
 674
 675         for (i = 0; i < p->params.swap_n_fields; i++) {
 676                 uint32_t offset0 = p->params.swap_field0_offset[i];
 677                 uint32_t offset1 = p->params.swap_field1_offset[i];
 678                 uint32_t size = offset1 - offset0;
 679                 uint32_t j;
 680
 681                 /* Check */
 682                 if ((offset0 >= offset1) ||
 683                         (size > PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX) ||
 684                         (p->swap_n_fields >= SWAP_DIM))
 685                         return -1;
 686
 687                 for (j = 0; j < (size / sizeof(uint64_t)); j++) {
 688                         p->swap_field0_offset[p->swap_n_fields] = offset0;
 689                         p->swap_field1_offset[p->swap_n_fields] = offset1;
 690                         p->swap_field_mask[p->swap_n_fields] = UINT64_MAX;
 691                         p->swap_n_fields++;
 692                         offset0 += sizeof(uint64_t);
 693                         offset1 += sizeof(uint64_t);
 694                 }
 695                 if (size % sizeof(uint64_t)) {
 696                         uint32_t n_bits = (size % sizeof(uint64_t)) * 8;
 697
 698                         p->swap_field0_offset[p->swap_n_fields] = offset0;
 699                         p->swap_field1_offset[p->swap_n_fields] = offset1;
 700                         p->swap_field_mask[p->swap_n_fields] =
 701                                 RTE_LEN2MASK(n_bits, uint64_t);
 702                         p->swap_n_fields++;
 703                 }
 704         }
 705
 706         return 0;
 707 }
 708
 709 static void*
 710 pipeline_passthrough_init(struct pipeline_params *params,
 711         __rte_unused void *arg)
 712 {
 713         struct pipeline *p;
 714         struct pipeline_passthrough *p_pt;
 715         uint32_t size, i;
 716
 717         /* Check input arguments */
 718         if ((params == NULL) ||
 719                 (params->n_ports_in == 0) ||
 720                 (params->n_ports_out == 0))
 721                 return NULL;
 722
 723         /* Memory allocation */
 724         size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_passthrough));
 725         p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
 726         p_pt = (struct pipeline_passthrough *) p;
 727         if (p == NULL)
 728                 return NULL;
 729
 730         strcpy(p->name, params->name);
 731         p->log_level = params->log_level;
 732
 733         PLOG(p, HIGH, "Pass-through");
 734
 735         /* Parse arguments */
 736         if (pipeline_passthrough_parse_args(&p_pt->params, params))
 737                 return NULL;
 738         if (pipeline_passthrough_swap_convert(p_pt))
 739                 return NULL;
 740         p_pt->f_hash = get_hash_function(p_pt);
 741
 742         /* Pipeline */
 743         {
 744                 struct rte_pipeline_params pipeline_params = {
 745                         .name = "PASS-THROUGH",
 746                         .socket_id = params->socket_id,
 747                         .offset_port_id = 0,
 748                 };
 749
 750                 p->p = rte_pipeline_create(&pipeline_params);
 751                 if (p->p == NULL) {
 752                         rte_free(p);
 753                         return NULL;
 754                 }
 755         }
 756
 757         p->n_ports_in = params->n_ports_in;
 758         p->n_ports_out = params->n_ports_out;
 759         p->n_tables = p->n_ports_in;
 760
 761         /*Input ports*/
 762         for (i = 0; i < p->n_ports_in; i++) {
 763                 struct rte_pipeline_port_in_params port_params = {
 764                         .ops = pipeline_port_in_params_get_ops(
 765                                 &params->port_in[i]),
 766                         .arg_create = pipeline_port_in_params_convert(
 767                                 &params->port_in[i]),
 768                         .f_action = get_port_in_ah(p_pt),
 769                         .arg_ah = p_pt,
 770                         .burst_size = params->port_in[i].burst_size,
 771                 };
 772
 773                 int status = rte_pipeline_port_in_create(p->p,
 774                         &port_params,
 775                         &p->port_in_id[i]);
 776
 777                 if (status) {
 778                         rte_pipeline_free(p->p);
 779                         rte_free(p);
 780                         return NULL;
 781                 }
 782         }
 783
 784         /* Output ports */
 785         for (i = 0; i < p->n_ports_out; i++) {
 786                 struct rte_pipeline_port_out_params port_params = {
 787                         .ops = pipeline_port_out_params_get_ops(
 788                                 &params->port_out[i]),
 789                         .arg_create = pipeline_port_out_params_convert(
 790                                 &params->port_out[i]),
 791                         .f_action = NULL,
 792                         .arg_ah = NULL,
 793                 };
 794
 795                 int status = rte_pipeline_port_out_create(p->p,
 796                         &port_params,
 797                         &p->port_out_id[i]);
 798
 799                 if (status) {
 800                         rte_pipeline_free(p->p);
 801                         rte_free(p);
 802                         return NULL;
 803                 }
 804         }
 805
 806         /* Tables */
 807         for (i = 0; i < p->n_ports_in; i++) {
 808                 struct rte_pipeline_table_params table_params = {
 809                         .ops = &rte_table_stub_ops,
 810                         .arg_create = NULL,
 811                         .f_action_hit = NULL,
 812                         .f_action_miss = NULL,
 813                         .arg_ah = NULL,
 814                         .action_data_size = 0,
 815                 };
 816
 817                 int status = rte_pipeline_table_create(p->p,
 818                         &table_params,
 819                         &p->table_id[i]);
 820
 821                 if (status) {
 822                         rte_pipeline_free(p->p);
 823                         rte_free(p);
 824                         return NULL;
 825                 }
 826         }
 827
 828         /* Connecting input ports to tables */
 829         for (i = 0; i < p->n_ports_in; i++) {
 830                 int status = rte_pipeline_port_in_connect_to_table(p->p,
 831                         p->port_in_id[i],
 832                         p->table_id[i]);
 833
 834                 if (status) {
 835                         rte_pipeline_free(p->p);
 836                         rte_free(p);
 837                         return NULL;
 838                 }
 839         }
 840
 841         /* Add entries to tables */
 842         for (i = 0; i < p->n_ports_in; i++) {
 843                 uint32_t port_out_id = (p_pt->params.dma_hash_lb_enabled == 0) ?
 844                         (i / (p->n_ports_in / p->n_ports_out)) :
 845                         0;
 846
 847                 struct rte_pipeline_table_entry default_entry = {
 848                         .action = RTE_PIPELINE_ACTION_PORT,
 849                         {.port_id = p->port_out_id[port_out_id]},
 850                 };
 851
 852                 struct rte_pipeline_table_entry *default_entry_ptr;
 853
 854                 int status = rte_pipeline_table_default_entry_add(p->p,
 855                         p->table_id[i],
 856                         &default_entry,
 857                         &default_entry_ptr);
 858
 859                 if (status) {
 860                         rte_pipeline_free(p->p);
 861                         rte_free(p);
 862                         return NULL;
 863                 }
 864         }
 865
 866         /* Enable input ports */
 867         for (i = 0; i < p->n_ports_in; i++) {
 868                 int status = rte_pipeline_port_in_enable(p->p,
 869                         p->port_in_id[i]);
 870
 871                 if (status) {
 872                         rte_pipeline_free(p->p);
 873                         rte_free(p);
 874                         return NULL;
 875                 }
 876         }
 877
 878         /* Check pipeline consistency */
 879         if (rte_pipeline_check(p->p) < 0) {
 880                 rte_pipeline_free(p->p);
 881                 rte_free(p);
 882                 return NULL;
 883         }
 884
 885         /* Message queues */
 886         p->n_msgq = params->n_msgq;
 887         for (i = 0; i < p->n_msgq; i++)
 888                 p->msgq_in[i] = params->msgq_in[i];
 889         for (i = 0; i < p->n_msgq; i++)
 890                 p->msgq_out[i] = params->msgq_out[i];
 891
 892         /* Message handlers */
 893         memcpy(p->handlers, handlers, sizeof(p->handlers));
 894
 895         return p;
 896 }
 897
 898 static int
 899 pipeline_passthrough_free(void *pipeline)
 900 {
 901         struct pipeline *p = (struct pipeline *) pipeline;
 902
 903         /* Check input arguments */
 904         if (p == NULL)
 905                 return -1;
 906
 907         /* Free resources */
 908         rte_pipeline_free(p->p);
 909         rte_free(p);
 910         return 0;
 911 }
 912
 913 static int
 914 pipeline_passthrough_timer(void *pipeline)
 915 {
 916         struct pipeline *p = (struct pipeline *) pipeline;
 917
 918         pipeline_msg_req_handle(p);
 919         rte_pipeline_flush(p->p);
 920
 921         return 0;
 922 }
 923
 924 struct pipeline_be_ops pipeline_passthrough_be_ops = {
 925         .f_init = pipeline_passthrough_init,
 926         .f_free = pipeline_passthrough_free,
 927         .f_run = NULL,
 928         .f_timer = pipeline_passthrough_timer,
 929 };