lib/librte_port/rte_port_sym_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4 #include <string.h>
   5
   6 #include <rte_common.h>
   7 #include <rte_malloc.h>
   8
   9 #include "rte_port_sym_crypto.h"
  10
  11 /*
  12  * Port Crypto Reader
  13  */
  14 #ifdef RTE_PORT_STATS_COLLECT
  15
  16 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) \
  17         (port)->stats.n_pkts_in += (val)
  18 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) \
  19         (port)->stats.n_pkts_drop += (val)
  20
  21 #else
  22
  23 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val)
  24 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val)
  25
  26 #endif
  27
  28 struct rte_port_sym_crypto_reader {
  29         struct rte_port_in_stats stats;
  30
  31         uint8_t cryptodev_id;
  32         uint16_t queue_id;
  33         struct rte_crypto_op *ops[RTE_PORT_IN_BURST_SIZE_MAX];
  34         rte_port_sym_crypto_reader_callback_fn f_callback;
  35         void *arg_callback;
  36 };
  37
  38 static void *
  39 rte_port_sym_crypto_reader_create(void *params, int socket_id)
  40 {
  41         struct rte_port_sym_crypto_reader_params *conf =
  42                         params;
  43         struct rte_port_sym_crypto_reader *port;
  44
  45         /* Check input parameters */
  46         if (conf == NULL) {
  47                 RTE_LOG(ERR, PORT, "%s: params is NULL\n", __func__);
  48                 return NULL;
  49         }
  50
  51         /* Memory allocation */
  52         port = rte_zmalloc_socket("PORT", sizeof(*port),
  53                 RTE_CACHE_LINE_SIZE, socket_id);
  54         if (port == NULL) {
  55                 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
  56                 return NULL;
  57         }
  58
  59         /* Initialization */
  60         port->cryptodev_id = conf->cryptodev_id;
  61         port->queue_id = conf->queue_id;
  62         port->f_callback = conf->f_callback;
  63         port->arg_callback = conf->arg_callback;
  64
  65         return port;
  66 }
  67
  68 static int
  69 rte_port_sym_crypto_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts)
  70 {
  71         struct rte_port_sym_crypto_reader *p =
  72                         port;
  73         uint16_t rx_ops_cnt, i, n = 0;
  74
  75         rx_ops_cnt = rte_cryptodev_dequeue_burst(p->cryptodev_id, p->queue_id,
  76                         p->ops, n_pkts);
  77
  78         for (i = 0; i < rx_ops_cnt; i++) {
  79                 struct rte_crypto_op *op = p->ops[i];
  80
  81                 /** Drop failed pkts */
  82                 if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) {
  83                         rte_pktmbuf_free(op->sym->m_src);
  84                         continue;
  85                 }
  86
  87                 pkts[n++] = op->sym->m_src;
  88         }
  89
  90         if (p->f_callback)
  91                 (*p->f_callback)(pkts, n, p->arg_callback);
  92
  93         RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(p, n);
  94         RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(p, rx_ops_cnt - n);
  95
  96         return n;
  97 }
  98
  99 static int
 100 rte_port_sym_crypto_reader_free(void *port)
 101 {
 102         if (port == NULL) {
 103                 RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__);
 104                 return -EINVAL;
 105         }
 106
 107         rte_free(port);
 108
 109         return 0;
 110 }
 111
 112 static int rte_port_sym_crypto_reader_stats_read(void *port,
 113         struct rte_port_in_stats *stats, int clear)
 114 {
 115         struct rte_port_sym_crypto_reader *p =
 116                         port;
 117
 118         if (stats != NULL)
 119                 memcpy(stats, &p->stats, sizeof(p->stats));
 120
 121         if (clear)
 122                 memset(&p->stats, 0, sizeof(p->stats));
 123
 124         return 0;
 125 }
 126
 127 /*
 128  * Port crypto Writer
 129  */
 130 #ifdef RTE_PORT_STATS_COLLECT
 131
 132 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) \
 133         (port)->stats.n_pkts_in += (val)
 134 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) \
 135         (port)->stats.n_pkts_drop += (val)
 136
 137 #else
 138
 139 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val)
 140 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val)
 141
 142 #endif
 143
 144 struct rte_port_sym_crypto_writer {
 145         struct rte_port_out_stats stats;
 146
 147         struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX];
 148
 149         uint32_t tx_burst_sz;
 150         uint32_t tx_buf_count;
 151         uint64_t bsz_mask;
 152
 153         uint8_t cryptodev_id;
 154         uint16_t queue_id;
 155         uint16_t crypto_op_offset;
 156 };
 157
 158 static void *
 159 rte_port_sym_crypto_writer_create(void *params, int socket_id)
 160 {
 161         struct rte_port_sym_crypto_writer_params *conf =
 162                         params;
 163         struct rte_port_sym_crypto_writer *port;
 164
 165         /* Check input parameters */
 166         if ((conf == NULL) ||
 167                 (conf->tx_burst_sz == 0) ||
 168                 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
 169                 (!rte_is_power_of_2(conf->tx_burst_sz))) {
 170                 RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
 171                 return NULL;
 172         }
 173
 174         /* Memory allocation */
 175         port = rte_zmalloc_socket("PORT", sizeof(*port),
 176                 RTE_CACHE_LINE_SIZE, socket_id);
 177         if (port == NULL) {
 178                 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
 179                 return NULL;
 180         }
 181
 182         /* Initialization */
 183         port->tx_burst_sz = conf->tx_burst_sz;
 184         port->tx_buf_count = 0;
 185         port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1);
 186
 187         port->cryptodev_id = conf->cryptodev_id;
 188         port->queue_id = conf->queue_id;
 189         port->crypto_op_offset = conf->crypto_op_offset;
 190
 191         return port;
 192 }
 193
 194 static inline void
 195 send_burst(struct rte_port_sym_crypto_writer *p)
 196 {
 197         uint32_t nb_tx;
 198
 199         nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id,
 200                         p->tx_buf, p->tx_buf_count);
 201
 202         RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count -
 203                         nb_tx);
 204         for (; nb_tx < p->tx_buf_count; nb_tx++)
 205                 rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src);
 206
 207         p->tx_buf_count = 0;
 208 }
 209
 210 static int
 211 rte_port_sym_crypto_writer_tx(void *port, struct rte_mbuf *pkt)
 212 {
 213         struct rte_port_sym_crypto_writer *p =
 214                         port;
 215
 216         p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
 217                         RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset);
 218         RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
 219         if (p->tx_buf_count >= p->tx_burst_sz)
 220                 send_burst(p);
 221
 222         return 0;
 223 }
 224
 225 static int
 226 rte_port_sym_crypto_writer_tx_bulk(void *port,
 227         struct rte_mbuf **pkts,
 228         uint64_t pkts_mask)
 229 {
 230         struct rte_port_sym_crypto_writer *p =
 231                         port;
 232         uint64_t bsz_mask = p->bsz_mask;
 233         uint32_t tx_buf_count = p->tx_buf_count;
 234         uint64_t expr = (pkts_mask & (pkts_mask + 1)) |
 235                                         ((pkts_mask & bsz_mask) ^ bsz_mask);
 236
 237         if (expr == 0) {
 238                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
 239                 uint32_t i;
 240
 241                 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, n_pkts);
 242
 243                 for (i = 0; i < n_pkts; i++)
 244                         p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
 245                                         RTE_MBUF_METADATA_UINT8_PTR(pkts[i],
 246                                                         p->crypto_op_offset);
 247
 248                 if (p->tx_buf_count >= p->tx_burst_sz)
 249                         send_burst(p);
 250         } else {
 251                 for (; pkts_mask;) {
 252                         uint32_t pkt_index = __builtin_ctzll(pkts_mask);
 253                         uint64_t pkt_mask = 1LLU << pkt_index;
 254                         struct rte_mbuf *pkt = pkts[pkt_index];
 255
 256                         p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *)
 257                                         RTE_MBUF_METADATA_UINT8_PTR(pkt,
 258                                                         p->crypto_op_offset);
 259
 260                         RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
 261                         pkts_mask &= ~pkt_mask;
 262                 }
 263
 264                 p->tx_buf_count = tx_buf_count;
 265                 if (tx_buf_count >= p->tx_burst_sz)
 266                         send_burst(p);
 267         }
 268
 269         return 0;
 270 }
 271
 272 static int
 273 rte_port_sym_crypto_writer_flush(void *port)
 274 {
 275         struct rte_port_sym_crypto_writer *p =
 276                         port;
 277
 278         if (p->tx_buf_count > 0)
 279                 send_burst(p);
 280
 281         return 0;
 282 }
 283
 284 static int
 285 rte_port_sym_crypto_writer_free(void *port)
 286 {
 287         if (port == NULL) {
 288                 RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__);
 289                 return -EINVAL;
 290         }
 291
 292         rte_port_sym_crypto_writer_flush(port);
 293         rte_free(port);
 294
 295         return 0;
 296 }
 297
 298 static int rte_port_sym_crypto_writer_stats_read(void *port,
 299         struct rte_port_out_stats *stats, int clear)
 300 {
 301         struct rte_port_sym_crypto_writer *p =
 302                         port;
 303
 304         if (stats != NULL)
 305                 memcpy(stats, &p->stats, sizeof(p->stats));
 306
 307         if (clear)
 308                 memset(&p->stats, 0, sizeof(p->stats));
 309
 310         return 0;
 311 }
 312
 313 /*
 314  * Port crypto Writer Nodrop
 315  */
 316 #ifdef RTE_PORT_STATS_COLLECT
 317
 318 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \
 319         (port)->stats.n_pkts_in += (val)
 320 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \
 321         (port)->stats.n_pkts_drop += (val)
 322
 323 #else
 324
 325 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val)
 326 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val)
 327
 328 #endif
 329
 330 struct rte_port_sym_crypto_writer_nodrop {
 331         struct rte_port_out_stats stats;
 332
 333         struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX];
 334         uint32_t tx_burst_sz;
 335         uint32_t tx_buf_count;
 336         uint64_t bsz_mask;
 337         uint64_t n_retries;
 338
 339         uint8_t cryptodev_id;
 340         uint16_t queue_id;
 341         uint16_t crypto_op_offset;
 342 };
 343
 344 static void *
 345 rte_port_sym_crypto_writer_nodrop_create(void *params, int socket_id)
 346 {
 347         struct rte_port_sym_crypto_writer_nodrop_params *conf =
 348                 params;
 349         struct rte_port_sym_crypto_writer_nodrop *port;
 350
 351         /* Check input parameters */
 352         if ((conf == NULL) ||
 353                 (conf->tx_burst_sz == 0) ||
 354                 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
 355                 (!rte_is_power_of_2(conf->tx_burst_sz))) {
 356                 RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
 357                 return NULL;
 358         }
 359
 360         /* Memory allocation */
 361         port = rte_zmalloc_socket("PORT", sizeof(*port),
 362                 RTE_CACHE_LINE_SIZE, socket_id);
 363         if (port == NULL) {
 364                 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
 365                 return NULL;
 366         }
 367
 368         /* Initialization */
 369         port->cryptodev_id = conf->cryptodev_id;
 370         port->queue_id = conf->queue_id;
 371         port->crypto_op_offset = conf->crypto_op_offset;
 372         port->tx_burst_sz = conf->tx_burst_sz;
 373         port->tx_buf_count = 0;
 374         port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1);
 375
 376         /*
 377          * When n_retries is 0 it means that we should wait for every packet to
 378          * send no matter how many retries should it take. To limit number of
 379          * branches in fast path, we use UINT64_MAX instead of branching.
 380          */
 381         port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries;
 382
 383         return port;
 384 }
 385
 386 static inline void
 387 send_burst_nodrop(struct rte_port_sym_crypto_writer_nodrop *p)
 388 {
 389         uint32_t nb_tx = 0, i;
 390
 391         nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id,
 392                         p->tx_buf, p->tx_buf_count);
 393
 394         /* We sent all the packets in a first try */
 395         if (nb_tx >= p->tx_buf_count) {
 396                 p->tx_buf_count = 0;
 397                 return;
 398         }
 399
 400         for (i = 0; i < p->n_retries; i++) {
 401                 nb_tx += rte_cryptodev_enqueue_burst(p->cryptodev_id,
 402                                 p->queue_id, p->tx_buf + nb_tx,
 403                                 p->tx_buf_count - nb_tx);
 404
 405                 /* We sent all the packets in more than one try */
 406                 if (nb_tx >= p->tx_buf_count) {
 407                         p->tx_buf_count = 0;
 408                         return;
 409                 }
 410         }
 411
 412         /* We didn't send the packets in maximum allowed attempts */
 413         RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(p,
 414                         p->tx_buf_count - nb_tx);
 415         for ( ; nb_tx < p->tx_buf_count; nb_tx++)
 416                 rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src);
 417
 418         p->tx_buf_count = 0;
 419 }
 420
 421 static int
 422 rte_port_sym_crypto_writer_nodrop_tx(void *port, struct rte_mbuf *pkt)
 423 {
 424         struct rte_port_sym_crypto_writer_nodrop *p =
 425                         port;
 426
 427         p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
 428                         RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset);
 429         RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
 430         if (p->tx_buf_count >= p->tx_burst_sz)
 431                 send_burst_nodrop(p);
 432
 433         return 0;
 434 }
 435
 436 static int
 437 rte_port_sym_crypto_writer_nodrop_tx_bulk(void *port,
 438         struct rte_mbuf **pkts,
 439         uint64_t pkts_mask)
 440 {
 441         struct rte_port_sym_crypto_writer_nodrop *p =
 442                         port;
 443
 444         uint64_t bsz_mask = p->bsz_mask;
 445         uint32_t tx_buf_count = p->tx_buf_count;
 446         uint64_t expr = (pkts_mask & (pkts_mask + 1)) |
 447                                         ((pkts_mask & bsz_mask) ^ bsz_mask);
 448
 449         if (expr == 0) {
 450                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
 451                 uint32_t i;
 452
 453                 RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts);
 454
 455                 for (i = 0; i < n_pkts; i++)
 456                         p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
 457                                         RTE_MBUF_METADATA_UINT8_PTR(pkts[i],
 458                                                         p->crypto_op_offset);
 459
 460                 if (p->tx_buf_count >= p->tx_burst_sz)
 461                         send_burst_nodrop(p);
 462         } else {
 463                 for ( ; pkts_mask; ) {
 464                         uint32_t pkt_index = __builtin_ctzll(pkts_mask);
 465                         uint64_t pkt_mask = 1LLU << pkt_index;
 466                         struct rte_mbuf *pkt = pkts[pkt_index];
 467
 468                         p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *)
 469                                         RTE_MBUF_METADATA_UINT8_PTR(pkt,
 470                                                         p->crypto_op_offset);
 471                         RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p,
 472                                         1);
 473                         pkts_mask &= ~pkt_mask;
 474                 }
 475
 476                 p->tx_buf_count = tx_buf_count;
 477                 if (tx_buf_count >= p->tx_burst_sz)
 478                         send_burst_nodrop(p);
 479         }
 480
 481         return 0;
 482 }
 483
 484 static int
 485 rte_port_sym_crypto_writer_nodrop_flush(void *port)
 486 {
 487         struct rte_port_sym_crypto_writer_nodrop *p =
 488                 port;
 489
 490         if (p->tx_buf_count > 0)
 491                 send_burst_nodrop(p);
 492
 493         return 0;
 494 }
 495
 496 static int
 497 rte_port_sym_crypto_writer_nodrop_free(void *port)
 498 {
 499         if (port == NULL) {
 500                 RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__);
 501                 return -EINVAL;
 502         }
 503
 504         rte_port_sym_crypto_writer_nodrop_flush(port);
 505         rte_free(port);
 506
 507         return 0;
 508 }
 509
 510 static int rte_port_sym_crypto_writer_nodrop_stats_read(void *port,
 511         struct rte_port_out_stats *stats, int clear)
 512 {
 513         struct rte_port_sym_crypto_writer_nodrop *p =
 514                         port;
 515
 516         if (stats != NULL)
 517                 memcpy(stats, &p->stats, sizeof(p->stats));
 518
 519         if (clear)
 520                 memset(&p->stats, 0, sizeof(p->stats));
 521
 522         return 0;
 523 }
 524
 525
 526 /*
 527  * Summary of port operations
 528  */
 529 struct rte_port_in_ops rte_port_sym_crypto_reader_ops = {
 530         .f_create = rte_port_sym_crypto_reader_create,
 531         .f_free = rte_port_sym_crypto_reader_free,
 532         .f_rx = rte_port_sym_crypto_reader_rx,
 533         .f_stats = rte_port_sym_crypto_reader_stats_read,
 534 };
 535
 536 struct rte_port_out_ops rte_port_sym_crypto_writer_ops = {
 537         .f_create = rte_port_sym_crypto_writer_create,
 538         .f_free = rte_port_sym_crypto_writer_free,
 539         .f_tx = rte_port_sym_crypto_writer_tx,
 540         .f_tx_bulk = rte_port_sym_crypto_writer_tx_bulk,
 541         .f_flush = rte_port_sym_crypto_writer_flush,
 542         .f_stats = rte_port_sym_crypto_writer_stats_read,
 543 };
 544
 545 struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops = {
 546         .f_create = rte_port_sym_crypto_writer_nodrop_create,
 547         .f_free = rte_port_sym_crypto_writer_nodrop_free,
 548         .f_tx = rte_port_sym_crypto_writer_nodrop_tx,
 549         .f_tx_bulk = rte_port_sym_crypto_writer_nodrop_tx_bulk,
 550         .f_flush = rte_port_sym_crypto_writer_nodrop_flush,
 551         .f_stats = rte_port_sym_crypto_writer_nodrop_stats_read,
 552 };