drivers/compress/isal/isal_compress_pmd_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4 #include <isa-l.h>
   5
   6 #include <rte_common.h>
   7 #include <rte_compressdev_pmd.h>
   8 #include <rte_malloc.h>
   9
  10 #include "isal_compress_pmd_private.h"
  11
  12 static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = {
  13         {
  14                 .algo = RTE_COMP_ALGO_DEFLATE,
  15                 .comp_feature_flags =   RTE_COMP_FF_OOP_SGL_IN_SGL_OUT |
  16                                         RTE_COMP_FF_OOP_SGL_IN_LB_OUT |
  17                                         RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
  18                                         RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
  19                                         RTE_COMP_FF_HUFFMAN_FIXED |
  20                                         RTE_COMP_FF_HUFFMAN_DYNAMIC,
  21                 .window_size = {
  22                         .min = 15,
  23                         .max = 15,
  24                         .increment = 0
  25                 },
  26         },
  27         RTE_COMP_END_OF_CAPABILITIES_LIST()
  28 };
  29
  30 /** Configure device */
  31 static int
  32 isal_comp_pmd_config(struct rte_compressdev *dev,
  33                 struct rte_compressdev_config *config)
  34 {
  35         int ret = 0;
  36         unsigned int n;
  37         char mp_name[RTE_COMPRESSDEV_NAME_MAX_LEN];
  38         unsigned int elt_size = sizeof(struct isal_priv_xform);
  39         struct isal_comp_private *internals = dev->data->dev_private;
  40
  41         n = snprintf(mp_name, sizeof(mp_name), "compdev_%d_xform_mp",
  42                         dev->data->dev_id);
  43         if (n > sizeof(mp_name)) {
  44                 ISAL_PMD_LOG(ERR,
  45                         "Unable to create unique name for xform mempool");
  46                 return -ENOMEM;
  47         }
  48
  49         internals->priv_xform_mp = rte_mempool_lookup(mp_name);
  50
  51         if (internals->priv_xform_mp != NULL) {
  52                 if (((internals->priv_xform_mp)->elt_size != elt_size) ||
  53                                 ((internals->priv_xform_mp)->size <
  54                                         config->max_nb_priv_xforms)) {
  55
  56                         ISAL_PMD_LOG(ERR, "%s mempool already exists with different"
  57                                 " initialization parameters", mp_name);
  58                         internals->priv_xform_mp = NULL;
  59                         return -ENOMEM;
  60                 }
  61         } else { /* First time configuration */
  62                 internals->priv_xform_mp = rte_mempool_create(
  63                                 mp_name, /* mempool name */
  64                                 /* number of elements*/
  65                                 config->max_nb_priv_xforms,
  66                                 elt_size, /* element size*/
  67                                 0, /* Cache size*/
  68                                 0, /* private data size */
  69                                 NULL, /* obj initialization constructor */
  70                                 NULL, /* obj initialization constructor arg */
  71                                 NULL, /**< obj constructor*/
  72                                 NULL, /* obj constructor arg */
  73                                 config->socket_id, /* socket id */
  74                                 0); /* flags */
  75         }
  76
  77         if (internals->priv_xform_mp == NULL) {
  78                 ISAL_PMD_LOG(ERR, "%s mempool allocation failed", mp_name);
  79                 return -ENOMEM;
  80         }
  81
  82         dev->data->dev_private = internals;
  83
  84         return ret;
  85 }
  86
  87 /** Start device */
  88 static int
  89 isal_comp_pmd_start(__rte_unused struct rte_compressdev *dev)
  90 {
  91         return 0;
  92 }
  93
  94 /** Stop device */
  95 static void
  96 isal_comp_pmd_stop(__rte_unused struct rte_compressdev *dev)
  97 {
  98 }
  99
 100 /** Close device */
 101 static int
 102 isal_comp_pmd_close(struct rte_compressdev *dev)
 103 {
 104         /* Free private data */
 105         struct isal_comp_private *internals = dev->data->dev_private;
 106
 107         rte_mempool_free(internals->priv_xform_mp);
 108         return 0;
 109 }
 110
 111 /** Get device statistics */
 112 static void
 113 isal_comp_pmd_stats_get(struct rte_compressdev *dev,
 114                 struct rte_compressdev_stats *stats)
 115 {
 116         uint16_t qp_id;
 117
 118         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 119                 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
 120
 121                 stats->enqueued_count += qp->qp_stats.enqueued_count;
 122                 stats->dequeued_count += qp->qp_stats.dequeued_count;
 123
 124                 stats->enqueue_err_count += qp->qp_stats.enqueue_err_count;
 125                 stats->dequeue_err_count += qp->qp_stats.dequeue_err_count;
 126         }
 127 }
 128
 129 /** Get device info */
 130 static void
 131 isal_comp_pmd_info_get(struct rte_compressdev *dev __rte_unused,
 132                 struct rte_compressdev_info *dev_info)
 133 {
 134         if (dev_info != NULL) {
 135                 dev_info->capabilities = isal_pmd_capabilities;
 136
 137                 /* Check CPU for supported vector instruction and set
 138                  * feature_flags
 139                  */
 140                 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
 141                         dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX512;
 142                 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
 143                         dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX2;
 144                 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
 145                         dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX;
 146                 else
 147                         dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_SSE;
 148         }
 149 }
 150
 151 /** Reset device statistics */
 152 static void
 153 isal_comp_pmd_stats_reset(struct rte_compressdev *dev)
 154 {
 155         uint16_t qp_id;
 156
 157         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 158                 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
 159                 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
 160         }
 161 }
 162
 163 /** Release queue pair */
 164 static int
 165 isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
 166 {
 167         struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
 168
 169         if (qp == NULL)
 170                 return -EINVAL;
 171
 172         if (qp->stream != NULL)
 173                 rte_free(qp->stream);
 174
 175         if (qp->stream->level_buf != NULL)
 176                 rte_free(qp->stream->level_buf);
 177
 178         if (qp->state != NULL)
 179                 rte_free(qp->state);
 180
 181         if (qp->processed_pkts != NULL)
 182                 rte_ring_free(qp->processed_pkts);
 183
 184         rte_free(qp);
 185         dev->data->queue_pairs[qp_id] = NULL;
 186
 187         return 0;
 188 }
 189
 190 /** Create a ring to place process packets on */
 191 static struct rte_ring *
 192 isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp *qp,
 193                 unsigned int ring_size, int socket_id)
 194 {
 195         struct rte_ring *r;
 196
 197         r = rte_ring_lookup(qp->name);
 198         if (r) {
 199                 if (rte_ring_get_size(r) >= ring_size) {
 200                         ISAL_PMD_LOG(DEBUG,
 201                                 "Reusing existing ring %s for processed packets",
 202                                 qp->name);
 203                         return r;
 204                 }
 205
 206                         ISAL_PMD_LOG(ERR,
 207                                 "Unable to reuse existing ring %s"
 208                                 " for processed packets",
 209                          qp->name);
 210                 return NULL;
 211         }
 212
 213         return rte_ring_create(qp->name, ring_size, socket_id,
 214                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 215 }
 216
 217 /** set a unique name for the queue pair based on its name, dev_id and qp_id */
 218 static int
 219 isal_comp_pmd_qp_set_unique_name(struct rte_compressdev *dev,
 220 struct isal_comp_qp *qp)
 221 {
 222         unsigned int n = snprintf(qp->name, sizeof(qp->name),
 223                         "isal_compression_pmd_%u_qp_%u",
 224                         dev->data->dev_id, qp->id);
 225
 226         if (n >= sizeof(qp->name))
 227                 return -1;
 228
 229         return 0;
 230 }
 231
 232 /* Setup a queue pair */
 233 static int
 234 isal_comp_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
 235                 uint32_t max_inflight_ops, int socket_id)
 236 {
 237         struct isal_comp_qp *qp = NULL;
 238         int retval;
 239
 240         /* Free memory prior to re-allocation if needed. */
 241         if (dev->data->queue_pairs[qp_id] != NULL)
 242                 isal_comp_pmd_qp_release(dev, qp_id);
 243
 244         /* Allocate the queue pair data structure. */
 245         qp = rte_zmalloc_socket("Isa-l compression PMD Queue Pair", sizeof(*qp),
 246                                         RTE_CACHE_LINE_SIZE, socket_id);
 247         if (qp == NULL) {
 248                 ISAL_PMD_LOG(ERR, "Failed to allocate queue pair memory");
 249                 return (-ENOMEM);
 250         }
 251
 252         /* Initialize memory for compression stream structure */
 253         qp->stream = rte_zmalloc_socket("Isa-l compression stream ",
 254                         sizeof(struct isal_zstream),  RTE_CACHE_LINE_SIZE,
 255                         socket_id);
 256
 257         /* Initialize memory for compression level buffer */
 258         qp->stream->level_buf = rte_zmalloc_socket("Isa-l compression lev_buf",
 259                         ISAL_DEF_LVL3_DEFAULT, RTE_CACHE_LINE_SIZE,
 260                         socket_id);
 261
 262         /* Initialize memory for decompression state structure */
 263         qp->state = rte_zmalloc_socket("Isa-l decompression state",
 264                         sizeof(struct inflate_state), RTE_CACHE_LINE_SIZE,
 265                         socket_id);
 266
 267         qp->id = qp_id;
 268         dev->data->queue_pairs[qp_id] = qp;
 269
 270         retval = isal_comp_pmd_qp_set_unique_name(dev, qp);
 271         if (retval) {
 272                 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal "
 273                                 "compression device");
 274                 goto qp_setup_cleanup;
 275         }
 276
 277         qp->processed_pkts = isal_comp_pmd_qp_create_processed_pkts_ring(qp,
 278                         max_inflight_ops, socket_id);
 279         if (qp->processed_pkts == NULL) {
 280                 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal "
 281                                 "compression device");
 282                 goto qp_setup_cleanup;
 283         }
 284
 285         qp->num_free_elements = rte_ring_free_count(qp->processed_pkts);
 286
 287         memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
 288         return 0;
 289
 290 qp_setup_cleanup:
 291         if (qp)
 292                 rte_free(qp);
 293
 294         return -1;
 295 }
 296
 297 /** Set private xform data*/
 298 static int
 299 isal_comp_pmd_priv_xform_create(struct rte_compressdev *dev,
 300                         const struct rte_comp_xform *xform, void **priv_xform)
 301 {
 302         int ret;
 303         struct isal_comp_private *internals = dev->data->dev_private;
 304
 305         if (xform == NULL) {
 306                 ISAL_PMD_LOG(ERR, "Invalid Xform struct");
 307                 return -EINVAL;
 308         }
 309
 310         if (rte_mempool_get(internals->priv_xform_mp, priv_xform)) {
 311                 ISAL_PMD_LOG(ERR,
 312                         "Couldn't get object from private xform mempool");
 313                 return -ENOMEM;
 314         }
 315
 316         ret = isal_comp_set_priv_xform_parameters(*priv_xform, xform);
 317         if (ret != 0) {
 318                 ISAL_PMD_LOG(ERR, "Failed to configure private xform parameters");
 319
 320                 /* Return private xform to mempool */
 321                 rte_mempool_put(internals->priv_xform_mp, priv_xform);
 322                 return ret;
 323         }
 324         return 0;
 325 }
 326
 327 /** Clear memory of the private xform so it doesn't leave key material behind */
 328 static int
 329 isal_comp_pmd_priv_xform_free(struct rte_compressdev *dev, void *priv_xform)
 330 {
 331         struct isal_comp_private *internals = dev->data->dev_private;
 332
 333         /* Zero out the whole structure */
 334         if (priv_xform) {
 335                 memset(priv_xform, 0, sizeof(struct isal_priv_xform));
 336                 rte_mempool_put(internals->priv_xform_mp, priv_xform);
 337         }
 338         return 0;
 339 }
 340
 341 struct rte_compressdev_ops isal_pmd_ops = {
 342                 .dev_configure          = isal_comp_pmd_config,
 343                 .dev_start              = isal_comp_pmd_start,
 344                 .dev_stop               = isal_comp_pmd_stop,
 345                 .dev_close              = isal_comp_pmd_close,
 346
 347                 .stats_get              = isal_comp_pmd_stats_get,
 348                 .stats_reset            = isal_comp_pmd_stats_reset,
 349
 350                 .dev_infos_get          = isal_comp_pmd_info_get,
 351
 352                 .queue_pair_setup       = isal_comp_pmd_qp_setup,
 353                 .queue_pair_release     = isal_comp_pmd_qp_release,
 354
 355                 .private_xform_create   = isal_comp_pmd_priv_xform_create,
 356                 .private_xform_free     = isal_comp_pmd_priv_xform_free,
 357 };
 358
 359 struct rte_compressdev_ops *isal_compress_pmd_ops = &isal_pmd_ops;