drivers/net/ixgbe/ixgbe_ipsec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_ip.h>
   8 #include <rte_jhash.h>
   9 #include <rte_security_driver.h>
  10 #include <rte_cryptodev.h>
  11 #include <rte_flow.h>
  12
  13 #include "base/ixgbe_type.h"
  14 #include "base/ixgbe_api.h"
  15 #include "ixgbe_ethdev.h"
  16 #include "ixgbe_ipsec.h"
  17
  18 #define RTE_IXGBE_REGISTER_POLL_WAIT_5_MS  5
  19
  20 #define IXGBE_WAIT_RREAD \
  21         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \
  22         IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  23 #define IXGBE_WAIT_RWRITE \
  24         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \
  25         IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  26 #define IXGBE_WAIT_TREAD \
  27         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \
  28         IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  29 #define IXGBE_WAIT_TWRITE \
  30         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \
  31         IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  32
  33 #define CMP_IP(a, b) (\
  34         (a).ipv6[0] == (b).ipv6[0] && \
  35         (a).ipv6[1] == (b).ipv6[1] && \
  36         (a).ipv6[2] == (b).ipv6[2] && \
  37         (a).ipv6[3] == (b).ipv6[3])
  38
  39
  40 static void
  41 ixgbe_crypto_clear_ipsec_tables(struct rte_eth_dev *dev)
  42 {
  43         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
  44         struct ixgbe_ipsec *priv = IXGBE_DEV_PRIVATE_TO_IPSEC(
  45                                 dev->data->dev_private);
  46         int i = 0;
  47
  48         /* clear Rx IP table*/
  49         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
  50                 uint16_t index = i << 3;
  51                 uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP | index;
  52                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
  53                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
  54                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
  55                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0);
  56                 IXGBE_WAIT_RWRITE;
  57         }
  58
  59         /* clear Rx SPI and Rx/Tx SA tables*/
  60         for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
  61                 uint32_t index = i << 3;
  62                 uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | index;
  63                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0);
  64                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0);
  65                 IXGBE_WAIT_RWRITE;
  66                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | index;
  67                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0);
  68                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0);
  69                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0);
  70                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0);
  71                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0);
  72                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0);
  73                 IXGBE_WAIT_RWRITE;
  74                 reg_val = IPSRXIDX_WRITE | index;
  75                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0);
  76                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0);
  77                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0);
  78                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0);
  79                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0);
  80                 IXGBE_WAIT_TWRITE;
  81         }
  82
  83         memset(priv->rx_ip_tbl, 0, sizeof(priv->rx_ip_tbl));
  84         memset(priv->rx_sa_tbl, 0, sizeof(priv->rx_sa_tbl));
  85         memset(priv->tx_sa_tbl, 0, sizeof(priv->tx_sa_tbl));
  86 }
  87
  88 static int
  89 ixgbe_crypto_add_sa(struct ixgbe_crypto_session *ic_session)
  90 {
  91         struct rte_eth_dev *dev = ic_session->dev;
  92         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
  93         struct ixgbe_ipsec *priv = IXGBE_DEV_PRIVATE_TO_IPSEC(
  94                         dev->data->dev_private);
  95         uint32_t reg_val;
  96         int sa_index = -1;
  97
  98         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
  99                 int i, ip_index = -1;
 100
 101                 /* Find a match in the IP table*/
 102                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
 103                         if (CMP_IP(priv->rx_ip_tbl[i].ip,
 104                                    ic_session->dst_ip)) {
 105                                 ip_index = i;
 106                                 break;
 107                         }
 108                 }
 109                 /* If no match, find a free entry in the IP table*/
 110                 if (ip_index < 0) {
 111                         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
 112                                 if (priv->rx_ip_tbl[i].ref_count == 0) {
 113                                         ip_index = i;
 114                                         break;
 115                                 }
 116                         }
 117                 }
 118
 119                 /* Fail if no match and no free entries*/
 120                 if (ip_index < 0) {
 121                         PMD_DRV_LOG(ERR,
 122                                     "No free entry left in the Rx IP table\n");
 123                         return -1;
 124                 }
 125
 126                 /* Find a free entry in the SA table*/
 127                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 128                         if (priv->rx_sa_tbl[i].used == 0) {
 129                                 sa_index = i;
 130                                 break;
 131                         }
 132                 }
 133                 /* Fail if no free entries*/
 134                 if (sa_index < 0) {
 135                         PMD_DRV_LOG(ERR,
 136                                     "No free entry left in the Rx SA table\n");
 137                         return -1;
 138                 }
 139
 140                 priv->rx_ip_tbl[ip_index].ip.ipv6[0] =
 141                                 ic_session->dst_ip.ipv6[0];
 142                 priv->rx_ip_tbl[ip_index].ip.ipv6[1] =
 143                                 ic_session->dst_ip.ipv6[1];
 144                 priv->rx_ip_tbl[ip_index].ip.ipv6[2] =
 145                                 ic_session->dst_ip.ipv6[2];
 146                 priv->rx_ip_tbl[ip_index].ip.ipv6[3] =
 147                                 ic_session->dst_ip.ipv6[3];
 148                 priv->rx_ip_tbl[ip_index].ref_count++;
 149
 150                 priv->rx_sa_tbl[sa_index].spi =
 151                         rte_cpu_to_be_32(ic_session->spi);
 152                 priv->rx_sa_tbl[sa_index].ip_index = ip_index;
 153                 priv->rx_sa_tbl[sa_index].mode = IPSRXMOD_VALID;
 154                 if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION)
 155                         priv->rx_sa_tbl[sa_index].mode |=
 156                                         (IPSRXMOD_PROTO | IPSRXMOD_DECRYPT);
 157                 if (ic_session->dst_ip.type == IPv6)
 158                         priv->rx_sa_tbl[sa_index].mode |= IPSRXMOD_IPV6;
 159                 priv->rx_sa_tbl[sa_index].used = 1;
 160
 161                 /* write IP table entry*/
 162                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
 163                                 IPSRXIDX_TABLE_IP | (ip_index << 3);
 164                 if (priv->rx_ip_tbl[ip_index].ip.type == IPv4) {
 165                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
 166                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
 167                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
 168                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3),
 169                                         priv->rx_ip_tbl[ip_index].ip.ipv4);
 170                 } else {
 171                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0),
 172                                         priv->rx_ip_tbl[ip_index].ip.ipv6[0]);
 173                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1),
 174                                         priv->rx_ip_tbl[ip_index].ip.ipv6[1]);
 175                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2),
 176                                         priv->rx_ip_tbl[ip_index].ip.ipv6[2]);
 177                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3),
 178                                         priv->rx_ip_tbl[ip_index].ip.ipv6[3]);
 179                 }
 180                 IXGBE_WAIT_RWRITE;
 181
 182                 /* write SPI table entry*/
 183                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
 184                                 IPSRXIDX_TABLE_SPI | (sa_index << 3);
 185                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI,
 186                                 priv->rx_sa_tbl[sa_index].spi);
 187                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX,
 188                                 priv->rx_sa_tbl[sa_index].ip_index);
 189                 IXGBE_WAIT_RWRITE;
 190
 191                 /* write Key table entry*/
 192                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
 193                                 IPSRXIDX_TABLE_KEY | (sa_index << 3);
 194                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0),
 195                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[12]));
 196                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1),
 197                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[8]));
 198                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2),
 199                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[4]));
 200                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3),
 201                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[0]));
 202                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT,
 203                                 rte_cpu_to_be_32(ic_session->salt));
 204                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD,
 205                                 priv->rx_sa_tbl[sa_index].mode);
 206                 IXGBE_WAIT_RWRITE;
 207
 208         } else { /* sess->dir == RTE_CRYPTO_OUTBOUND */
 209                 int i;
 210
 211                 /* Find a free entry in the SA table*/
 212                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 213                         if (priv->tx_sa_tbl[i].used == 0) {
 214                                 sa_index = i;
 215                                 break;
 216                         }
 217                 }
 218                 /* Fail if no free entries*/
 219                 if (sa_index < 0) {
 220                         PMD_DRV_LOG(ERR,
 221                                     "No free entry left in the Tx SA table\n");
 222                         return -1;
 223                 }
 224
 225                 priv->tx_sa_tbl[sa_index].spi =
 226                         rte_cpu_to_be_32(ic_session->spi);
 227                 priv->tx_sa_tbl[i].used = 1;
 228                 ic_session->sa_index = sa_index;
 229
 230                 /* write Key table entry*/
 231                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE | (sa_index << 3);
 232                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0),
 233                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[12]));
 234                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1),
 235                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[8]));
 236                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2),
 237                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[4]));
 238                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3),
 239                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[0]));
 240                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT,
 241                                 rte_cpu_to_be_32(ic_session->salt));
 242                 IXGBE_WAIT_TWRITE;
 243         }
 244
 245         return 0;
 246 }
 247
 248 static int
 249 ixgbe_crypto_remove_sa(struct rte_eth_dev *dev,
 250                        struct ixgbe_crypto_session *ic_session)
 251 {
 252         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 253         struct ixgbe_ipsec *priv =
 254                         IXGBE_DEV_PRIVATE_TO_IPSEC(dev->data->dev_private);
 255         uint32_t reg_val;
 256         int sa_index = -1;
 257
 258         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
 259                 int i, ip_index = -1;
 260
 261                 /* Find a match in the IP table*/
 262                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
 263                         if (CMP_IP(priv->rx_ip_tbl[i].ip, ic_session->dst_ip)) {
 264                                 ip_index = i;
 265                                 break;
 266                         }
 267                 }
 268
 269                 /* Fail if no match*/
 270                 if (ip_index < 0) {
 271                         PMD_DRV_LOG(ERR,
 272                                     "Entry not found in the Rx IP table\n");
 273                         return -1;
 274                 }
 275
 276                 /* Find a free entry in the SA table*/
 277                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 278                         if (priv->rx_sa_tbl[i].spi ==
 279                                   rte_cpu_to_be_32(ic_session->spi)) {
 280                                 sa_index = i;
 281                                 break;
 282                         }
 283                 }
 284                 /* Fail if no match*/
 285                 if (sa_index < 0) {
 286                         PMD_DRV_LOG(ERR,
 287                                     "Entry not found in the Rx SA table\n");
 288                         return -1;
 289                 }
 290
 291                 /* Disable and clear Rx SPI and key table table entryes*/
 292                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | (sa_index << 3);
 293                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0);
 294                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0);
 295                 IXGBE_WAIT_RWRITE;
 296                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | (sa_index << 3);
 297                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0);
 298                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0);
 299                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0);
 300                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0);
 301                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0);
 302                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0);
 303                 IXGBE_WAIT_RWRITE;
 304                 priv->rx_sa_tbl[sa_index].used = 0;
 305
 306                 /* If last used then clear the IP table entry*/
 307                 priv->rx_ip_tbl[ip_index].ref_count--;
 308                 if (priv->rx_ip_tbl[ip_index].ref_count == 0) {
 309                         reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP |
 310                                         (ip_index << 3);
 311                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
 312                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
 313                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
 314                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0);
 315                 }
 316         } else { /* session->dir == RTE_CRYPTO_OUTBOUND */
 317                 int i;
 318
 319                 /* Find a match in the SA table*/
 320                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 321                         if (priv->tx_sa_tbl[i].spi ==
 322                                     rte_cpu_to_be_32(ic_session->spi)) {
 323                                 sa_index = i;
 324                                 break;
 325                         }
 326                 }
 327                 /* Fail if no match entries*/
 328                 if (sa_index < 0) {
 329                         PMD_DRV_LOG(ERR,
 330                                     "Entry not found in the Tx SA table\n");
 331                         return -1;
 332                 }
 333                 reg_val = IPSRXIDX_WRITE | (sa_index << 3);
 334                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0);
 335                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0);
 336                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0);
 337                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0);
 338                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0);
 339                 IXGBE_WAIT_TWRITE;
 340
 341                 priv->tx_sa_tbl[sa_index].used = 0;
 342         }
 343
 344         return 0;
 345 }
 346
 347 static int
 348 ixgbe_crypto_create_session(void *device,
 349                 struct rte_security_session_conf *conf,
 350                 struct rte_security_session *session,
 351                 struct rte_mempool *mempool)
 352 {
 353         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
 354         struct ixgbe_crypto_session *ic_session = NULL;
 355         struct rte_crypto_aead_xform *aead_xform;
 356         struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
 357
 358         if (rte_mempool_get(mempool, (void **)&ic_session)) {
 359                 PMD_DRV_LOG(ERR, "Cannot get object from ic_session mempool");
 360                 return -ENOMEM;
 361         }
 362
 363         if (conf->crypto_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD ||
 364                         conf->crypto_xform->aead.algo !=
 365                                         RTE_CRYPTO_AEAD_AES_GCM) {
 366                 PMD_DRV_LOG(ERR, "Unsupported crypto transformation mode\n");
 367                 rte_mempool_put(mempool, (void *)ic_session);
 368                 return -ENOTSUP;
 369         }
 370         aead_xform = &conf->crypto_xform->aead;
 371
 372         if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
 373                 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) {
 374                         ic_session->op = IXGBE_OP_AUTHENTICATED_DECRYPTION;
 375                 } else {
 376                         PMD_DRV_LOG(ERR, "IPsec decryption not enabled\n");
 377                         rte_mempool_put(mempool, (void *)ic_session);
 378                         return -ENOTSUP;
 379                 }
 380         } else {
 381                 if (dev_conf->txmode.offloads & DEV_TX_OFFLOAD_SECURITY) {
 382                         ic_session->op = IXGBE_OP_AUTHENTICATED_ENCRYPTION;
 383                 } else {
 384                         PMD_DRV_LOG(ERR, "IPsec encryption not enabled\n");
 385                         rte_mempool_put(mempool, (void *)ic_session);
 386                         return -ENOTSUP;
 387                 }
 388         }
 389
 390         ic_session->key = aead_xform->key.data;
 391         memcpy(&ic_session->salt,
 392                &aead_xform->key.data[aead_xform->key.length], 4);
 393         ic_session->spi = conf->ipsec.spi;
 394         ic_session->dev = eth_dev;
 395
 396         set_sec_session_private_data(session, ic_session);
 397
 398         if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) {
 399                 if (ixgbe_crypto_add_sa(ic_session)) {
 400                         PMD_DRV_LOG(ERR, "Failed to add SA\n");
 401                         rte_mempool_put(mempool, (void *)ic_session);
 402                         return -EPERM;
 403                 }
 404         }
 405
 406         return 0;
 407 }
 408
 409 static unsigned int
 410 ixgbe_crypto_session_get_size(__rte_unused void *device)
 411 {
 412         return sizeof(struct ixgbe_crypto_session);
 413 }
 414
 415 static int
 416 ixgbe_crypto_remove_session(void *device,
 417                 struct rte_security_session *session)
 418 {
 419         struct rte_eth_dev *eth_dev = device;
 420         struct ixgbe_crypto_session *ic_session =
 421                 (struct ixgbe_crypto_session *)
 422                 get_sec_session_private_data(session);
 423         struct rte_mempool *mempool = rte_mempool_from_obj(ic_session);
 424
 425         if (eth_dev != ic_session->dev) {
 426                 PMD_DRV_LOG(ERR, "Session not bound to this device\n");
 427                 return -ENODEV;
 428         }
 429
 430         if (ixgbe_crypto_remove_sa(eth_dev, ic_session)) {
 431                 PMD_DRV_LOG(ERR, "Failed to remove session\n");
 432                 return -EFAULT;
 433         }
 434
 435         rte_mempool_put(mempool, (void *)ic_session);
 436
 437         return 0;
 438 }
 439
 440 static inline uint8_t
 441 ixgbe_crypto_compute_pad_len(struct rte_mbuf *m)
 442 {
 443         if (m->nb_segs == 1) {
 444                 /* 16 bytes ICV + 2 bytes ESP trailer + payload padding size
 445                  * payload padding size is stored at <pkt_len - 18>
 446                  */
 447                 uint8_t *esp_pad_len = rte_pktmbuf_mtod_offset(m, uint8_t *,
 448                                         rte_pktmbuf_pkt_len(m) -
 449                                         (ESP_TRAILER_SIZE + ESP_ICV_SIZE));
 450                 return *esp_pad_len + ESP_TRAILER_SIZE + ESP_ICV_SIZE;
 451         }
 452         return 0;
 453 }
 454
 455 static int
 456 ixgbe_crypto_update_mb(void *device __rte_unused,
 457                 struct rte_security_session *session,
 458                        struct rte_mbuf *m, void *params __rte_unused)
 459 {
 460         struct ixgbe_crypto_session *ic_session =
 461                         get_sec_session_private_data(session);
 462         if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) {
 463                 union ixgbe_crypto_tx_desc_md *mdata =
 464                         (union ixgbe_crypto_tx_desc_md *)&m->udata64;
 465                 mdata->enc = 1;
 466                 mdata->sa_idx = ic_session->sa_index;
 467                 mdata->pad_len = ixgbe_crypto_compute_pad_len(m);
 468         }
 469         return 0;
 470 }
 471
 472
 473 static const struct rte_security_capability *
 474 ixgbe_crypto_capabilities_get(void *device __rte_unused)
 475 {
 476         static const struct rte_cryptodev_capabilities
 477         aes_gcm_gmac_crypto_capabilities[] = {
 478                 {       /* AES GMAC (128-bit) */
 479                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 480                         {.sym = {
 481                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 482                                 {.auth = {
 483                                         .algo = RTE_CRYPTO_AUTH_AES_GMAC,
 484                                         .block_size = 16,
 485                                         .key_size = {
 486                                                 .min = 16,
 487                                                 .max = 16,
 488                                                 .increment = 0
 489                                         },
 490                                         .digest_size = {
 491                                                 .min = 16,
 492                                                 .max = 16,
 493                                                 .increment = 0
 494                                         },
 495                                         .iv_size = {
 496                                                 .min = 12,
 497                                                 .max = 12,
 498                                                 .increment = 0
 499                                         }
 500                                 }, }
 501                         }, }
 502                 },
 503                 {       /* AES GCM (128-bit) */
 504                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 505                         {.sym = {
 506                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 507                                 {.aead = {
 508                                         .algo = RTE_CRYPTO_AEAD_AES_GCM,
 509                                         .block_size = 16,
 510                                         .key_size = {
 511                                                 .min = 16,
 512                                                 .max = 16,
 513                                                 .increment = 0
 514                                         },
 515                                         .digest_size = {
 516                                                 .min = 16,
 517                                                 .max = 16,
 518                                                 .increment = 0
 519                                         },
 520                                         .aad_size = {
 521                                                 .min = 0,
 522                                                 .max = 65535,
 523                                                 .increment = 1
 524                                         },
 525                                         .iv_size = {
 526                                                 .min = 12,
 527                                                 .max = 12,
 528                                                 .increment = 0
 529                                         }
 530                                 }, }
 531                         }, }
 532                 },
 533                 {
 534                         .op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
 535                         {.sym = {
 536                                 .xform_type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED
 537                         }, }
 538                 },
 539         };
 540
 541         static const struct rte_security_capability
 542         ixgbe_security_capabilities[] = {
 543                 { /* IPsec Inline Crypto ESP Transport Egress */
 544                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 545                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 546                         {.ipsec = {
 547                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 548                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 549                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 550                                 .options = { 0 }
 551                         } },
 552                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 553                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 554                 },
 555                 { /* IPsec Inline Crypto ESP Transport Ingress */
 556                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 557                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 558                         {.ipsec = {
 559                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 560                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 561                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 562                                 .options = { 0 }
 563                         } },
 564                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 565                         .ol_flags = 0
 566                 },
 567                 { /* IPsec Inline Crypto ESP Tunnel Egress */
 568                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 569                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 570                         {.ipsec = {
 571                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 572                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 573                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 574                                 .options = { 0 }
 575                         } },
 576                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 577                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 578                 },
 579                 { /* IPsec Inline Crypto ESP Tunnel Ingress */
 580                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 581                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 582                         {.ipsec = {
 583                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 584                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 585                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 586                                 .options = { 0 }
 587                         } },
 588                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 589                         .ol_flags = 0
 590                 },
 591                 {
 592                         .action = RTE_SECURITY_ACTION_TYPE_NONE
 593                 }
 594         };
 595
 596         return ixgbe_security_capabilities;
 597 }
 598
 599
 600 int
 601 ixgbe_crypto_enable_ipsec(struct rte_eth_dev *dev)
 602 {
 603         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 604         uint32_t reg;
 605         uint64_t rx_offloads;
 606         uint64_t tx_offloads;
 607
 608         rx_offloads = dev->data->dev_conf.rxmode.offloads;
 609         tx_offloads = dev->data->dev_conf.txmode.offloads;
 610
 611         /* sanity checks */
 612         if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) {
 613                 PMD_DRV_LOG(ERR, "RSC and IPsec not supported");
 614                 return -1;
 615         }
 616         if (rx_offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
 617                 PMD_DRV_LOG(ERR, "HW CRC strip needs to be enabled for IPsec");
 618                 return -1;
 619         }
 620
 621
 622         /* Set IXGBE_SECTXBUFFAF to 0x15 as required in the datasheet*/
 623         IXGBE_WRITE_REG(hw, IXGBE_SECTXBUFFAF, 0x15);
 624
 625         /* IFG needs to be set to 3 when we are using security. Otherwise a Tx
 626          * hang will occur with heavy traffic.
 627          */
 628         reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
 629         reg = (reg & 0xFFFFFFF0) | 0x3;
 630         IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
 631
 632         reg  = IXGBE_READ_REG(hw, IXGBE_HLREG0);
 633         reg |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_RXCRCSTRP;
 634         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg);
 635
 636         if (rx_offloads & DEV_RX_OFFLOAD_SECURITY) {
 637                 IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0);
 638                 reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
 639                 if (reg != 0) {
 640                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
 641                         return -1;
 642                 }
 643         }
 644         if (tx_offloads & DEV_TX_OFFLOAD_SECURITY) {
 645                 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL,
 646                                 IXGBE_SECTXCTRL_STORE_FORWARD);
 647                 reg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL);
 648                 if (reg != IXGBE_SECTXCTRL_STORE_FORWARD) {
 649                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
 650                         return -1;
 651                 }
 652         }
 653
 654         ixgbe_crypto_clear_ipsec_tables(dev);
 655
 656         return 0;
 657 }
 658
 659 int
 660 ixgbe_crypto_add_ingress_sa_from_flow(const void *sess,
 661                                       const void *ip_spec,
 662                                       uint8_t is_ipv6)
 663 {
 664         struct ixgbe_crypto_session *ic_session
 665                 = get_sec_session_private_data(sess);
 666
 667         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
 668                 if (is_ipv6) {
 669                         const struct rte_flow_item_ipv6 *ipv6 = ip_spec;
 670                         ic_session->src_ip.type = IPv6;
 671                         ic_session->dst_ip.type = IPv6;
 672                         rte_memcpy(ic_session->src_ip.ipv6,
 673                                    ipv6->hdr.src_addr, 16);
 674                         rte_memcpy(ic_session->dst_ip.ipv6,
 675                                    ipv6->hdr.dst_addr, 16);
 676                 } else {
 677                         const struct rte_flow_item_ipv4 *ipv4 = ip_spec;
 678                         ic_session->src_ip.type = IPv4;
 679                         ic_session->dst_ip.type = IPv4;
 680                         ic_session->src_ip.ipv4 = ipv4->hdr.src_addr;
 681                         ic_session->dst_ip.ipv4 = ipv4->hdr.dst_addr;
 682                 }
 683                 return ixgbe_crypto_add_sa(ic_session);
 684         }
 685
 686         return 0;
 687 }
 688
 689 static struct rte_security_ops ixgbe_security_ops = {
 690         .session_create = ixgbe_crypto_create_session,
 691         .session_update = NULL,
 692         .session_get_size = ixgbe_crypto_session_get_size,
 693         .session_stats_get = NULL,
 694         .session_destroy = ixgbe_crypto_remove_session,
 695         .set_pkt_metadata = ixgbe_crypto_update_mb,
 696         .capabilities_get = ixgbe_crypto_capabilities_get
 697 };
 698
 699 static int
 700 ixgbe_crypto_capable(struct rte_eth_dev *dev)
 701 {
 702         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 703         uint32_t reg_i, reg, capable = 1;
 704         /* test if rx crypto can be enabled and then write back initial value*/
 705         reg_i = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
 706         IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0);
 707         reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
 708         if (reg != 0)
 709                 capable = 0;
 710         IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, reg_i);
 711         return capable;
 712 }
 713
 714 int
 715 ixgbe_ipsec_ctx_create(struct rte_eth_dev *dev)
 716 {
 717         struct rte_security_ctx *ctx = NULL;
 718
 719         if (ixgbe_crypto_capable(dev)) {
 720                 ctx = rte_malloc("rte_security_instances_ops",
 721                                  sizeof(struct rte_security_ctx), 0);
 722                 if (ctx) {
 723                         ctx->device = (void *)dev;
 724                         ctx->ops = &ixgbe_security_ops;
 725                         ctx->sess_cnt = 0;
 726                         dev->security_ctx = ctx;
 727                 } else {
 728                         return -ENOMEM;
 729                 }
 730         }
 731         return 0;
 732 }