drivers/net/szedata2/rte_eth_szedata2.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) 2015 - 2016 CESNET
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of CESNET nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <unistd.h>
  36 #include <stdbool.h>
  37 #include <err.h>
  38 #include <sys/types.h>
  39 #include <dirent.h>
  40 #include <sys/stat.h>
  41 #include <fcntl.h>
  42 #include <sys/mman.h>
  43
  44 #include <libsze2.h>
  45
  46 #include <rte_mbuf.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_ethdev_pci.h>
  49 #include <rte_malloc.h>
  50 #include <rte_memcpy.h>
  51 #include <rte_kvargs.h>
  52 #include <rte_dev.h>
  53 #include <rte_atomic.h>
  54
  55 #include "rte_eth_szedata2.h"
  56
  57 #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
  58 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
  59 #define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
  60
  61 /**
  62  * size of szedata2_packet header with alignment
  63  */
  64 #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
  65
  66 #define RTE_SZEDATA2_DRIVER_NAME net_szedata2
  67 #define RTE_SZEDATA2_PCI_DRIVER_NAME "rte_szedata2_pmd"
  68
  69 #define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u"
  70
  71 struct szedata2_rx_queue {
  72         struct szedata *sze;
  73         uint8_t rx_channel;
  74         uint8_t in_port;
  75         struct rte_mempool *mb_pool;
  76         volatile uint64_t rx_pkts;
  77         volatile uint64_t rx_bytes;
  78         volatile uint64_t err_pkts;
  79 };
  80
  81 struct szedata2_tx_queue {
  82         struct szedata *sze;
  83         uint8_t tx_channel;
  84         volatile uint64_t tx_pkts;
  85         volatile uint64_t tx_bytes;
  86         volatile uint64_t err_pkts;
  87 };
  88
  89 struct pmd_internals {
  90         struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
  91         struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
  92         uint16_t max_rx_queues;
  93         uint16_t max_tx_queues;
  94         char sze_dev[PATH_MAX];
  95         struct rte_mem_resource *pci_rsc;
  96 };
  97
  98 static struct ether_addr eth_addr = {
  99         .addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
 100 };
 101
 102 static uint16_t
 103 eth_szedata2_rx(void *queue,
 104                 struct rte_mbuf **bufs,
 105                 uint16_t nb_pkts)
 106 {
 107         unsigned int i;
 108         struct rte_mbuf *mbuf;
 109         struct szedata2_rx_queue *sze_q = queue;
 110         struct rte_pktmbuf_pool_private *mbp_priv;
 111         uint16_t num_rx = 0;
 112         uint16_t buf_size;
 113         uint16_t sg_size;
 114         uint16_t hw_size;
 115         uint16_t packet_size;
 116         uint64_t num_bytes = 0;
 117         struct szedata *sze = sze_q->sze;
 118         uint8_t *header_ptr = NULL; /* header of packet */
 119         uint8_t *packet_ptr1 = NULL;
 120         uint8_t *packet_ptr2 = NULL;
 121         uint16_t packet_len1 = 0;
 122         uint16_t packet_len2 = 0;
 123         uint16_t hw_data_align;
 124
 125         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 126                 return 0;
 127
 128         /*
 129          * Reads the given number of packets from szedata2 channel given
 130          * by queue and copies the packet data into a newly allocated mbuf
 131          * to return.
 132          */
 133         for (i = 0; i < nb_pkts; i++) {
 134                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 135
 136                 if (unlikely(mbuf == NULL))
 137                         break;
 138
 139                 /* get the next sze packet */
 140                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 141                                 sze->ct_rx_lck->next == NULL) {
 142                         /* unlock old data */
 143                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 144                         sze->ct_rx_lck_orig = NULL;
 145                         sze->ct_rx_lck = NULL;
 146                 }
 147
 148                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 149                         /* nothing to read, lock new data */
 150                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 151                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 152
 153                         if (sze->ct_rx_lck == NULL) {
 154                                 /* nothing to lock */
 155                                 rte_pktmbuf_free(mbuf);
 156                                 break;
 157                         }
 158
 159                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 160                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 161
 162                         if (!sze->ct_rx_rem_bytes) {
 163                                 rte_pktmbuf_free(mbuf);
 164                                 break;
 165                         }
 166                 }
 167
 168                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 169                         /*
 170                          * cut in header
 171                          * copy parts of header to merge buffer
 172                          */
 173                         if (sze->ct_rx_lck->next == NULL) {
 174                                 rte_pktmbuf_free(mbuf);
 175                                 break;
 176                         }
 177
 178                         /* copy first part of header */
 179                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 180                                         sze->ct_rx_rem_bytes);
 181
 182                         /* copy second part of header */
 183                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 184                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 185                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 186                                 sze->ct_rx_cur_ptr,
 187                                 RTE_SZE2_PACKET_HEADER_SIZE -
 188                                 sze->ct_rx_rem_bytes);
 189
 190                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 191                                 sze->ct_rx_rem_bytes;
 192                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 193                                 RTE_SZE2_PACKET_HEADER_SIZE +
 194                                 sze->ct_rx_rem_bytes;
 195
 196                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 197                 } else {
 198                         /* not cut */
 199                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 200                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 201                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 202                 }
 203
 204                 sg_size = le16toh(*((uint16_t *)header_ptr));
 205                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 206                 packet_size = sg_size -
 207                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 208
 209
 210                 /* checks if packet all right */
 211                 if (!sg_size)
 212                         errx(5, "Zero segsize");
 213
 214                 /* check sg_size and hwsize */
 215                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 216                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 217                                 "hwsize: %d", sg_size, hw_size);
 218                 }
 219
 220                 hw_data_align =
 221                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
 222                         RTE_SZE2_PACKET_HEADER_SIZE;
 223
 224                 if (sze->ct_rx_rem_bytes >=
 225                                 (uint16_t)(sg_size -
 226                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 227                         /* no cut */
 228                         /* one packet ready - go to another */
 229                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 230                         packet_len1 = packet_size;
 231                         packet_ptr2 = NULL;
 232                         packet_len2 = 0;
 233
 234                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 235                                 RTE_SZE2_PACKET_HEADER_SIZE;
 236                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 237                                 RTE_SZE2_PACKET_HEADER_SIZE;
 238                 } else {
 239                         /* cut in data */
 240                         if (sze->ct_rx_lck->next == NULL) {
 241                                 errx(6, "Need \"next\" lock, "
 242                                         "but it is missing: %u",
 243                                         sze->ct_rx_rem_bytes);
 244                         }
 245
 246                         /* skip hw data */
 247                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 248                                 uint16_t rem_size = hw_data_align -
 249                                         sze->ct_rx_rem_bytes;
 250
 251                                 /* MOVE to next lock */
 252                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 253                                 sze->ct_rx_cur_ptr =
 254                                         (void *)(((uint8_t *)
 255                                         (sze->ct_rx_lck->start)) + rem_size);
 256
 257                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 258                                 packet_len1 = packet_size;
 259                                 packet_ptr2 = NULL;
 260                                 packet_len2 = 0;
 261
 262                                 sze->ct_rx_cur_ptr +=
 263                                         RTE_SZE2_ALIGN8(packet_size);
 264                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 265                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 266                         } else {
 267                                 /* get pointer and length from first part */
 268                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 269                                         hw_data_align;
 270                                 packet_len1 = sze->ct_rx_rem_bytes -
 271                                         hw_data_align;
 272
 273                                 /* MOVE to next lock */
 274                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 275                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 276
 277                                 /* get pointer and length from second part */
 278                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 279                                 packet_len2 = packet_size - packet_len1;
 280
 281                                 sze->ct_rx_cur_ptr +=
 282                                         RTE_SZE2_ALIGN8(packet_size) -
 283                                         packet_len1;
 284                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 285                                         (RTE_SZE2_ALIGN8(packet_size) -
 286                                          packet_len1);
 287                         }
 288                 }
 289
 290                 if (unlikely(packet_ptr1 == NULL)) {
 291                         rte_pktmbuf_free(mbuf);
 292                         break;
 293                 }
 294
 295                 /* get the space available for data in the mbuf */
 296                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 297                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 298                                 RTE_PKTMBUF_HEADROOM);
 299
 300                 if (packet_size <= buf_size) {
 301                         /* sze packet will fit in one mbuf, go ahead and copy */
 302                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 303                                         packet_ptr1, packet_len1);
 304                         if (packet_ptr2 != NULL) {
 305                                 rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
 306                                         uint8_t *) + packet_len1),
 307                                         packet_ptr2, packet_len2);
 308                         }
 309                         mbuf->data_len = (uint16_t)packet_size;
 310
 311                         mbuf->pkt_len = packet_size;
 312                         mbuf->port = sze_q->in_port;
 313                         bufs[num_rx] = mbuf;
 314                         num_rx++;
 315                         num_bytes += packet_size;
 316                 } else {
 317                         /*
 318                          * sze packet will not fit in one mbuf,
 319                          * scattered mode is not enabled, drop packet
 320                          */
 321                         RTE_LOG(ERR, PMD,
 322                                 "SZE segment %d bytes will not fit in one mbuf "
 323                                 "(%d bytes), scattered mode is not enabled, "
 324                                 "drop packet!!\n",
 325                                 packet_size, buf_size);
 326                         rte_pktmbuf_free(mbuf);
 327                 }
 328         }
 329
 330         sze_q->rx_pkts += num_rx;
 331         sze_q->rx_bytes += num_bytes;
 332         return num_rx;
 333 }
 334
 335 static uint16_t
 336 eth_szedata2_rx_scattered(void *queue,
 337                 struct rte_mbuf **bufs,
 338                 uint16_t nb_pkts)
 339 {
 340         unsigned int i;
 341         struct rte_mbuf *mbuf;
 342         struct szedata2_rx_queue *sze_q = queue;
 343         struct rte_pktmbuf_pool_private *mbp_priv;
 344         uint16_t num_rx = 0;
 345         uint16_t buf_size;
 346         uint16_t sg_size;
 347         uint16_t hw_size;
 348         uint16_t packet_size;
 349         uint64_t num_bytes = 0;
 350         struct szedata *sze = sze_q->sze;
 351         uint8_t *header_ptr = NULL; /* header of packet */
 352         uint8_t *packet_ptr1 = NULL;
 353         uint8_t *packet_ptr2 = NULL;
 354         uint16_t packet_len1 = 0;
 355         uint16_t packet_len2 = 0;
 356         uint16_t hw_data_align;
 357
 358         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 359                 return 0;
 360
 361         /*
 362          * Reads the given number of packets from szedata2 channel given
 363          * by queue and copies the packet data into a newly allocated mbuf
 364          * to return.
 365          */
 366         for (i = 0; i < nb_pkts; i++) {
 367                 const struct szedata_lock *ct_rx_lck_backup;
 368                 unsigned int ct_rx_rem_bytes_backup;
 369                 unsigned char *ct_rx_cur_ptr_backup;
 370
 371                 /* get the next sze packet */
 372                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 373                                 sze->ct_rx_lck->next == NULL) {
 374                         /* unlock old data */
 375                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 376                         sze->ct_rx_lck_orig = NULL;
 377                         sze->ct_rx_lck = NULL;
 378                 }
 379
 380                 /*
 381                  * Store items from sze structure which can be changed
 382                  * before mbuf allocating. Use these items in case of mbuf
 383                  * allocating failure.
 384                  */
 385                 ct_rx_lck_backup = sze->ct_rx_lck;
 386                 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 387                 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 388
 389                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 390                         /* nothing to read, lock new data */
 391                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 392                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 393
 394                         /*
 395                          * Backup items from sze structure must be updated
 396                          * after locking to contain pointers to new locks.
 397                          */
 398                         ct_rx_lck_backup = sze->ct_rx_lck;
 399                         ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 400                         ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 401
 402                         if (sze->ct_rx_lck == NULL)
 403                                 /* nothing to lock */
 404                                 break;
 405
 406                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 407                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 408
 409                         if (!sze->ct_rx_rem_bytes)
 410                                 break;
 411                 }
 412
 413                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 414                         /*
 415                          * cut in header - copy parts of header to merge buffer
 416                          */
 417                         if (sze->ct_rx_lck->next == NULL)
 418                                 break;
 419
 420                         /* copy first part of header */
 421                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 422                                         sze->ct_rx_rem_bytes);
 423
 424                         /* copy second part of header */
 425                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 426                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 427                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 428                                 sze->ct_rx_cur_ptr,
 429                                 RTE_SZE2_PACKET_HEADER_SIZE -
 430                                 sze->ct_rx_rem_bytes);
 431
 432                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 433                                 sze->ct_rx_rem_bytes;
 434                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 435                                 RTE_SZE2_PACKET_HEADER_SIZE +
 436                                 sze->ct_rx_rem_bytes;
 437
 438                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 439                 } else {
 440                         /* not cut */
 441                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 442                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 443                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 444                 }
 445
 446                 sg_size = le16toh(*((uint16_t *)header_ptr));
 447                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 448                 packet_size = sg_size -
 449                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 450
 451
 452                 /* checks if packet all right */
 453                 if (!sg_size)
 454                         errx(5, "Zero segsize");
 455
 456                 /* check sg_size and hwsize */
 457                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 458                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 459                                         "hwsize: %d", sg_size, hw_size);
 460                 }
 461
 462                 hw_data_align =
 463                         RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
 464                         hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
 465
 466                 if (sze->ct_rx_rem_bytes >=
 467                                 (uint16_t)(sg_size -
 468                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 469                         /* no cut */
 470                         /* one packet ready - go to another */
 471                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 472                         packet_len1 = packet_size;
 473                         packet_ptr2 = NULL;
 474                         packet_len2 = 0;
 475
 476                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 477                                 RTE_SZE2_PACKET_HEADER_SIZE;
 478                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 479                                 RTE_SZE2_PACKET_HEADER_SIZE;
 480                 } else {
 481                         /* cut in data */
 482                         if (sze->ct_rx_lck->next == NULL) {
 483                                 errx(6, "Need \"next\" lock, but it is "
 484                                         "missing: %u", sze->ct_rx_rem_bytes);
 485                         }
 486
 487                         /* skip hw data */
 488                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 489                                 uint16_t rem_size = hw_data_align -
 490                                         sze->ct_rx_rem_bytes;
 491
 492                                 /* MOVE to next lock */
 493                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 494                                 sze->ct_rx_cur_ptr =
 495                                         (void *)(((uint8_t *)
 496                                         (sze->ct_rx_lck->start)) + rem_size);
 497
 498                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 499                                 packet_len1 = packet_size;
 500                                 packet_ptr2 = NULL;
 501                                 packet_len2 = 0;
 502
 503                                 sze->ct_rx_cur_ptr +=
 504                                         RTE_SZE2_ALIGN8(packet_size);
 505                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 506                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 507                         } else {
 508                                 /* get pointer and length from first part */
 509                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 510                                         hw_data_align;
 511                                 packet_len1 = sze->ct_rx_rem_bytes -
 512                                         hw_data_align;
 513
 514                                 /* MOVE to next lock */
 515                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 516                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 517
 518                                 /* get pointer and length from second part */
 519                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 520                                 packet_len2 = packet_size - packet_len1;
 521
 522                                 sze->ct_rx_cur_ptr +=
 523                                         RTE_SZE2_ALIGN8(packet_size) -
 524                                         packet_len1;
 525                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 526                                         (RTE_SZE2_ALIGN8(packet_size) -
 527                                          packet_len1);
 528                         }
 529                 }
 530
 531                 if (unlikely(packet_ptr1 == NULL))
 532                         break;
 533
 534                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 535
 536                 if (unlikely(mbuf == NULL)) {
 537                         /*
 538                          * Restore items from sze structure to state after
 539                          * unlocking (eventually locking).
 540                          */
 541                         sze->ct_rx_lck = ct_rx_lck_backup;
 542                         sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
 543                         sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
 544                         break;
 545                 }
 546
 547                 /* get the space available for data in the mbuf */
 548                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 549                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 550                                 RTE_PKTMBUF_HEADROOM);
 551
 552                 if (packet_size <= buf_size) {
 553                         /* sze packet will fit in one mbuf, go ahead and copy */
 554                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 555                                         packet_ptr1, packet_len1);
 556                         if (packet_ptr2 != NULL) {
 557                                 rte_memcpy((void *)
 558                                         (rte_pktmbuf_mtod(mbuf, uint8_t *) +
 559                                         packet_len1), packet_ptr2, packet_len2);
 560                         }
 561                         mbuf->data_len = (uint16_t)packet_size;
 562                 } else {
 563                         /*
 564                          * sze packet will not fit in one mbuf,
 565                          * scatter packet into more mbufs
 566                          */
 567                         struct rte_mbuf *m = mbuf;
 568                         uint16_t len = rte_pktmbuf_tailroom(mbuf);
 569
 570                         /* copy first part of packet */
 571                         /* fill first mbuf */
 572                         rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
 573                                 len);
 574                         packet_len1 -= len;
 575                         packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 576
 577                         while (packet_len1 > 0) {
 578                                 /* fill new mbufs */
 579                                 m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
 580
 581                                 if (unlikely(m->next == NULL)) {
 582                                         rte_pktmbuf_free(mbuf);
 583                                         /*
 584                                          * Restore items from sze structure
 585                                          * to state after unlocking (eventually
 586                                          * locking).
 587                                          */
 588                                         sze->ct_rx_lck = ct_rx_lck_backup;
 589                                         sze->ct_rx_rem_bytes =
 590                                                 ct_rx_rem_bytes_backup;
 591                                         sze->ct_rx_cur_ptr =
 592                                                 ct_rx_cur_ptr_backup;
 593                                         goto finish;
 594                                 }
 595
 596                                 m = m->next;
 597
 598                                 len = RTE_MIN(rte_pktmbuf_tailroom(m),
 599                                         packet_len1);
 600                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 601                                         packet_ptr1, len);
 602
 603                                 (mbuf->nb_segs)++;
 604                                 packet_len1 -= len;
 605                                 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 606                         }
 607
 608                         if (packet_ptr2 != NULL) {
 609                                 /* copy second part of packet, if exists */
 610                                 /* fill the rest of currently last mbuf */
 611                                 len = rte_pktmbuf_tailroom(m);
 612                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 613                                         packet_ptr2, len);
 614                                 packet_len2 -= len;
 615                                 packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
 616
 617                                 while (packet_len2 > 0) {
 618                                         /* fill new mbufs */
 619                                         m->next = rte_pktmbuf_alloc(
 620                                                         sze_q->mb_pool);
 621
 622                                         if (unlikely(m->next == NULL)) {
 623                                                 rte_pktmbuf_free(mbuf);
 624                                                 /*
 625                                                  * Restore items from sze
 626                                                  * structure to state after
 627                                                  * unlocking (eventually
 628                                                  * locking).
 629                                                  */
 630                                                 sze->ct_rx_lck =
 631                                                         ct_rx_lck_backup;
 632                                                 sze->ct_rx_rem_bytes =
 633                                                         ct_rx_rem_bytes_backup;
 634                                                 sze->ct_rx_cur_ptr =
 635                                                         ct_rx_cur_ptr_backup;
 636                                                 goto finish;
 637                                         }
 638
 639                                         m = m->next;
 640
 641                                         len = RTE_MIN(rte_pktmbuf_tailroom(m),
 642                                                 packet_len2);
 643                                         rte_memcpy(
 644                                                 rte_pktmbuf_append(mbuf, len),
 645                                                 packet_ptr2, len);
 646
 647                                         (mbuf->nb_segs)++;
 648                                         packet_len2 -= len;
 649                                         packet_ptr2 = ((uint8_t *)packet_ptr2) +
 650                                                 len;
 651                                 }
 652                         }
 653                 }
 654                 mbuf->pkt_len = packet_size;
 655                 mbuf->port = sze_q->in_port;
 656                 bufs[num_rx] = mbuf;
 657                 num_rx++;
 658                 num_bytes += packet_size;
 659         }
 660
 661 finish:
 662         sze_q->rx_pkts += num_rx;
 663         sze_q->rx_bytes += num_bytes;
 664         return num_rx;
 665 }
 666
 667 static uint16_t
 668 eth_szedata2_tx(void *queue,
 669                 struct rte_mbuf **bufs,
 670                 uint16_t nb_pkts)
 671 {
 672         struct rte_mbuf *mbuf;
 673         struct szedata2_tx_queue *sze_q = queue;
 674         uint16_t num_tx = 0;
 675         uint64_t num_bytes = 0;
 676
 677         const struct szedata_lock *lck;
 678         uint32_t lock_size;
 679         uint32_t lock_size2;
 680         void *dst;
 681         uint32_t pkt_len;
 682         uint32_t hwpkt_len;
 683         uint32_t unlock_size;
 684         uint32_t rem_len;
 685         uint8_t mbuf_segs;
 686         uint16_t pkt_left = nb_pkts;
 687
 688         if (sze_q->sze == NULL || nb_pkts == 0)
 689                 return 0;
 690
 691         while (pkt_left > 0) {
 692                 unlock_size = 0;
 693                 lck = szedata_tx_lock_data(sze_q->sze,
 694                         RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
 695                         sze_q->tx_channel);
 696                 if (lck == NULL)
 697                         continue;
 698
 699                 dst = lck->start;
 700                 lock_size = lck->len;
 701                 lock_size2 = lck->next ? lck->next->len : 0;
 702
 703 next_packet:
 704                 mbuf = bufs[nb_pkts - pkt_left];
 705
 706                 pkt_len = mbuf->pkt_len;
 707                 mbuf_segs = mbuf->nb_segs;
 708
 709                 hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 710                         RTE_SZE2_ALIGN8(pkt_len);
 711
 712                 if (lock_size + lock_size2 < hwpkt_len) {
 713                         szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 714                         continue;
 715                 }
 716
 717                 num_bytes += pkt_len;
 718
 719                 if (lock_size > hwpkt_len) {
 720                         void *tmp_dst;
 721
 722                         rem_len = 0;
 723
 724                         /* write packet length at first 2 bytes in 8B header */
 725                         *((uint16_t *)dst) = htole16(
 726                                         RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 727                                         pkt_len);
 728                         *(((uint16_t *)dst) + 1) = htole16(0);
 729
 730                         /* copy packet from mbuf */
 731                         tmp_dst = ((uint8_t *)(dst)) +
 732                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 733                         if (mbuf_segs == 1) {
 734                                 /*
 735                                  * non-scattered packet,
 736                                  * transmit from one mbuf
 737                                  */
 738                                 rte_memcpy(tmp_dst,
 739                                         rte_pktmbuf_mtod(mbuf, const void *),
 740                                         pkt_len);
 741                         } else {
 742                                 /* scattered packet, transmit from more mbufs */
 743                                 struct rte_mbuf *m = mbuf;
 744                                 while (m) {
 745                                         rte_memcpy(tmp_dst,
 746                                                 rte_pktmbuf_mtod(m,
 747                                                 const void *),
 748                                                 m->data_len);
 749                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 750                                                 m->data_len;
 751                                         m = m->next;
 752                                 }
 753                         }
 754
 755
 756                         dst = ((uint8_t *)dst) + hwpkt_len;
 757                         unlock_size += hwpkt_len;
 758                         lock_size -= hwpkt_len;
 759
 760                         rte_pktmbuf_free(mbuf);
 761                         num_tx++;
 762                         pkt_left--;
 763                         if (pkt_left == 0) {
 764                                 szedata_tx_unlock_data(sze_q->sze, lck,
 765                                         unlock_size);
 766                                 break;
 767                         }
 768                         goto next_packet;
 769                 } else if (lock_size + lock_size2 >= hwpkt_len) {
 770                         void *tmp_dst;
 771                         uint16_t write_len;
 772
 773                         /* write packet length at first 2 bytes in 8B header */
 774                         *((uint16_t *)dst) =
 775                                 htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 776                                         pkt_len);
 777                         *(((uint16_t *)dst) + 1) = htole16(0);
 778
 779                         /*
 780                          * If the raw packet (pkt_len) is smaller than lock_size
 781                          * get the correct length for memcpy
 782                          */
 783                         write_len =
 784                                 pkt_len < lock_size -
 785                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
 786                                 pkt_len :
 787                                 lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 788
 789                         rem_len = hwpkt_len - lock_size;
 790
 791                         tmp_dst = ((uint8_t *)(dst)) +
 792                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 793                         if (mbuf_segs == 1) {
 794                                 /*
 795                                  * non-scattered packet,
 796                                  * transmit from one mbuf
 797                                  */
 798                                 /* copy part of packet to first area */
 799                                 rte_memcpy(tmp_dst,
 800                                         rte_pktmbuf_mtod(mbuf, const void *),
 801                                         write_len);
 802
 803                                 if (lck->next)
 804                                         dst = lck->next->start;
 805
 806                                 /* copy part of packet to second area */
 807                                 rte_memcpy(dst,
 808                                         (const void *)(rte_pktmbuf_mtod(mbuf,
 809                                                         const uint8_t *) +
 810                                         write_len), pkt_len - write_len);
 811                         } else {
 812                                 /* scattered packet, transmit from more mbufs */
 813                                 struct rte_mbuf *m = mbuf;
 814                                 uint16_t written = 0;
 815                                 uint16_t to_write = 0;
 816                                 bool new_mbuf = true;
 817                                 uint16_t write_off = 0;
 818
 819                                 /* copy part of packet to first area */
 820                                 while (m && written < write_len) {
 821                                         to_write = RTE_MIN(m->data_len,
 822                                                         write_len - written);
 823                                         rte_memcpy(tmp_dst,
 824                                                 rte_pktmbuf_mtod(m,
 825                                                         const void *),
 826                                                 to_write);
 827
 828                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 829                                                 to_write;
 830                                         if (m->data_len <= write_len -
 831                                                         written) {
 832                                                 m = m->next;
 833                                                 new_mbuf = true;
 834                                         } else {
 835                                                 new_mbuf = false;
 836                                         }
 837                                         written += to_write;
 838                                 }
 839
 840                                 if (lck->next)
 841                                         dst = lck->next->start;
 842
 843                                 tmp_dst = dst;
 844                                 written = 0;
 845                                 write_off = new_mbuf ? 0 : to_write;
 846
 847                                 /* copy part of packet to second area */
 848                                 while (m && written < pkt_len - write_len) {
 849                                         rte_memcpy(tmp_dst, (const void *)
 850                                                 (rte_pktmbuf_mtod(m,
 851                                                 uint8_t *) + write_off),
 852                                                 m->data_len - write_off);
 853
 854                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 855                                                 (m->data_len - write_off);
 856                                         written += m->data_len - write_off;
 857                                         m = m->next;
 858                                         write_off = 0;
 859                                 }
 860                         }
 861
 862                         dst = ((uint8_t *)dst) + rem_len;
 863                         unlock_size += hwpkt_len;
 864                         lock_size = lock_size2 - rem_len;
 865                         lock_size2 = 0;
 866
 867                         rte_pktmbuf_free(mbuf);
 868                         num_tx++;
 869                 }
 870
 871                 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 872                 pkt_left--;
 873         }
 874
 875         sze_q->tx_pkts += num_tx;
 876         sze_q->err_pkts += nb_pkts - num_tx;
 877         sze_q->tx_bytes += num_bytes;
 878         return num_tx;
 879 }
 880
 881 static int
 882 eth_rx_queue_start(struct rte_eth_dev *dev, uint16_t rxq_id)
 883 {
 884         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 885         int ret;
 886         struct pmd_internals *internals = (struct pmd_internals *)
 887                 dev->data->dev_private;
 888
 889         if (rxq->sze == NULL) {
 890                 uint32_t rx = 1 << rxq->rx_channel;
 891                 uint32_t tx = 0;
 892                 rxq->sze = szedata_open(internals->sze_dev);
 893                 if (rxq->sze == NULL)
 894                         return -EINVAL;
 895                 ret = szedata_subscribe3(rxq->sze, &rx, &tx);
 896                 if (ret != 0 || rx == 0)
 897                         goto err;
 898         }
 899
 900         ret = szedata_start(rxq->sze);
 901         if (ret != 0)
 902                 goto err;
 903         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 904         return 0;
 905
 906 err:
 907         szedata_close(rxq->sze);
 908         rxq->sze = NULL;
 909         return -EINVAL;
 910 }
 911
 912 static int
 913 eth_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rxq_id)
 914 {
 915         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 916
 917         if (rxq->sze != NULL) {
 918                 szedata_close(rxq->sze);
 919                 rxq->sze = NULL;
 920         }
 921
 922         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 923         return 0;
 924 }
 925
 926 static int
 927 eth_tx_queue_start(struct rte_eth_dev *dev, uint16_t txq_id)
 928 {
 929         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 930         int ret;
 931         struct pmd_internals *internals = (struct pmd_internals *)
 932                 dev->data->dev_private;
 933
 934         if (txq->sze == NULL) {
 935                 uint32_t rx = 0;
 936                 uint32_t tx = 1 << txq->tx_channel;
 937                 txq->sze = szedata_open(internals->sze_dev);
 938                 if (txq->sze == NULL)
 939                         return -EINVAL;
 940                 ret = szedata_subscribe3(txq->sze, &rx, &tx);
 941                 if (ret != 0 || tx == 0)
 942                         goto err;
 943         }
 944
 945         ret = szedata_start(txq->sze);
 946         if (ret != 0)
 947                 goto err;
 948         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 949         return 0;
 950
 951 err:
 952         szedata_close(txq->sze);
 953         txq->sze = NULL;
 954         return -EINVAL;
 955 }
 956
 957 static int
 958 eth_tx_queue_stop(struct rte_eth_dev *dev, uint16_t txq_id)
 959 {
 960         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 961
 962         if (txq->sze != NULL) {
 963                 szedata_close(txq->sze);
 964                 txq->sze = NULL;
 965         }
 966
 967         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 968         return 0;
 969 }
 970
 971 static int
 972 eth_dev_start(struct rte_eth_dev *dev)
 973 {
 974         int ret;
 975         uint16_t i;
 976         uint16_t nb_rx = dev->data->nb_rx_queues;
 977         uint16_t nb_tx = dev->data->nb_tx_queues;
 978
 979         for (i = 0; i < nb_rx; i++) {
 980                 ret = eth_rx_queue_start(dev, i);
 981                 if (ret != 0)
 982                         goto err_rx;
 983         }
 984
 985         for (i = 0; i < nb_tx; i++) {
 986                 ret = eth_tx_queue_start(dev, i);
 987                 if (ret != 0)
 988                         goto err_tx;
 989         }
 990
 991         return 0;
 992
 993 err_tx:
 994         for (i = 0; i < nb_tx; i++)
 995                 eth_tx_queue_stop(dev, i);
 996 err_rx:
 997         for (i = 0; i < nb_rx; i++)
 998                 eth_rx_queue_stop(dev, i);
 999         return ret;
1000 }
1001
1002 static void
1003 eth_dev_stop(struct rte_eth_dev *dev)
1004 {
1005         uint16_t i;
1006         uint16_t nb_rx = dev->data->nb_rx_queues;
1007         uint16_t nb_tx = dev->data->nb_tx_queues;
1008
1009         for (i = 0; i < nb_tx; i++)
1010                 eth_tx_queue_stop(dev, i);
1011
1012         for (i = 0; i < nb_rx; i++)
1013                 eth_rx_queue_stop(dev, i);
1014 }
1015
1016 static int
1017 eth_dev_configure(struct rte_eth_dev *dev)
1018 {
1019         struct rte_eth_dev_data *data = dev->data;
1020         if (data->dev_conf.rxmode.enable_scatter == 1) {
1021                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1022                 data->scattered_rx = 1;
1023         } else {
1024                 dev->rx_pkt_burst = eth_szedata2_rx;
1025                 data->scattered_rx = 0;
1026         }
1027         return 0;
1028 }
1029
1030 static void
1031 eth_dev_info(struct rte_eth_dev *dev,
1032                 struct rte_eth_dev_info *dev_info)
1033 {
1034         struct pmd_internals *internals = dev->data->dev_private;
1035         dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device);
1036         dev_info->if_index = 0;
1037         dev_info->max_mac_addrs = 1;
1038         dev_info->max_rx_pktlen = (uint32_t)-1;
1039         dev_info->max_rx_queues = internals->max_rx_queues;
1040         dev_info->max_tx_queues = internals->max_tx_queues;
1041         dev_info->min_rx_bufsize = 0;
1042         dev_info->speed_capa = ETH_LINK_SPEED_100G;
1043 }
1044
1045 static void
1046 eth_stats_get(struct rte_eth_dev *dev,
1047                 struct rte_eth_stats *stats)
1048 {
1049         uint16_t i;
1050         uint16_t nb_rx = dev->data->nb_rx_queues;
1051         uint16_t nb_tx = dev->data->nb_tx_queues;
1052         uint64_t rx_total = 0;
1053         uint64_t tx_total = 0;
1054         uint64_t tx_err_total = 0;
1055         uint64_t rx_total_bytes = 0;
1056         uint64_t tx_total_bytes = 0;
1057         const struct pmd_internals *internals = dev->data->dev_private;
1058
1059         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_rx; i++) {
1060                 stats->q_ipackets[i] = internals->rx_queue[i].rx_pkts;
1061                 stats->q_ibytes[i] = internals->rx_queue[i].rx_bytes;
1062                 rx_total += stats->q_ipackets[i];
1063                 rx_total_bytes += stats->q_ibytes[i];
1064         }
1065
1066         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_tx; i++) {
1067                 stats->q_opackets[i] = internals->tx_queue[i].tx_pkts;
1068                 stats->q_obytes[i] = internals->tx_queue[i].tx_bytes;
1069                 stats->q_errors[i] = internals->tx_queue[i].err_pkts;
1070                 tx_total += stats->q_opackets[i];
1071                 tx_total_bytes += stats->q_obytes[i];
1072                 tx_err_total += stats->q_errors[i];
1073         }
1074
1075         stats->ipackets = rx_total;
1076         stats->opackets = tx_total;
1077         stats->ibytes = rx_total_bytes;
1078         stats->obytes = tx_total_bytes;
1079         stats->oerrors = tx_err_total;
1080 }
1081
1082 static void
1083 eth_stats_reset(struct rte_eth_dev *dev)
1084 {
1085         uint16_t i;
1086         uint16_t nb_rx = dev->data->nb_rx_queues;
1087         uint16_t nb_tx = dev->data->nb_tx_queues;
1088         struct pmd_internals *internals = dev->data->dev_private;
1089
1090         for (i = 0; i < nb_rx; i++) {
1091                 internals->rx_queue[i].rx_pkts = 0;
1092                 internals->rx_queue[i].rx_bytes = 0;
1093                 internals->rx_queue[i].err_pkts = 0;
1094         }
1095         for (i = 0; i < nb_tx; i++) {
1096                 internals->tx_queue[i].tx_pkts = 0;
1097                 internals->tx_queue[i].tx_bytes = 0;
1098                 internals->tx_queue[i].err_pkts = 0;
1099         }
1100 }
1101
1102 static void
1103 eth_rx_queue_release(void *q)
1104 {
1105         struct szedata2_rx_queue *rxq = (struct szedata2_rx_queue *)q;
1106         if (rxq->sze != NULL) {
1107                 szedata_close(rxq->sze);
1108                 rxq->sze = NULL;
1109         }
1110 }
1111
1112 static void
1113 eth_tx_queue_release(void *q)
1114 {
1115         struct szedata2_tx_queue *txq = (struct szedata2_tx_queue *)q;
1116         if (txq->sze != NULL) {
1117                 szedata_close(txq->sze);
1118                 txq->sze = NULL;
1119         }
1120 }
1121
1122 static void
1123 eth_dev_close(struct rte_eth_dev *dev)
1124 {
1125         uint16_t i;
1126         uint16_t nb_rx = dev->data->nb_rx_queues;
1127         uint16_t nb_tx = dev->data->nb_tx_queues;
1128
1129         eth_dev_stop(dev);
1130
1131         for (i = 0; i < nb_rx; i++) {
1132                 eth_rx_queue_release(dev->data->rx_queues[i]);
1133                 dev->data->rx_queues[i] = NULL;
1134         }
1135         dev->data->nb_rx_queues = 0;
1136         for (i = 0; i < nb_tx; i++) {
1137                 eth_tx_queue_release(dev->data->tx_queues[i]);
1138                 dev->data->tx_queues[i] = NULL;
1139         }
1140         dev->data->nb_tx_queues = 0;
1141 }
1142
1143 static int
1144 eth_link_update(struct rte_eth_dev *dev,
1145                 int wait_to_complete __rte_unused)
1146 {
1147         struct rte_eth_link link;
1148         struct rte_eth_link *link_ptr = &link;
1149         struct rte_eth_link *dev_link = &dev->data->dev_link;
1150         struct pmd_internals *internals = (struct pmd_internals *)
1151                 dev->data->dev_private;
1152         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1153                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1154                         volatile struct szedata2_cgmii_ibuf *);
1155
1156         switch (cgmii_link_speed(ibuf)) {
1157         case SZEDATA2_LINK_SPEED_10G:
1158                 link.link_speed = ETH_SPEED_NUM_10G;
1159                 break;
1160         case SZEDATA2_LINK_SPEED_40G:
1161                 link.link_speed = ETH_SPEED_NUM_40G;
1162                 break;
1163         case SZEDATA2_LINK_SPEED_100G:
1164                 link.link_speed = ETH_SPEED_NUM_100G;
1165                 break;
1166         default:
1167                 link.link_speed = ETH_SPEED_NUM_10G;
1168                 break;
1169         }
1170
1171         /* szedata2 uses only full duplex */
1172         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1173
1174         link.link_status = (cgmii_ibuf_is_enabled(ibuf) &&
1175                         cgmii_ibuf_is_link_up(ibuf)) ? ETH_LINK_UP : ETH_LINK_DOWN;
1176
1177         link.link_autoneg = ETH_LINK_SPEED_FIXED;
1178
1179         rte_atomic64_cmpset((uint64_t *)dev_link, *(uint64_t *)dev_link,
1180                         *(uint64_t *)link_ptr);
1181
1182         return 0;
1183 }
1184
1185 static int
1186 eth_dev_set_link_up(struct rte_eth_dev *dev)
1187 {
1188         struct pmd_internals *internals = (struct pmd_internals *)
1189                 dev->data->dev_private;
1190         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1191                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1192                         volatile struct szedata2_cgmii_ibuf *);
1193         volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR(
1194                         internals->pci_rsc, SZEDATA2_CGMII_OBUF_BASE_OFF,
1195                         volatile struct szedata2_cgmii_obuf *);
1196
1197         cgmii_ibuf_enable(ibuf);
1198         cgmii_obuf_enable(obuf);
1199         return 0;
1200 }
1201
1202 static int
1203 eth_dev_set_link_down(struct rte_eth_dev *dev)
1204 {
1205         struct pmd_internals *internals = (struct pmd_internals *)
1206                 dev->data->dev_private;
1207         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1208                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1209                         volatile struct szedata2_cgmii_ibuf *);
1210         volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR(
1211                         internals->pci_rsc, SZEDATA2_CGMII_OBUF_BASE_OFF,
1212                         volatile struct szedata2_cgmii_obuf *);
1213
1214         cgmii_ibuf_disable(ibuf);
1215         cgmii_obuf_disable(obuf);
1216         return 0;
1217 }
1218
1219 static int
1220 eth_rx_queue_setup(struct rte_eth_dev *dev,
1221                 uint16_t rx_queue_id,
1222                 uint16_t nb_rx_desc __rte_unused,
1223                 unsigned int socket_id __rte_unused,
1224                 const struct rte_eth_rxconf *rx_conf __rte_unused,
1225                 struct rte_mempool *mb_pool)
1226 {
1227         struct pmd_internals *internals = dev->data->dev_private;
1228         struct szedata2_rx_queue *rxq = &internals->rx_queue[rx_queue_id];
1229         int ret;
1230         uint32_t rx = 1 << rx_queue_id;
1231         uint32_t tx = 0;
1232
1233         rxq->sze = szedata_open(internals->sze_dev);
1234         if (rxq->sze == NULL)
1235                 return -EINVAL;
1236         ret = szedata_subscribe3(rxq->sze, &rx, &tx);
1237         if (ret != 0 || rx == 0) {
1238                 szedata_close(rxq->sze);
1239                 rxq->sze = NULL;
1240                 return -EINVAL;
1241         }
1242         rxq->rx_channel = rx_queue_id;
1243         rxq->in_port = dev->data->port_id;
1244         rxq->mb_pool = mb_pool;
1245         rxq->rx_pkts = 0;
1246         rxq->rx_bytes = 0;
1247         rxq->err_pkts = 0;
1248
1249         dev->data->rx_queues[rx_queue_id] = rxq;
1250         return 0;
1251 }
1252
1253 static int
1254 eth_tx_queue_setup(struct rte_eth_dev *dev,
1255                 uint16_t tx_queue_id,
1256                 uint16_t nb_tx_desc __rte_unused,
1257                 unsigned int socket_id __rte_unused,
1258                 const struct rte_eth_txconf *tx_conf __rte_unused)
1259 {
1260         struct pmd_internals *internals = dev->data->dev_private;
1261         struct szedata2_tx_queue *txq = &internals->tx_queue[tx_queue_id];
1262         int ret;
1263         uint32_t rx = 0;
1264         uint32_t tx = 1 << tx_queue_id;
1265
1266         txq->sze = szedata_open(internals->sze_dev);
1267         if (txq->sze == NULL)
1268                 return -EINVAL;
1269         ret = szedata_subscribe3(txq->sze, &rx, &tx);
1270         if (ret != 0 || tx == 0) {
1271                 szedata_close(txq->sze);
1272                 txq->sze = NULL;
1273                 return -EINVAL;
1274         }
1275         txq->tx_channel = tx_queue_id;
1276         txq->tx_pkts = 0;
1277         txq->tx_bytes = 0;
1278         txq->err_pkts = 0;
1279
1280         dev->data->tx_queues[tx_queue_id] = txq;
1281         return 0;
1282 }
1283
1284 static void
1285 eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1286                 struct ether_addr *mac_addr __rte_unused)
1287 {
1288 }
1289
1290 static void
1291 eth_promiscuous_enable(struct rte_eth_dev *dev)
1292 {
1293         struct pmd_internals *internals = (struct pmd_internals *)
1294                 dev->data->dev_private;
1295         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1296                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1297                         volatile struct szedata2_cgmii_ibuf *);
1298         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_PROMISC);
1299 }
1300
1301 static void
1302 eth_promiscuous_disable(struct rte_eth_dev *dev)
1303 {
1304         struct pmd_internals *internals = (struct pmd_internals *)
1305                 dev->data->dev_private;
1306         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1307                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1308                         volatile struct szedata2_cgmii_ibuf *);
1309         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID);
1310 }
1311
1312 static void
1313 eth_allmulticast_enable(struct rte_eth_dev *dev)
1314 {
1315         struct pmd_internals *internals = (struct pmd_internals *)
1316                 dev->data->dev_private;
1317         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1318                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1319                         volatile struct szedata2_cgmii_ibuf *);
1320         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ALL_MULTICAST);
1321 }
1322
1323 static void
1324 eth_allmulticast_disable(struct rte_eth_dev *dev)
1325 {
1326         struct pmd_internals *internals = (struct pmd_internals *)
1327                 dev->data->dev_private;
1328         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1329                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1330                         volatile struct szedata2_cgmii_ibuf *);
1331         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID);
1332 }
1333
1334 static const struct eth_dev_ops ops = {
1335         .dev_start          = eth_dev_start,
1336         .dev_stop           = eth_dev_stop,
1337         .dev_set_link_up    = eth_dev_set_link_up,
1338         .dev_set_link_down  = eth_dev_set_link_down,
1339         .dev_close          = eth_dev_close,
1340         .dev_configure      = eth_dev_configure,
1341         .dev_infos_get      = eth_dev_info,
1342         .promiscuous_enable   = eth_promiscuous_enable,
1343         .promiscuous_disable  = eth_promiscuous_disable,
1344         .allmulticast_enable  = eth_allmulticast_enable,
1345         .allmulticast_disable = eth_allmulticast_disable,
1346         .rx_queue_start     = eth_rx_queue_start,
1347         .rx_queue_stop      = eth_rx_queue_stop,
1348         .tx_queue_start     = eth_tx_queue_start,
1349         .tx_queue_stop      = eth_tx_queue_stop,
1350         .rx_queue_setup     = eth_rx_queue_setup,
1351         .tx_queue_setup     = eth_tx_queue_setup,
1352         .rx_queue_release   = eth_rx_queue_release,
1353         .tx_queue_release   = eth_tx_queue_release,
1354         .link_update        = eth_link_update,
1355         .stats_get          = eth_stats_get,
1356         .stats_reset        = eth_stats_reset,
1357         .mac_addr_set       = eth_mac_addr_set,
1358 };
1359
1360 /*
1361  * This function goes through sysfs and looks for an index of szedata2
1362  * device file (/dev/szedataIIX, where X is the index).
1363  *
1364  * @return
1365  *           0 on success
1366  *          -1 on error
1367  */
1368 static int
1369 get_szedata2_index(const struct rte_pci_addr *pcislot_addr, uint32_t *index)
1370 {
1371         DIR *dir;
1372         struct dirent *entry;
1373         int ret;
1374         uint32_t tmp_index;
1375         FILE *fd;
1376         char pcislot_path[PATH_MAX];
1377         uint32_t domain;
1378         uint32_t bus;
1379         uint32_t devid;
1380         uint32_t function;
1381
1382         dir = opendir("/sys/class/combo");
1383         if (dir == NULL)
1384                 return -1;
1385
1386         /*
1387          * Iterate through all combosixX directories.
1388          * When the value in /sys/class/combo/combosixX/device/pcislot
1389          * file is the location of the ethernet device dev, "X" is the
1390          * index of the device.
1391          */
1392         while ((entry = readdir(dir)) != NULL) {
1393                 ret = sscanf(entry->d_name, "combosix%u", &tmp_index);
1394                 if (ret != 1)
1395                         continue;
1396
1397                 snprintf(pcislot_path, PATH_MAX,
1398                         "/sys/class/combo/combosix%u/device/pcislot",
1399                         tmp_index);
1400
1401                 fd = fopen(pcislot_path, "r");
1402                 if (fd == NULL)
1403                         continue;
1404
1405                 ret = fscanf(fd, "%4" PRIx16 ":%2" PRIx8 ":%2" PRIx8 ".%" PRIx8,
1406                                 &domain, &bus, &devid, &function);
1407                 fclose(fd);
1408                 if (ret != 4)
1409                         continue;
1410
1411                 if (pcislot_addr->domain == domain &&
1412                                 pcislot_addr->bus == bus &&
1413                                 pcislot_addr->devid == devid &&
1414                                 pcislot_addr->function == function) {
1415                         *index = tmp_index;
1416                         closedir(dir);
1417                         return 0;
1418                 }
1419         }
1420
1421         closedir(dir);
1422         return -1;
1423 }
1424
1425 static int
1426 rte_szedata2_eth_dev_init(struct rte_eth_dev *dev)
1427 {
1428         struct rte_eth_dev_data *data = dev->data;
1429         struct pmd_internals *internals = (struct pmd_internals *)
1430                 data->dev_private;
1431         struct szedata *szedata_temp;
1432         int ret;
1433         uint32_t szedata2_index;
1434         struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
1435         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1436         struct rte_mem_resource *pci_rsc =
1437                 &pci_dev->mem_resource[PCI_RESOURCE_NUMBER];
1438         char rsc_filename[PATH_MAX];
1439         void *pci_resource_ptr = NULL;
1440         int fd;
1441
1442         RTE_LOG(INFO, PMD, "Initializing szedata2 device (" PCI_PRI_FMT ")\n",
1443                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1444                         pci_addr->function);
1445
1446         /* Get index of szedata2 device file and create path to device file */
1447         ret = get_szedata2_index(pci_addr, &szedata2_index);
1448         if (ret != 0) {
1449                 RTE_LOG(ERR, PMD, "Failed to get szedata2 device index!\n");
1450                 return -ENODEV;
1451         }
1452         snprintf(internals->sze_dev, PATH_MAX, SZEDATA2_DEV_PATH_FMT,
1453                         szedata2_index);
1454
1455         RTE_LOG(INFO, PMD, "SZEDATA2 path: %s\n", internals->sze_dev);
1456
1457         /*
1458          * Get number of available DMA RX and TX channels, which is maximum
1459          * number of queues that can be created and store it in private device
1460          * data structure.
1461          */
1462         szedata_temp = szedata_open(internals->sze_dev);
1463         if (szedata_temp == NULL) {
1464                 RTE_LOG(ERR, PMD, "szedata_open(): failed to open %s",
1465                                 internals->sze_dev);
1466                 return -EINVAL;
1467         }
1468         internals->max_rx_queues = szedata_ifaces_available(szedata_temp,
1469                         SZE2_DIR_RX);
1470         internals->max_tx_queues = szedata_ifaces_available(szedata_temp,
1471                         SZE2_DIR_TX);
1472         szedata_close(szedata_temp);
1473
1474         RTE_LOG(INFO, PMD, "Available DMA channels RX: %u TX: %u\n",
1475                         internals->max_rx_queues, internals->max_tx_queues);
1476
1477         /* Set rx, tx burst functions */
1478         if (data->dev_conf.rxmode.enable_scatter == 1 ||
1479                 data->scattered_rx == 1) {
1480                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1481                 data->scattered_rx = 1;
1482         } else {
1483                 dev->rx_pkt_burst = eth_szedata2_rx;
1484                 data->scattered_rx = 0;
1485         }
1486         dev->tx_pkt_burst = eth_szedata2_tx;
1487
1488         /* Set function callbacks for Ethernet API */
1489         dev->dev_ops = &ops;
1490
1491         rte_eth_copy_pci_info(dev, pci_dev);
1492
1493         /* mmap pci resource0 file to rte_mem_resource structure */
1494         if (pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr ==
1495                         0) {
1496                 RTE_LOG(ERR, PMD, "Missing resource%u file\n",
1497                                 PCI_RESOURCE_NUMBER);
1498                 return -EINVAL;
1499         }
1500         snprintf(rsc_filename, PATH_MAX,
1501                 "%s/" PCI_PRI_FMT "/resource%u", pci_get_sysfs_path(),
1502                 pci_addr->domain, pci_addr->bus,
1503                 pci_addr->devid, pci_addr->function, PCI_RESOURCE_NUMBER);
1504         fd = open(rsc_filename, O_RDWR);
1505         if (fd < 0) {
1506                 RTE_LOG(ERR, PMD, "Could not open file %s\n", rsc_filename);
1507                 return -EINVAL;
1508         }
1509
1510         pci_resource_ptr = mmap(0,
1511                         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len,
1512                         PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1513         close(fd);
1514         if (pci_resource_ptr == NULL) {
1515                 RTE_LOG(ERR, PMD, "Could not mmap file %s (fd = %d)\n",
1516                                 rsc_filename, fd);
1517                 return -EINVAL;
1518         }
1519         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr = pci_resource_ptr;
1520         internals->pci_rsc = pci_rsc;
1521
1522         RTE_LOG(DEBUG, PMD, "resource%u phys_addr = 0x%llx len = %llu "
1523                         "virt addr = %llx\n", PCI_RESOURCE_NUMBER,
1524                         (unsigned long long)pci_rsc->phys_addr,
1525                         (unsigned long long)pci_rsc->len,
1526                         (unsigned long long)pci_rsc->addr);
1527
1528         /* Get link state */
1529         eth_link_update(dev, 0);
1530
1531         /* Allocate space for one mac address */
1532         data->mac_addrs = rte_zmalloc(data->name, sizeof(struct ether_addr),
1533                         RTE_CACHE_LINE_SIZE);
1534         if (data->mac_addrs == NULL) {
1535                 RTE_LOG(ERR, PMD, "Could not alloc space for MAC address!\n");
1536                 munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1537                        pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1538                 return -EINVAL;
1539         }
1540
1541         ether_addr_copy(&eth_addr, data->mac_addrs);
1542
1543         /* At initial state COMBO card is in promiscuous mode so disable it */
1544         eth_promiscuous_disable(dev);
1545
1546         RTE_LOG(INFO, PMD, "szedata2 device ("
1547                         PCI_PRI_FMT ") successfully initialized\n",
1548                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1549                         pci_addr->function);
1550
1551         return 0;
1552 }
1553
1554 static int
1555 rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev)
1556 {
1557         struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
1558         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1559
1560         rte_free(dev->data->mac_addrs);
1561         dev->data->mac_addrs = NULL;
1562         munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1563                pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1564
1565         RTE_LOG(INFO, PMD, "szedata2 device ("
1566                         PCI_PRI_FMT ") successfully uninitialized\n",
1567                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1568                         pci_addr->function);
1569
1570         return 0;
1571 }
1572
1573 static const struct rte_pci_id rte_szedata2_pci_id_table[] = {
1574         {
1575                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1576                                 PCI_DEVICE_ID_NETCOPE_COMBO80G)
1577         },
1578         {
1579                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1580                                 PCI_DEVICE_ID_NETCOPE_COMBO100G)
1581         },
1582         {
1583                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1584                                 PCI_DEVICE_ID_NETCOPE_COMBO100G2)
1585         },
1586         {
1587                 .vendor_id = 0,
1588         }
1589 };
1590
1591 static int szedata2_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1592         struct rte_pci_device *pci_dev)
1593 {
1594         return rte_eth_dev_pci_generic_probe(pci_dev,
1595                 sizeof(struct pmd_internals), rte_szedata2_eth_dev_init);
1596 }
1597
1598 static int szedata2_eth_pci_remove(struct rte_pci_device *pci_dev)
1599 {
1600         return rte_eth_dev_pci_generic_remove(pci_dev,
1601                 rte_szedata2_eth_dev_uninit);
1602 }
1603
1604 static struct rte_pci_driver szedata2_eth_driver = {
1605         .id_table = rte_szedata2_pci_id_table,
1606         .probe = szedata2_eth_pci_probe,
1607         .remove = szedata2_eth_pci_remove,
1608 };
1609
1610 RTE_PMD_REGISTER_PCI(RTE_SZEDATA2_DRIVER_NAME, szedata2_eth_driver);
1611 RTE_PMD_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table);
1612 RTE_PMD_REGISTER_KMOD_DEP(RTE_SZEDATA2_DRIVER_NAME,
1613         "* combo6core & combov3 & szedata2 & szedata2_cv3");