drivers/net/szedata2/rte_eth_szedata2.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) 2015 - 2016 CESNET
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of CESNET nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <unistd.h>
  36 #include <stdbool.h>
  37 #include <err.h>
  38 #include <sys/types.h>
  39 #include <dirent.h>
  40 #include <sys/stat.h>
  41 #include <fcntl.h>
  42 #include <sys/mman.h>
  43
  44 #include <libsze2.h>
  45
  46 #include <rte_mbuf.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_ethdev_pci.h>
  49 #include <rte_malloc.h>
  50 #include <rte_memcpy.h>
  51 #include <rte_kvargs.h>
  52 #include <rte_dev.h>
  53 #include <rte_atomic.h>
  54
  55 #include "rte_eth_szedata2.h"
  56 #include "szedata2_iobuf.h"
  57
  58 #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
  59 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
  60 #define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
  61
  62 /**
  63  * size of szedata2_packet header with alignment
  64  */
  65 #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
  66
  67 #define RTE_SZEDATA2_DRIVER_NAME net_szedata2
  68
  69 #define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u"
  70
  71 struct szedata2_rx_queue {
  72         struct szedata *sze;
  73         uint8_t rx_channel;
  74         uint16_t in_port;
  75         struct rte_mempool *mb_pool;
  76         volatile uint64_t rx_pkts;
  77         volatile uint64_t rx_bytes;
  78         volatile uint64_t err_pkts;
  79 };
  80
  81 struct szedata2_tx_queue {
  82         struct szedata *sze;
  83         uint8_t tx_channel;
  84         volatile uint64_t tx_pkts;
  85         volatile uint64_t tx_bytes;
  86         volatile uint64_t err_pkts;
  87 };
  88
  89 struct pmd_internals {
  90         struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
  91         struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
  92         uint16_t max_rx_queues;
  93         uint16_t max_tx_queues;
  94         char sze_dev[PATH_MAX];
  95         struct rte_mem_resource *pci_rsc;
  96 };
  97
  98 static struct ether_addr eth_addr = {
  99         .addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
 100 };
 101
 102 static uint16_t
 103 eth_szedata2_rx(void *queue,
 104                 struct rte_mbuf **bufs,
 105                 uint16_t nb_pkts)
 106 {
 107         unsigned int i;
 108         struct rte_mbuf *mbuf;
 109         struct szedata2_rx_queue *sze_q = queue;
 110         struct rte_pktmbuf_pool_private *mbp_priv;
 111         uint16_t num_rx = 0;
 112         uint16_t buf_size;
 113         uint16_t sg_size;
 114         uint16_t hw_size;
 115         uint16_t packet_size;
 116         uint64_t num_bytes = 0;
 117         struct szedata *sze = sze_q->sze;
 118         uint8_t *header_ptr = NULL; /* header of packet */
 119         uint8_t *packet_ptr1 = NULL;
 120         uint8_t *packet_ptr2 = NULL;
 121         uint16_t packet_len1 = 0;
 122         uint16_t packet_len2 = 0;
 123         uint16_t hw_data_align;
 124
 125         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 126                 return 0;
 127
 128         /*
 129          * Reads the given number of packets from szedata2 channel given
 130          * by queue and copies the packet data into a newly allocated mbuf
 131          * to return.
 132          */
 133         for (i = 0; i < nb_pkts; i++) {
 134                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 135
 136                 if (unlikely(mbuf == NULL))
 137                         break;
 138
 139                 /* get the next sze packet */
 140                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 141                                 sze->ct_rx_lck->next == NULL) {
 142                         /* unlock old data */
 143                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 144                         sze->ct_rx_lck_orig = NULL;
 145                         sze->ct_rx_lck = NULL;
 146                 }
 147
 148                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 149                         /* nothing to read, lock new data */
 150                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 151                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 152
 153                         if (sze->ct_rx_lck == NULL) {
 154                                 /* nothing to lock */
 155                                 rte_pktmbuf_free(mbuf);
 156                                 break;
 157                         }
 158
 159                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 160                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 161
 162                         if (!sze->ct_rx_rem_bytes) {
 163                                 rte_pktmbuf_free(mbuf);
 164                                 break;
 165                         }
 166                 }
 167
 168                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 169                         /*
 170                          * cut in header
 171                          * copy parts of header to merge buffer
 172                          */
 173                         if (sze->ct_rx_lck->next == NULL) {
 174                                 rte_pktmbuf_free(mbuf);
 175                                 break;
 176                         }
 177
 178                         /* copy first part of header */
 179                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 180                                         sze->ct_rx_rem_bytes);
 181
 182                         /* copy second part of header */
 183                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 184                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 185                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 186                                 sze->ct_rx_cur_ptr,
 187                                 RTE_SZE2_PACKET_HEADER_SIZE -
 188                                 sze->ct_rx_rem_bytes);
 189
 190                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 191                                 sze->ct_rx_rem_bytes;
 192                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 193                                 RTE_SZE2_PACKET_HEADER_SIZE +
 194                                 sze->ct_rx_rem_bytes;
 195
 196                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 197                 } else {
 198                         /* not cut */
 199                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 200                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 201                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 202                 }
 203
 204                 sg_size = le16toh(*((uint16_t *)header_ptr));
 205                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 206                 packet_size = sg_size -
 207                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 208
 209
 210                 /* checks if packet all right */
 211                 if (!sg_size)
 212                         errx(5, "Zero segsize");
 213
 214                 /* check sg_size and hwsize */
 215                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 216                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 217                                 "hwsize: %d", sg_size, hw_size);
 218                 }
 219
 220                 hw_data_align =
 221                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
 222                         RTE_SZE2_PACKET_HEADER_SIZE;
 223
 224                 if (sze->ct_rx_rem_bytes >=
 225                                 (uint16_t)(sg_size -
 226                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 227                         /* no cut */
 228                         /* one packet ready - go to another */
 229                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 230                         packet_len1 = packet_size;
 231                         packet_ptr2 = NULL;
 232                         packet_len2 = 0;
 233
 234                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 235                                 RTE_SZE2_PACKET_HEADER_SIZE;
 236                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 237                                 RTE_SZE2_PACKET_HEADER_SIZE;
 238                 } else {
 239                         /* cut in data */
 240                         if (sze->ct_rx_lck->next == NULL) {
 241                                 errx(6, "Need \"next\" lock, "
 242                                         "but it is missing: %u",
 243                                         sze->ct_rx_rem_bytes);
 244                         }
 245
 246                         /* skip hw data */
 247                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 248                                 uint16_t rem_size = hw_data_align -
 249                                         sze->ct_rx_rem_bytes;
 250
 251                                 /* MOVE to next lock */
 252                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 253                                 sze->ct_rx_cur_ptr =
 254                                         (void *)(((uint8_t *)
 255                                         (sze->ct_rx_lck->start)) + rem_size);
 256
 257                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 258                                 packet_len1 = packet_size;
 259                                 packet_ptr2 = NULL;
 260                                 packet_len2 = 0;
 261
 262                                 sze->ct_rx_cur_ptr +=
 263                                         RTE_SZE2_ALIGN8(packet_size);
 264                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 265                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 266                         } else {
 267                                 /* get pointer and length from first part */
 268                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 269                                         hw_data_align;
 270                                 packet_len1 = sze->ct_rx_rem_bytes -
 271                                         hw_data_align;
 272
 273                                 /* MOVE to next lock */
 274                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 275                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 276
 277                                 /* get pointer and length from second part */
 278                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 279                                 packet_len2 = packet_size - packet_len1;
 280
 281                                 sze->ct_rx_cur_ptr +=
 282                                         RTE_SZE2_ALIGN8(packet_size) -
 283                                         packet_len1;
 284                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 285                                         (RTE_SZE2_ALIGN8(packet_size) -
 286                                          packet_len1);
 287                         }
 288                 }
 289
 290                 if (unlikely(packet_ptr1 == NULL)) {
 291                         rte_pktmbuf_free(mbuf);
 292                         break;
 293                 }
 294
 295                 /* get the space available for data in the mbuf */
 296                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 297                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 298                                 RTE_PKTMBUF_HEADROOM);
 299
 300                 if (packet_size <= buf_size) {
 301                         /* sze packet will fit in one mbuf, go ahead and copy */
 302                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 303                                         packet_ptr1, packet_len1);
 304                         if (packet_ptr2 != NULL) {
 305                                 rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
 306                                         uint8_t *) + packet_len1),
 307                                         packet_ptr2, packet_len2);
 308                         }
 309                         mbuf->data_len = (uint16_t)packet_size;
 310
 311                         mbuf->pkt_len = packet_size;
 312                         mbuf->port = sze_q->in_port;
 313                         bufs[num_rx] = mbuf;
 314                         num_rx++;
 315                         num_bytes += packet_size;
 316                 } else {
 317                         /*
 318                          * sze packet will not fit in one mbuf,
 319                          * scattered mode is not enabled, drop packet
 320                          */
 321                         RTE_LOG(ERR, PMD,
 322                                 "SZE segment %d bytes will not fit in one mbuf "
 323                                 "(%d bytes), scattered mode is not enabled, "
 324                                 "drop packet!!\n",
 325                                 packet_size, buf_size);
 326                         rte_pktmbuf_free(mbuf);
 327                 }
 328         }
 329
 330         sze_q->rx_pkts += num_rx;
 331         sze_q->rx_bytes += num_bytes;
 332         return num_rx;
 333 }
 334
 335 static uint16_t
 336 eth_szedata2_rx_scattered(void *queue,
 337                 struct rte_mbuf **bufs,
 338                 uint16_t nb_pkts)
 339 {
 340         unsigned int i;
 341         struct rte_mbuf *mbuf;
 342         struct szedata2_rx_queue *sze_q = queue;
 343         struct rte_pktmbuf_pool_private *mbp_priv;
 344         uint16_t num_rx = 0;
 345         uint16_t buf_size;
 346         uint16_t sg_size;
 347         uint16_t hw_size;
 348         uint16_t packet_size;
 349         uint64_t num_bytes = 0;
 350         struct szedata *sze = sze_q->sze;
 351         uint8_t *header_ptr = NULL; /* header of packet */
 352         uint8_t *packet_ptr1 = NULL;
 353         uint8_t *packet_ptr2 = NULL;
 354         uint16_t packet_len1 = 0;
 355         uint16_t packet_len2 = 0;
 356         uint16_t hw_data_align;
 357
 358         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 359                 return 0;
 360
 361         /*
 362          * Reads the given number of packets from szedata2 channel given
 363          * by queue and copies the packet data into a newly allocated mbuf
 364          * to return.
 365          */
 366         for (i = 0; i < nb_pkts; i++) {
 367                 const struct szedata_lock *ct_rx_lck_backup;
 368                 unsigned int ct_rx_rem_bytes_backup;
 369                 unsigned char *ct_rx_cur_ptr_backup;
 370
 371                 /* get the next sze packet */
 372                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 373                                 sze->ct_rx_lck->next == NULL) {
 374                         /* unlock old data */
 375                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 376                         sze->ct_rx_lck_orig = NULL;
 377                         sze->ct_rx_lck = NULL;
 378                 }
 379
 380                 /*
 381                  * Store items from sze structure which can be changed
 382                  * before mbuf allocating. Use these items in case of mbuf
 383                  * allocating failure.
 384                  */
 385                 ct_rx_lck_backup = sze->ct_rx_lck;
 386                 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 387                 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 388
 389                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 390                         /* nothing to read, lock new data */
 391                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 392                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 393
 394                         /*
 395                          * Backup items from sze structure must be updated
 396                          * after locking to contain pointers to new locks.
 397                          */
 398                         ct_rx_lck_backup = sze->ct_rx_lck;
 399                         ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 400                         ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 401
 402                         if (sze->ct_rx_lck == NULL)
 403                                 /* nothing to lock */
 404                                 break;
 405
 406                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 407                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 408
 409                         if (!sze->ct_rx_rem_bytes)
 410                                 break;
 411                 }
 412
 413                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 414                         /*
 415                          * cut in header - copy parts of header to merge buffer
 416                          */
 417                         if (sze->ct_rx_lck->next == NULL)
 418                                 break;
 419
 420                         /* copy first part of header */
 421                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 422                                         sze->ct_rx_rem_bytes);
 423
 424                         /* copy second part of header */
 425                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 426                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 427                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 428                                 sze->ct_rx_cur_ptr,
 429                                 RTE_SZE2_PACKET_HEADER_SIZE -
 430                                 sze->ct_rx_rem_bytes);
 431
 432                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 433                                 sze->ct_rx_rem_bytes;
 434                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 435                                 RTE_SZE2_PACKET_HEADER_SIZE +
 436                                 sze->ct_rx_rem_bytes;
 437
 438                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 439                 } else {
 440                         /* not cut */
 441                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 442                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 443                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 444                 }
 445
 446                 sg_size = le16toh(*((uint16_t *)header_ptr));
 447                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 448                 packet_size = sg_size -
 449                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 450
 451
 452                 /* checks if packet all right */
 453                 if (!sg_size)
 454                         errx(5, "Zero segsize");
 455
 456                 /* check sg_size and hwsize */
 457                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 458                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 459                                         "hwsize: %d", sg_size, hw_size);
 460                 }
 461
 462                 hw_data_align =
 463                         RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
 464                         hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
 465
 466                 if (sze->ct_rx_rem_bytes >=
 467                                 (uint16_t)(sg_size -
 468                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 469                         /* no cut */
 470                         /* one packet ready - go to another */
 471                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 472                         packet_len1 = packet_size;
 473                         packet_ptr2 = NULL;
 474                         packet_len2 = 0;
 475
 476                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 477                                 RTE_SZE2_PACKET_HEADER_SIZE;
 478                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 479                                 RTE_SZE2_PACKET_HEADER_SIZE;
 480                 } else {
 481                         /* cut in data */
 482                         if (sze->ct_rx_lck->next == NULL) {
 483                                 errx(6, "Need \"next\" lock, but it is "
 484                                         "missing: %u", sze->ct_rx_rem_bytes);
 485                         }
 486
 487                         /* skip hw data */
 488                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 489                                 uint16_t rem_size = hw_data_align -
 490                                         sze->ct_rx_rem_bytes;
 491
 492                                 /* MOVE to next lock */
 493                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 494                                 sze->ct_rx_cur_ptr =
 495                                         (void *)(((uint8_t *)
 496                                         (sze->ct_rx_lck->start)) + rem_size);
 497
 498                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 499                                 packet_len1 = packet_size;
 500                                 packet_ptr2 = NULL;
 501                                 packet_len2 = 0;
 502
 503                                 sze->ct_rx_cur_ptr +=
 504                                         RTE_SZE2_ALIGN8(packet_size);
 505                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 506                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 507                         } else {
 508                                 /* get pointer and length from first part */
 509                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 510                                         hw_data_align;
 511                                 packet_len1 = sze->ct_rx_rem_bytes -
 512                                         hw_data_align;
 513
 514                                 /* MOVE to next lock */
 515                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 516                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 517
 518                                 /* get pointer and length from second part */
 519                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 520                                 packet_len2 = packet_size - packet_len1;
 521
 522                                 sze->ct_rx_cur_ptr +=
 523                                         RTE_SZE2_ALIGN8(packet_size) -
 524                                         packet_len1;
 525                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 526                                         (RTE_SZE2_ALIGN8(packet_size) -
 527                                          packet_len1);
 528                         }
 529                 }
 530
 531                 if (unlikely(packet_ptr1 == NULL))
 532                         break;
 533
 534                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 535
 536                 if (unlikely(mbuf == NULL)) {
 537                         /*
 538                          * Restore items from sze structure to state after
 539                          * unlocking (eventually locking).
 540                          */
 541                         sze->ct_rx_lck = ct_rx_lck_backup;
 542                         sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
 543                         sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
 544                         break;
 545                 }
 546
 547                 /* get the space available for data in the mbuf */
 548                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 549                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 550                                 RTE_PKTMBUF_HEADROOM);
 551
 552                 if (packet_size <= buf_size) {
 553                         /* sze packet will fit in one mbuf, go ahead and copy */
 554                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 555                                         packet_ptr1, packet_len1);
 556                         if (packet_ptr2 != NULL) {
 557                                 rte_memcpy((void *)
 558                                         (rte_pktmbuf_mtod(mbuf, uint8_t *) +
 559                                         packet_len1), packet_ptr2, packet_len2);
 560                         }
 561                         mbuf->data_len = (uint16_t)packet_size;
 562                 } else {
 563                         /*
 564                          * sze packet will not fit in one mbuf,
 565                          * scatter packet into more mbufs
 566                          */
 567                         struct rte_mbuf *m = mbuf;
 568                         uint16_t len = rte_pktmbuf_tailroom(mbuf);
 569
 570                         /* copy first part of packet */
 571                         /* fill first mbuf */
 572                         rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
 573                                 len);
 574                         packet_len1 -= len;
 575                         packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 576
 577                         while (packet_len1 > 0) {
 578                                 /* fill new mbufs */
 579                                 m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
 580
 581                                 if (unlikely(m->next == NULL)) {
 582                                         rte_pktmbuf_free(mbuf);
 583                                         /*
 584                                          * Restore items from sze structure
 585                                          * to state after unlocking (eventually
 586                                          * locking).
 587                                          */
 588                                         sze->ct_rx_lck = ct_rx_lck_backup;
 589                                         sze->ct_rx_rem_bytes =
 590                                                 ct_rx_rem_bytes_backup;
 591                                         sze->ct_rx_cur_ptr =
 592                                                 ct_rx_cur_ptr_backup;
 593                                         goto finish;
 594                                 }
 595
 596                                 m = m->next;
 597
 598                                 len = RTE_MIN(rte_pktmbuf_tailroom(m),
 599                                         packet_len1);
 600                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 601                                         packet_ptr1, len);
 602
 603                                 (mbuf->nb_segs)++;
 604                                 packet_len1 -= len;
 605                                 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 606                         }
 607
 608                         if (packet_ptr2 != NULL) {
 609                                 /* copy second part of packet, if exists */
 610                                 /* fill the rest of currently last mbuf */
 611                                 len = rte_pktmbuf_tailroom(m);
 612                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 613                                         packet_ptr2, len);
 614                                 packet_len2 -= len;
 615                                 packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
 616
 617                                 while (packet_len2 > 0) {
 618                                         /* fill new mbufs */
 619                                         m->next = rte_pktmbuf_alloc(
 620                                                         sze_q->mb_pool);
 621
 622                                         if (unlikely(m->next == NULL)) {
 623                                                 rte_pktmbuf_free(mbuf);
 624                                                 /*
 625                                                  * Restore items from sze
 626                                                  * structure to state after
 627                                                  * unlocking (eventually
 628                                                  * locking).
 629                                                  */
 630                                                 sze->ct_rx_lck =
 631                                                         ct_rx_lck_backup;
 632                                                 sze->ct_rx_rem_bytes =
 633                                                         ct_rx_rem_bytes_backup;
 634                                                 sze->ct_rx_cur_ptr =
 635                                                         ct_rx_cur_ptr_backup;
 636                                                 goto finish;
 637                                         }
 638
 639                                         m = m->next;
 640
 641                                         len = RTE_MIN(rte_pktmbuf_tailroom(m),
 642                                                 packet_len2);
 643                                         rte_memcpy(
 644                                                 rte_pktmbuf_append(mbuf, len),
 645                                                 packet_ptr2, len);
 646
 647                                         (mbuf->nb_segs)++;
 648                                         packet_len2 -= len;
 649                                         packet_ptr2 = ((uint8_t *)packet_ptr2) +
 650                                                 len;
 651                                 }
 652                         }
 653                 }
 654                 mbuf->pkt_len = packet_size;
 655                 mbuf->port = sze_q->in_port;
 656                 bufs[num_rx] = mbuf;
 657                 num_rx++;
 658                 num_bytes += packet_size;
 659         }
 660
 661 finish:
 662         sze_q->rx_pkts += num_rx;
 663         sze_q->rx_bytes += num_bytes;
 664         return num_rx;
 665 }
 666
 667 static uint16_t
 668 eth_szedata2_tx(void *queue,
 669                 struct rte_mbuf **bufs,
 670                 uint16_t nb_pkts)
 671 {
 672         struct rte_mbuf *mbuf;
 673         struct szedata2_tx_queue *sze_q = queue;
 674         uint16_t num_tx = 0;
 675         uint64_t num_bytes = 0;
 676
 677         const struct szedata_lock *lck;
 678         uint32_t lock_size;
 679         uint32_t lock_size2;
 680         void *dst;
 681         uint32_t pkt_len;
 682         uint32_t hwpkt_len;
 683         uint32_t unlock_size;
 684         uint32_t rem_len;
 685         uint16_t mbuf_segs;
 686         uint16_t pkt_left = nb_pkts;
 687
 688         if (sze_q->sze == NULL || nb_pkts == 0)
 689                 return 0;
 690
 691         while (pkt_left > 0) {
 692                 unlock_size = 0;
 693                 lck = szedata_tx_lock_data(sze_q->sze,
 694                         RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
 695                         sze_q->tx_channel);
 696                 if (lck == NULL)
 697                         continue;
 698
 699                 dst = lck->start;
 700                 lock_size = lck->len;
 701                 lock_size2 = lck->next ? lck->next->len : 0;
 702
 703 next_packet:
 704                 mbuf = bufs[nb_pkts - pkt_left];
 705
 706                 pkt_len = mbuf->pkt_len;
 707                 mbuf_segs = mbuf->nb_segs;
 708
 709                 hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 710                         RTE_SZE2_ALIGN8(pkt_len);
 711
 712                 if (lock_size + lock_size2 < hwpkt_len) {
 713                         szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 714                         continue;
 715                 }
 716
 717                 num_bytes += pkt_len;
 718
 719                 if (lock_size > hwpkt_len) {
 720                         void *tmp_dst;
 721
 722                         rem_len = 0;
 723
 724                         /* write packet length at first 2 bytes in 8B header */
 725                         *((uint16_t *)dst) = htole16(
 726                                         RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 727                                         pkt_len);
 728                         *(((uint16_t *)dst) + 1) = htole16(0);
 729
 730                         /* copy packet from mbuf */
 731                         tmp_dst = ((uint8_t *)(dst)) +
 732                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 733                         if (mbuf_segs == 1) {
 734                                 /*
 735                                  * non-scattered packet,
 736                                  * transmit from one mbuf
 737                                  */
 738                                 rte_memcpy(tmp_dst,
 739                                         rte_pktmbuf_mtod(mbuf, const void *),
 740                                         pkt_len);
 741                         } else {
 742                                 /* scattered packet, transmit from more mbufs */
 743                                 struct rte_mbuf *m = mbuf;
 744                                 while (m) {
 745                                         rte_memcpy(tmp_dst,
 746                                                 rte_pktmbuf_mtod(m,
 747                                                 const void *),
 748                                                 m->data_len);
 749                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 750                                                 m->data_len;
 751                                         m = m->next;
 752                                 }
 753                         }
 754
 755
 756                         dst = ((uint8_t *)dst) + hwpkt_len;
 757                         unlock_size += hwpkt_len;
 758                         lock_size -= hwpkt_len;
 759
 760                         rte_pktmbuf_free(mbuf);
 761                         num_tx++;
 762                         pkt_left--;
 763                         if (pkt_left == 0) {
 764                                 szedata_tx_unlock_data(sze_q->sze, lck,
 765                                         unlock_size);
 766                                 break;
 767                         }
 768                         goto next_packet;
 769                 } else if (lock_size + lock_size2 >= hwpkt_len) {
 770                         void *tmp_dst;
 771                         uint16_t write_len;
 772
 773                         /* write packet length at first 2 bytes in 8B header */
 774                         *((uint16_t *)dst) =
 775                                 htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 776                                         pkt_len);
 777                         *(((uint16_t *)dst) + 1) = htole16(0);
 778
 779                         /*
 780                          * If the raw packet (pkt_len) is smaller than lock_size
 781                          * get the correct length for memcpy
 782                          */
 783                         write_len =
 784                                 pkt_len < lock_size -
 785                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
 786                                 pkt_len :
 787                                 lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 788
 789                         rem_len = hwpkt_len - lock_size;
 790
 791                         tmp_dst = ((uint8_t *)(dst)) +
 792                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 793                         if (mbuf_segs == 1) {
 794                                 /*
 795                                  * non-scattered packet,
 796                                  * transmit from one mbuf
 797                                  */
 798                                 /* copy part of packet to first area */
 799                                 rte_memcpy(tmp_dst,
 800                                         rte_pktmbuf_mtod(mbuf, const void *),
 801                                         write_len);
 802
 803                                 if (lck->next)
 804                                         dst = lck->next->start;
 805
 806                                 /* copy part of packet to second area */
 807                                 rte_memcpy(dst,
 808                                         (const void *)(rte_pktmbuf_mtod(mbuf,
 809                                                         const uint8_t *) +
 810                                         write_len), pkt_len - write_len);
 811                         } else {
 812                                 /* scattered packet, transmit from more mbufs */
 813                                 struct rte_mbuf *m = mbuf;
 814                                 uint16_t written = 0;
 815                                 uint16_t to_write = 0;
 816                                 bool new_mbuf = true;
 817                                 uint16_t write_off = 0;
 818
 819                                 /* copy part of packet to first area */
 820                                 while (m && written < write_len) {
 821                                         to_write = RTE_MIN(m->data_len,
 822                                                         write_len - written);
 823                                         rte_memcpy(tmp_dst,
 824                                                 rte_pktmbuf_mtod(m,
 825                                                         const void *),
 826                                                 to_write);
 827
 828                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 829                                                 to_write;
 830                                         if (m->data_len <= write_len -
 831                                                         written) {
 832                                                 m = m->next;
 833                                                 new_mbuf = true;
 834                                         } else {
 835                                                 new_mbuf = false;
 836                                         }
 837                                         written += to_write;
 838                                 }
 839
 840                                 if (lck->next)
 841                                         dst = lck->next->start;
 842
 843                                 tmp_dst = dst;
 844                                 written = 0;
 845                                 write_off = new_mbuf ? 0 : to_write;
 846
 847                                 /* copy part of packet to second area */
 848                                 while (m && written < pkt_len - write_len) {
 849                                         rte_memcpy(tmp_dst, (const void *)
 850                                                 (rte_pktmbuf_mtod(m,
 851                                                 uint8_t *) + write_off),
 852                                                 m->data_len - write_off);
 853
 854                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 855                                                 (m->data_len - write_off);
 856                                         written += m->data_len - write_off;
 857                                         m = m->next;
 858                                         write_off = 0;
 859                                 }
 860                         }
 861
 862                         dst = ((uint8_t *)dst) + rem_len;
 863                         unlock_size += hwpkt_len;
 864                         lock_size = lock_size2 - rem_len;
 865                         lock_size2 = 0;
 866
 867                         rte_pktmbuf_free(mbuf);
 868                         num_tx++;
 869                 }
 870
 871                 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 872                 pkt_left--;
 873         }
 874
 875         sze_q->tx_pkts += num_tx;
 876         sze_q->err_pkts += nb_pkts - num_tx;
 877         sze_q->tx_bytes += num_bytes;
 878         return num_tx;
 879 }
 880
 881 static int
 882 eth_rx_queue_start(struct rte_eth_dev *dev, uint16_t rxq_id)
 883 {
 884         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 885         int ret;
 886         struct pmd_internals *internals = (struct pmd_internals *)
 887                 dev->data->dev_private;
 888
 889         if (rxq->sze == NULL) {
 890                 uint32_t rx = 1 << rxq->rx_channel;
 891                 uint32_t tx = 0;
 892                 rxq->sze = szedata_open(internals->sze_dev);
 893                 if (rxq->sze == NULL)
 894                         return -EINVAL;
 895                 ret = szedata_subscribe3(rxq->sze, &rx, &tx);
 896                 if (ret != 0 || rx == 0)
 897                         goto err;
 898         }
 899
 900         ret = szedata_start(rxq->sze);
 901         if (ret != 0)
 902                 goto err;
 903         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 904         return 0;
 905
 906 err:
 907         szedata_close(rxq->sze);
 908         rxq->sze = NULL;
 909         return -EINVAL;
 910 }
 911
 912 static int
 913 eth_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rxq_id)
 914 {
 915         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 916
 917         if (rxq->sze != NULL) {
 918                 szedata_close(rxq->sze);
 919                 rxq->sze = NULL;
 920         }
 921
 922         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 923         return 0;
 924 }
 925
 926 static int
 927 eth_tx_queue_start(struct rte_eth_dev *dev, uint16_t txq_id)
 928 {
 929         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 930         int ret;
 931         struct pmd_internals *internals = (struct pmd_internals *)
 932                 dev->data->dev_private;
 933
 934         if (txq->sze == NULL) {
 935                 uint32_t rx = 0;
 936                 uint32_t tx = 1 << txq->tx_channel;
 937                 txq->sze = szedata_open(internals->sze_dev);
 938                 if (txq->sze == NULL)
 939                         return -EINVAL;
 940                 ret = szedata_subscribe3(txq->sze, &rx, &tx);
 941                 if (ret != 0 || tx == 0)
 942                         goto err;
 943         }
 944
 945         ret = szedata_start(txq->sze);
 946         if (ret != 0)
 947                 goto err;
 948         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 949         return 0;
 950
 951 err:
 952         szedata_close(txq->sze);
 953         txq->sze = NULL;
 954         return -EINVAL;
 955 }
 956
 957 static int
 958 eth_tx_queue_stop(struct rte_eth_dev *dev, uint16_t txq_id)
 959 {
 960         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 961
 962         if (txq->sze != NULL) {
 963                 szedata_close(txq->sze);
 964                 txq->sze = NULL;
 965         }
 966
 967         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 968         return 0;
 969 }
 970
 971 static int
 972 eth_dev_start(struct rte_eth_dev *dev)
 973 {
 974         int ret;
 975         uint16_t i;
 976         uint16_t nb_rx = dev->data->nb_rx_queues;
 977         uint16_t nb_tx = dev->data->nb_tx_queues;
 978
 979         for (i = 0; i < nb_rx; i++) {
 980                 ret = eth_rx_queue_start(dev, i);
 981                 if (ret != 0)
 982                         goto err_rx;
 983         }
 984
 985         for (i = 0; i < nb_tx; i++) {
 986                 ret = eth_tx_queue_start(dev, i);
 987                 if (ret != 0)
 988                         goto err_tx;
 989         }
 990
 991         return 0;
 992
 993 err_tx:
 994         for (i = 0; i < nb_tx; i++)
 995                 eth_tx_queue_stop(dev, i);
 996 err_rx:
 997         for (i = 0; i < nb_rx; i++)
 998                 eth_rx_queue_stop(dev, i);
 999         return ret;
1000 }
1001
1002 static void
1003 eth_dev_stop(struct rte_eth_dev *dev)
1004 {
1005         uint16_t i;
1006         uint16_t nb_rx = dev->data->nb_rx_queues;
1007         uint16_t nb_tx = dev->data->nb_tx_queues;
1008
1009         for (i = 0; i < nb_tx; i++)
1010                 eth_tx_queue_stop(dev, i);
1011
1012         for (i = 0; i < nb_rx; i++)
1013                 eth_rx_queue_stop(dev, i);
1014 }
1015
1016 static int
1017 eth_dev_configure(struct rte_eth_dev *dev)
1018 {
1019         struct rte_eth_dev_data *data = dev->data;
1020         if (data->dev_conf.rxmode.enable_scatter == 1) {
1021                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1022                 data->scattered_rx = 1;
1023         } else {
1024                 dev->rx_pkt_burst = eth_szedata2_rx;
1025                 data->scattered_rx = 0;
1026         }
1027         return 0;
1028 }
1029
1030 static void
1031 eth_dev_info(struct rte_eth_dev *dev,
1032                 struct rte_eth_dev_info *dev_info)
1033 {
1034         struct pmd_internals *internals = dev->data->dev_private;
1035         dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1036         dev_info->if_index = 0;
1037         dev_info->max_mac_addrs = 1;
1038         dev_info->max_rx_pktlen = (uint32_t)-1;
1039         dev_info->max_rx_queues = internals->max_rx_queues;
1040         dev_info->max_tx_queues = internals->max_tx_queues;
1041         dev_info->min_rx_bufsize = 0;
1042         dev_info->speed_capa = ETH_LINK_SPEED_100G;
1043 }
1044
1045 static int
1046 eth_stats_get(struct rte_eth_dev *dev,
1047                 struct rte_eth_stats *stats)
1048 {
1049         uint16_t i;
1050         uint16_t nb_rx = dev->data->nb_rx_queues;
1051         uint16_t nb_tx = dev->data->nb_tx_queues;
1052         uint64_t rx_total = 0;
1053         uint64_t tx_total = 0;
1054         uint64_t tx_err_total = 0;
1055         uint64_t rx_total_bytes = 0;
1056         uint64_t tx_total_bytes = 0;
1057
1058         for (i = 0; i < nb_rx; i++) {
1059                 struct szedata2_rx_queue *rxq = dev->data->rx_queues[i];
1060
1061                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1062                         stats->q_ipackets[i] = rxq->rx_pkts;
1063                         stats->q_ibytes[i] = rxq->rx_bytes;
1064                 }
1065                 rx_total += rxq->rx_pkts;
1066                 rx_total_bytes += rxq->rx_bytes;
1067         }
1068
1069         for (i = 0; i < nb_tx; i++) {
1070                 struct szedata2_tx_queue *txq = dev->data->tx_queues[i];
1071
1072                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1073                         stats->q_opackets[i] = txq->tx_pkts;
1074                         stats->q_obytes[i] = txq->tx_bytes;
1075                         stats->q_errors[i] = txq->err_pkts;
1076                 }
1077                 tx_total += txq->tx_pkts;
1078                 tx_total_bytes += txq->tx_bytes;
1079                 tx_err_total += txq->err_pkts;
1080         }
1081
1082         stats->ipackets = rx_total;
1083         stats->opackets = tx_total;
1084         stats->ibytes = rx_total_bytes;
1085         stats->obytes = tx_total_bytes;
1086         stats->oerrors = tx_err_total;
1087
1088         return 0;
1089 }
1090
1091 static void
1092 eth_stats_reset(struct rte_eth_dev *dev)
1093 {
1094         uint16_t i;
1095         uint16_t nb_rx = dev->data->nb_rx_queues;
1096         uint16_t nb_tx = dev->data->nb_tx_queues;
1097         struct pmd_internals *internals = dev->data->dev_private;
1098
1099         for (i = 0; i < nb_rx; i++) {
1100                 internals->rx_queue[i].rx_pkts = 0;
1101                 internals->rx_queue[i].rx_bytes = 0;
1102                 internals->rx_queue[i].err_pkts = 0;
1103         }
1104         for (i = 0; i < nb_tx; i++) {
1105                 internals->tx_queue[i].tx_pkts = 0;
1106                 internals->tx_queue[i].tx_bytes = 0;
1107                 internals->tx_queue[i].err_pkts = 0;
1108         }
1109 }
1110
1111 static void
1112 eth_rx_queue_release(void *q)
1113 {
1114         struct szedata2_rx_queue *rxq = (struct szedata2_rx_queue *)q;
1115         if (rxq->sze != NULL) {
1116                 szedata_close(rxq->sze);
1117                 rxq->sze = NULL;
1118         }
1119 }
1120
1121 static void
1122 eth_tx_queue_release(void *q)
1123 {
1124         struct szedata2_tx_queue *txq = (struct szedata2_tx_queue *)q;
1125         if (txq->sze != NULL) {
1126                 szedata_close(txq->sze);
1127                 txq->sze = NULL;
1128         }
1129 }
1130
1131 static void
1132 eth_dev_close(struct rte_eth_dev *dev)
1133 {
1134         uint16_t i;
1135         uint16_t nb_rx = dev->data->nb_rx_queues;
1136         uint16_t nb_tx = dev->data->nb_tx_queues;
1137
1138         eth_dev_stop(dev);
1139
1140         for (i = 0; i < nb_rx; i++) {
1141                 eth_rx_queue_release(dev->data->rx_queues[i]);
1142                 dev->data->rx_queues[i] = NULL;
1143         }
1144         dev->data->nb_rx_queues = 0;
1145         for (i = 0; i < nb_tx; i++) {
1146                 eth_tx_queue_release(dev->data->tx_queues[i]);
1147                 dev->data->tx_queues[i] = NULL;
1148         }
1149         dev->data->nb_tx_queues = 0;
1150 }
1151
1152 /**
1153  * Function takes value from first IBUF status register.
1154  * Values in IBUF and OBUF should be same.
1155  *
1156  * @param internals
1157  *     Pointer to device private structure.
1158  * @return
1159  *     Link speed constant.
1160  */
1161 static inline enum szedata2_link_speed
1162 get_link_speed(const struct pmd_internals *internals)
1163 {
1164         const volatile struct szedata2_ibuf *ibuf =
1165                 ibuf_ptr_by_index(internals->pci_rsc, 0);
1166         uint32_t speed = (szedata2_read32(&ibuf->ibuf_st) & 0x70) >> 4;
1167         switch (speed) {
1168         case 0x03:
1169                 return SZEDATA2_LINK_SPEED_10G;
1170         case 0x04:
1171                 return SZEDATA2_LINK_SPEED_40G;
1172         case 0x05:
1173                 return SZEDATA2_LINK_SPEED_100G;
1174         default:
1175                 return SZEDATA2_LINK_SPEED_DEFAULT;
1176         }
1177 }
1178
1179 static int
1180 eth_link_update(struct rte_eth_dev *dev,
1181                 int wait_to_complete __rte_unused)
1182 {
1183         struct rte_eth_link link;
1184         struct rte_eth_link *link_ptr = &link;
1185         struct rte_eth_link *dev_link = &dev->data->dev_link;
1186         struct pmd_internals *internals = (struct pmd_internals *)
1187                 dev->data->dev_private;
1188         const volatile struct szedata2_ibuf *ibuf;
1189         uint32_t i;
1190         bool link_is_up = false;
1191
1192         switch (get_link_speed(internals)) {
1193         case SZEDATA2_LINK_SPEED_10G:
1194                 link.link_speed = ETH_SPEED_NUM_10G;
1195                 break;
1196         case SZEDATA2_LINK_SPEED_40G:
1197                 link.link_speed = ETH_SPEED_NUM_40G;
1198                 break;
1199         case SZEDATA2_LINK_SPEED_100G:
1200                 link.link_speed = ETH_SPEED_NUM_100G;
1201                 break;
1202         default:
1203                 link.link_speed = ETH_SPEED_NUM_10G;
1204                 break;
1205         }
1206
1207         /* szedata2 uses only full duplex */
1208         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1209
1210         for (i = 0; i < szedata2_ibuf_count; i++) {
1211                 ibuf = ibuf_ptr_by_index(internals->pci_rsc, i);
1212                 /*
1213                  * Link is considered up if at least one ibuf is enabled
1214                  * and up.
1215                  */
1216                 if (ibuf_is_enabled(ibuf) && ibuf_is_link_up(ibuf)) {
1217                         link_is_up = true;
1218                         break;
1219                 }
1220         }
1221
1222         link.link_status = (link_is_up) ? ETH_LINK_UP : ETH_LINK_DOWN;
1223
1224         link.link_autoneg = ETH_LINK_FIXED;
1225
1226         rte_atomic64_cmpset((uint64_t *)dev_link, *(uint64_t *)dev_link,
1227                         *(uint64_t *)link_ptr);
1228
1229         return 0;
1230 }
1231
1232 static int
1233 eth_dev_set_link_up(struct rte_eth_dev *dev)
1234 {
1235         struct pmd_internals *internals = (struct pmd_internals *)
1236                 dev->data->dev_private;
1237         uint32_t i;
1238
1239         for (i = 0; i < szedata2_ibuf_count; i++)
1240                 ibuf_enable(ibuf_ptr_by_index(internals->pci_rsc, i));
1241         for (i = 0; i < szedata2_obuf_count; i++)
1242                 obuf_enable(obuf_ptr_by_index(internals->pci_rsc, i));
1243         return 0;
1244 }
1245
1246 static int
1247 eth_dev_set_link_down(struct rte_eth_dev *dev)
1248 {
1249         struct pmd_internals *internals = (struct pmd_internals *)
1250                 dev->data->dev_private;
1251         uint32_t i;
1252
1253         for (i = 0; i < szedata2_ibuf_count; i++)
1254                 ibuf_disable(ibuf_ptr_by_index(internals->pci_rsc, i));
1255         for (i = 0; i < szedata2_obuf_count; i++)
1256                 obuf_disable(obuf_ptr_by_index(internals->pci_rsc, i));
1257         return 0;
1258 }
1259
1260 static int
1261 eth_rx_queue_setup(struct rte_eth_dev *dev,
1262                 uint16_t rx_queue_id,
1263                 uint16_t nb_rx_desc __rte_unused,
1264                 unsigned int socket_id __rte_unused,
1265                 const struct rte_eth_rxconf *rx_conf __rte_unused,
1266                 struct rte_mempool *mb_pool)
1267 {
1268         struct pmd_internals *internals = dev->data->dev_private;
1269         struct szedata2_rx_queue *rxq = &internals->rx_queue[rx_queue_id];
1270         int ret;
1271         uint32_t rx = 1 << rx_queue_id;
1272         uint32_t tx = 0;
1273
1274         rxq->sze = szedata_open(internals->sze_dev);
1275         if (rxq->sze == NULL)
1276                 return -EINVAL;
1277         ret = szedata_subscribe3(rxq->sze, &rx, &tx);
1278         if (ret != 0 || rx == 0) {
1279                 szedata_close(rxq->sze);
1280                 rxq->sze = NULL;
1281                 return -EINVAL;
1282         }
1283         rxq->rx_channel = rx_queue_id;
1284         rxq->in_port = dev->data->port_id;
1285         rxq->mb_pool = mb_pool;
1286         rxq->rx_pkts = 0;
1287         rxq->rx_bytes = 0;
1288         rxq->err_pkts = 0;
1289
1290         dev->data->rx_queues[rx_queue_id] = rxq;
1291         return 0;
1292 }
1293
1294 static int
1295 eth_tx_queue_setup(struct rte_eth_dev *dev,
1296                 uint16_t tx_queue_id,
1297                 uint16_t nb_tx_desc __rte_unused,
1298                 unsigned int socket_id __rte_unused,
1299                 const struct rte_eth_txconf *tx_conf __rte_unused)
1300 {
1301         struct pmd_internals *internals = dev->data->dev_private;
1302         struct szedata2_tx_queue *txq = &internals->tx_queue[tx_queue_id];
1303         int ret;
1304         uint32_t rx = 0;
1305         uint32_t tx = 1 << tx_queue_id;
1306
1307         txq->sze = szedata_open(internals->sze_dev);
1308         if (txq->sze == NULL)
1309                 return -EINVAL;
1310         ret = szedata_subscribe3(txq->sze, &rx, &tx);
1311         if (ret != 0 || tx == 0) {
1312                 szedata_close(txq->sze);
1313                 txq->sze = NULL;
1314                 return -EINVAL;
1315         }
1316         txq->tx_channel = tx_queue_id;
1317         txq->tx_pkts = 0;
1318         txq->tx_bytes = 0;
1319         txq->err_pkts = 0;
1320
1321         dev->data->tx_queues[tx_queue_id] = txq;
1322         return 0;
1323 }
1324
1325 static void
1326 eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1327                 struct ether_addr *mac_addr __rte_unused)
1328 {
1329 }
1330
1331 static void
1332 eth_promiscuous_enable(struct rte_eth_dev *dev)
1333 {
1334         struct pmd_internals *internals = (struct pmd_internals *)
1335                 dev->data->dev_private;
1336         uint32_t i;
1337
1338         for (i = 0; i < szedata2_ibuf_count; i++) {
1339                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1340                                 SZEDATA2_MAC_CHMODE_PROMISC);
1341         }
1342 }
1343
1344 static void
1345 eth_promiscuous_disable(struct rte_eth_dev *dev)
1346 {
1347         struct pmd_internals *internals = (struct pmd_internals *)
1348                 dev->data->dev_private;
1349         uint32_t i;
1350
1351         for (i = 0; i < szedata2_ibuf_count; i++) {
1352                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1353                                 SZEDATA2_MAC_CHMODE_ONLY_VALID);
1354         }
1355 }
1356
1357 static void
1358 eth_allmulticast_enable(struct rte_eth_dev *dev)
1359 {
1360         struct pmd_internals *internals = (struct pmd_internals *)
1361                 dev->data->dev_private;
1362         uint32_t i;
1363
1364         for (i = 0; i < szedata2_ibuf_count; i++) {
1365                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1366                                 SZEDATA2_MAC_CHMODE_ALL_MULTICAST);
1367         }
1368 }
1369
1370 static void
1371 eth_allmulticast_disable(struct rte_eth_dev *dev)
1372 {
1373         struct pmd_internals *internals = (struct pmd_internals *)
1374                 dev->data->dev_private;
1375         uint32_t i;
1376
1377         for (i = 0; i < szedata2_ibuf_count; i++) {
1378                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1379                                 SZEDATA2_MAC_CHMODE_ONLY_VALID);
1380         }
1381 }
1382
1383 static const struct eth_dev_ops ops = {
1384         .dev_start          = eth_dev_start,
1385         .dev_stop           = eth_dev_stop,
1386         .dev_set_link_up    = eth_dev_set_link_up,
1387         .dev_set_link_down  = eth_dev_set_link_down,
1388         .dev_close          = eth_dev_close,
1389         .dev_configure      = eth_dev_configure,
1390         .dev_infos_get      = eth_dev_info,
1391         .promiscuous_enable   = eth_promiscuous_enable,
1392         .promiscuous_disable  = eth_promiscuous_disable,
1393         .allmulticast_enable  = eth_allmulticast_enable,
1394         .allmulticast_disable = eth_allmulticast_disable,
1395         .rx_queue_start     = eth_rx_queue_start,
1396         .rx_queue_stop      = eth_rx_queue_stop,
1397         .tx_queue_start     = eth_tx_queue_start,
1398         .tx_queue_stop      = eth_tx_queue_stop,
1399         .rx_queue_setup     = eth_rx_queue_setup,
1400         .tx_queue_setup     = eth_tx_queue_setup,
1401         .rx_queue_release   = eth_rx_queue_release,
1402         .tx_queue_release   = eth_tx_queue_release,
1403         .link_update        = eth_link_update,
1404         .stats_get          = eth_stats_get,
1405         .stats_reset        = eth_stats_reset,
1406         .mac_addr_set       = eth_mac_addr_set,
1407 };
1408
1409 /*
1410  * This function goes through sysfs and looks for an index of szedata2
1411  * device file (/dev/szedataIIX, where X is the index).
1412  *
1413  * @return
1414  *           0 on success
1415  *          -1 on error
1416  */
1417 static int
1418 get_szedata2_index(const struct rte_pci_addr *pcislot_addr, uint32_t *index)
1419 {
1420         DIR *dir;
1421         struct dirent *entry;
1422         int ret;
1423         uint32_t tmp_index;
1424         FILE *fd;
1425         char pcislot_path[PATH_MAX];
1426         uint32_t domain;
1427         uint8_t bus;
1428         uint8_t devid;
1429         uint8_t function;
1430
1431         dir = opendir("/sys/class/combo");
1432         if (dir == NULL)
1433                 return -1;
1434
1435         /*
1436          * Iterate through all combosixX directories.
1437          * When the value in /sys/class/combo/combosixX/device/pcislot
1438          * file is the location of the ethernet device dev, "X" is the
1439          * index of the device.
1440          */
1441         while ((entry = readdir(dir)) != NULL) {
1442                 ret = sscanf(entry->d_name, "combosix%u", &tmp_index);
1443                 if (ret != 1)
1444                         continue;
1445
1446                 snprintf(pcislot_path, PATH_MAX,
1447                         "/sys/class/combo/combosix%u/device/pcislot",
1448                         tmp_index);
1449
1450                 fd = fopen(pcislot_path, "r");
1451                 if (fd == NULL)
1452                         continue;
1453
1454                 ret = fscanf(fd, "%8" SCNx32 ":%2" SCNx8 ":%2" SCNx8 ".%" SCNx8,
1455                                 &domain, &bus, &devid, &function);
1456                 fclose(fd);
1457                 if (ret != 4)
1458                         continue;
1459
1460                 if (pcislot_addr->domain == domain &&
1461                                 pcislot_addr->bus == bus &&
1462                                 pcislot_addr->devid == devid &&
1463                                 pcislot_addr->function == function) {
1464                         *index = tmp_index;
1465                         closedir(dir);
1466                         return 0;
1467                 }
1468         }
1469
1470         closedir(dir);
1471         return -1;
1472 }
1473
1474 static int
1475 rte_szedata2_eth_dev_init(struct rte_eth_dev *dev)
1476 {
1477         struct rte_eth_dev_data *data = dev->data;
1478         struct pmd_internals *internals = (struct pmd_internals *)
1479                 data->dev_private;
1480         struct szedata *szedata_temp;
1481         int ret;
1482         uint32_t szedata2_index;
1483         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1484         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1485         struct rte_mem_resource *pci_rsc =
1486                 &pci_dev->mem_resource[PCI_RESOURCE_NUMBER];
1487         char rsc_filename[PATH_MAX];
1488         void *pci_resource_ptr = NULL;
1489         int fd;
1490
1491         RTE_LOG(INFO, PMD, "Initializing szedata2 device (" PCI_PRI_FMT ")\n",
1492                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1493                         pci_addr->function);
1494
1495         /* Get index of szedata2 device file and create path to device file */
1496         ret = get_szedata2_index(pci_addr, &szedata2_index);
1497         if (ret != 0) {
1498                 RTE_LOG(ERR, PMD, "Failed to get szedata2 device index!\n");
1499                 return -ENODEV;
1500         }
1501         snprintf(internals->sze_dev, PATH_MAX, SZEDATA2_DEV_PATH_FMT,
1502                         szedata2_index);
1503
1504         RTE_LOG(INFO, PMD, "SZEDATA2 path: %s\n", internals->sze_dev);
1505
1506         /*
1507          * Get number of available DMA RX and TX channels, which is maximum
1508          * number of queues that can be created and store it in private device
1509          * data structure.
1510          */
1511         szedata_temp = szedata_open(internals->sze_dev);
1512         if (szedata_temp == NULL) {
1513                 RTE_LOG(ERR, PMD, "szedata_open(): failed to open %s",
1514                                 internals->sze_dev);
1515                 return -EINVAL;
1516         }
1517         internals->max_rx_queues = szedata_ifaces_available(szedata_temp,
1518                         SZE2_DIR_RX);
1519         internals->max_tx_queues = szedata_ifaces_available(szedata_temp,
1520                         SZE2_DIR_TX);
1521         szedata_close(szedata_temp);
1522
1523         RTE_LOG(INFO, PMD, "Available DMA channels RX: %u TX: %u\n",
1524                         internals->max_rx_queues, internals->max_tx_queues);
1525
1526         /* Set rx, tx burst functions */
1527         if (data->dev_conf.rxmode.enable_scatter == 1 ||
1528                 data->scattered_rx == 1) {
1529                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1530                 data->scattered_rx = 1;
1531         } else {
1532                 dev->rx_pkt_burst = eth_szedata2_rx;
1533                 data->scattered_rx = 0;
1534         }
1535         dev->tx_pkt_burst = eth_szedata2_tx;
1536
1537         /* Set function callbacks for Ethernet API */
1538         dev->dev_ops = &ops;
1539
1540         rte_eth_copy_pci_info(dev, pci_dev);
1541
1542         /* mmap pci resource0 file to rte_mem_resource structure */
1543         if (pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr ==
1544                         0) {
1545                 RTE_LOG(ERR, PMD, "Missing resource%u file\n",
1546                                 PCI_RESOURCE_NUMBER);
1547                 return -EINVAL;
1548         }
1549         snprintf(rsc_filename, PATH_MAX,
1550                 "%s/" PCI_PRI_FMT "/resource%u", rte_pci_get_sysfs_path(),
1551                 pci_addr->domain, pci_addr->bus,
1552                 pci_addr->devid, pci_addr->function, PCI_RESOURCE_NUMBER);
1553         fd = open(rsc_filename, O_RDWR);
1554         if (fd < 0) {
1555                 RTE_LOG(ERR, PMD, "Could not open file %s\n", rsc_filename);
1556                 return -EINVAL;
1557         }
1558
1559         pci_resource_ptr = mmap(0,
1560                         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len,
1561                         PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1562         close(fd);
1563         if (pci_resource_ptr == MAP_FAILED) {
1564                 RTE_LOG(ERR, PMD, "Could not mmap file %s (fd = %d)\n",
1565                                 rsc_filename, fd);
1566                 return -EINVAL;
1567         }
1568         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr = pci_resource_ptr;
1569         internals->pci_rsc = pci_rsc;
1570
1571         RTE_LOG(DEBUG, PMD, "resource%u phys_addr = 0x%llx len = %llu "
1572                         "virt addr = %llx\n", PCI_RESOURCE_NUMBER,
1573                         (unsigned long long)pci_rsc->phys_addr,
1574                         (unsigned long long)pci_rsc->len,
1575                         (unsigned long long)pci_rsc->addr);
1576
1577         /* Get link state */
1578         eth_link_update(dev, 0);
1579
1580         /* Allocate space for one mac address */
1581         data->mac_addrs = rte_zmalloc(data->name, sizeof(struct ether_addr),
1582                         RTE_CACHE_LINE_SIZE);
1583         if (data->mac_addrs == NULL) {
1584                 RTE_LOG(ERR, PMD, "Could not alloc space for MAC address!\n");
1585                 munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1586                        pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1587                 return -EINVAL;
1588         }
1589
1590         ether_addr_copy(&eth_addr, data->mac_addrs);
1591
1592         /* At initial state COMBO card is in promiscuous mode so disable it */
1593         eth_promiscuous_disable(dev);
1594
1595         RTE_LOG(INFO, PMD, "szedata2 device ("
1596                         PCI_PRI_FMT ") successfully initialized\n",
1597                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1598                         pci_addr->function);
1599
1600         return 0;
1601 }
1602
1603 static int
1604 rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev)
1605 {
1606         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1607         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1608
1609         rte_free(dev->data->mac_addrs);
1610         dev->data->mac_addrs = NULL;
1611         munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1612                pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1613
1614         RTE_LOG(INFO, PMD, "szedata2 device ("
1615                         PCI_PRI_FMT ") successfully uninitialized\n",
1616                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1617                         pci_addr->function);
1618
1619         return 0;
1620 }
1621
1622 static const struct rte_pci_id rte_szedata2_pci_id_table[] = {
1623         {
1624                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1625                                 PCI_DEVICE_ID_NETCOPE_COMBO80G)
1626         },
1627         {
1628                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1629                                 PCI_DEVICE_ID_NETCOPE_COMBO100G)
1630         },
1631         {
1632                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1633                                 PCI_DEVICE_ID_NETCOPE_COMBO100G2)
1634         },
1635         {
1636                 .vendor_id = 0,
1637         }
1638 };
1639
1640 static int szedata2_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1641         struct rte_pci_device *pci_dev)
1642 {
1643         return rte_eth_dev_pci_generic_probe(pci_dev,
1644                 sizeof(struct pmd_internals), rte_szedata2_eth_dev_init);
1645 }
1646
1647 static int szedata2_eth_pci_remove(struct rte_pci_device *pci_dev)
1648 {
1649         return rte_eth_dev_pci_generic_remove(pci_dev,
1650                 rte_szedata2_eth_dev_uninit);
1651 }
1652
1653 static struct rte_pci_driver szedata2_eth_driver = {
1654         .id_table = rte_szedata2_pci_id_table,
1655         .probe = szedata2_eth_pci_probe,
1656         .remove = szedata2_eth_pci_remove,
1657 };
1658
1659 RTE_PMD_REGISTER_PCI(RTE_SZEDATA2_DRIVER_NAME, szedata2_eth_driver);
1660 RTE_PMD_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table);
1661 RTE_PMD_REGISTER_KMOD_DEP(RTE_SZEDATA2_DRIVER_NAME,
1662         "* combo6core & combov3 & szedata2 & szedata2_cv3");