drivers/net/szedata2/rte_eth_szedata2.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) 2015 - 2016 CESNET
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of CESNET nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <unistd.h>
  36 #include <stdbool.h>
  37 #include <err.h>
  38 #include <sys/types.h>
  39 #include <dirent.h>
  40 #include <sys/stat.h>
  41 #include <fcntl.h>
  42 #include <sys/mman.h>
  43
  44 #include <libsze2.h>
  45
  46 #include <rte_mbuf.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_ethdev_pci.h>
  49 #include <rte_malloc.h>
  50 #include <rte_memcpy.h>
  51 #include <rte_kvargs.h>
  52 #include <rte_dev.h>
  53 #include <rte_atomic.h>
  54
  55 #include "rte_eth_szedata2.h"
  56 #include "szedata2_iobuf.h"
  57
  58 #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
  59 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
  60 #define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
  61
  62 /**
  63  * size of szedata2_packet header with alignment
  64  */
  65 #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
  66
  67 #define RTE_SZEDATA2_DRIVER_NAME net_szedata2
  68
  69 #define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u"
  70
  71 struct szedata2_rx_queue {
  72         struct szedata *sze;
  73         uint8_t rx_channel;
  74         uint8_t in_port;
  75         struct rte_mempool *mb_pool;
  76         volatile uint64_t rx_pkts;
  77         volatile uint64_t rx_bytes;
  78         volatile uint64_t err_pkts;
  79 };
  80
  81 struct szedata2_tx_queue {
  82         struct szedata *sze;
  83         uint8_t tx_channel;
  84         volatile uint64_t tx_pkts;
  85         volatile uint64_t tx_bytes;
  86         volatile uint64_t err_pkts;
  87 };
  88
  89 struct pmd_internals {
  90         struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
  91         struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
  92         uint16_t max_rx_queues;
  93         uint16_t max_tx_queues;
  94         char sze_dev[PATH_MAX];
  95         struct rte_mem_resource *pci_rsc;
  96 };
  97
  98 static struct ether_addr eth_addr = {
  99         .addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
 100 };
 101
 102 static uint16_t
 103 eth_szedata2_rx(void *queue,
 104                 struct rte_mbuf **bufs,
 105                 uint16_t nb_pkts)
 106 {
 107         unsigned int i;
 108         struct rte_mbuf *mbuf;
 109         struct szedata2_rx_queue *sze_q = queue;
 110         struct rte_pktmbuf_pool_private *mbp_priv;
 111         uint16_t num_rx = 0;
 112         uint16_t buf_size;
 113         uint16_t sg_size;
 114         uint16_t hw_size;
 115         uint16_t packet_size;
 116         uint64_t num_bytes = 0;
 117         struct szedata *sze = sze_q->sze;
 118         uint8_t *header_ptr = NULL; /* header of packet */
 119         uint8_t *packet_ptr1 = NULL;
 120         uint8_t *packet_ptr2 = NULL;
 121         uint16_t packet_len1 = 0;
 122         uint16_t packet_len2 = 0;
 123         uint16_t hw_data_align;
 124
 125         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 126                 return 0;
 127
 128         /*
 129          * Reads the given number of packets from szedata2 channel given
 130          * by queue and copies the packet data into a newly allocated mbuf
 131          * to return.
 132          */
 133         for (i = 0; i < nb_pkts; i++) {
 134                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 135
 136                 if (unlikely(mbuf == NULL))
 137                         break;
 138
 139                 /* get the next sze packet */
 140                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 141                                 sze->ct_rx_lck->next == NULL) {
 142                         /* unlock old data */
 143                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 144                         sze->ct_rx_lck_orig = NULL;
 145                         sze->ct_rx_lck = NULL;
 146                 }
 147
 148                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 149                         /* nothing to read, lock new data */
 150                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 151                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 152
 153                         if (sze->ct_rx_lck == NULL) {
 154                                 /* nothing to lock */
 155                                 rte_pktmbuf_free(mbuf);
 156                                 break;
 157                         }
 158
 159                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 160                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 161
 162                         if (!sze->ct_rx_rem_bytes) {
 163                                 rte_pktmbuf_free(mbuf);
 164                                 break;
 165                         }
 166                 }
 167
 168                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 169                         /*
 170                          * cut in header
 171                          * copy parts of header to merge buffer
 172                          */
 173                         if (sze->ct_rx_lck->next == NULL) {
 174                                 rte_pktmbuf_free(mbuf);
 175                                 break;
 176                         }
 177
 178                         /* copy first part of header */
 179                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 180                                         sze->ct_rx_rem_bytes);
 181
 182                         /* copy second part of header */
 183                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 184                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 185                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 186                                 sze->ct_rx_cur_ptr,
 187                                 RTE_SZE2_PACKET_HEADER_SIZE -
 188                                 sze->ct_rx_rem_bytes);
 189
 190                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 191                                 sze->ct_rx_rem_bytes;
 192                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 193                                 RTE_SZE2_PACKET_HEADER_SIZE +
 194                                 sze->ct_rx_rem_bytes;
 195
 196                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 197                 } else {
 198                         /* not cut */
 199                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 200                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 201                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 202                 }
 203
 204                 sg_size = le16toh(*((uint16_t *)header_ptr));
 205                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 206                 packet_size = sg_size -
 207                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 208
 209
 210                 /* checks if packet all right */
 211                 if (!sg_size)
 212                         errx(5, "Zero segsize");
 213
 214                 /* check sg_size and hwsize */
 215                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 216                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 217                                 "hwsize: %d", sg_size, hw_size);
 218                 }
 219
 220                 hw_data_align =
 221                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
 222                         RTE_SZE2_PACKET_HEADER_SIZE;
 223
 224                 if (sze->ct_rx_rem_bytes >=
 225                                 (uint16_t)(sg_size -
 226                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 227                         /* no cut */
 228                         /* one packet ready - go to another */
 229                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 230                         packet_len1 = packet_size;
 231                         packet_ptr2 = NULL;
 232                         packet_len2 = 0;
 233
 234                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 235                                 RTE_SZE2_PACKET_HEADER_SIZE;
 236                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 237                                 RTE_SZE2_PACKET_HEADER_SIZE;
 238                 } else {
 239                         /* cut in data */
 240                         if (sze->ct_rx_lck->next == NULL) {
 241                                 errx(6, "Need \"next\" lock, "
 242                                         "but it is missing: %u",
 243                                         sze->ct_rx_rem_bytes);
 244                         }
 245
 246                         /* skip hw data */
 247                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 248                                 uint16_t rem_size = hw_data_align -
 249                                         sze->ct_rx_rem_bytes;
 250
 251                                 /* MOVE to next lock */
 252                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 253                                 sze->ct_rx_cur_ptr =
 254                                         (void *)(((uint8_t *)
 255                                         (sze->ct_rx_lck->start)) + rem_size);
 256
 257                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 258                                 packet_len1 = packet_size;
 259                                 packet_ptr2 = NULL;
 260                                 packet_len2 = 0;
 261
 262                                 sze->ct_rx_cur_ptr +=
 263                                         RTE_SZE2_ALIGN8(packet_size);
 264                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 265                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 266                         } else {
 267                                 /* get pointer and length from first part */
 268                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 269                                         hw_data_align;
 270                                 packet_len1 = sze->ct_rx_rem_bytes -
 271                                         hw_data_align;
 272
 273                                 /* MOVE to next lock */
 274                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 275                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 276
 277                                 /* get pointer and length from second part */
 278                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 279                                 packet_len2 = packet_size - packet_len1;
 280
 281                                 sze->ct_rx_cur_ptr +=
 282                                         RTE_SZE2_ALIGN8(packet_size) -
 283                                         packet_len1;
 284                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 285                                         (RTE_SZE2_ALIGN8(packet_size) -
 286                                          packet_len1);
 287                         }
 288                 }
 289
 290                 if (unlikely(packet_ptr1 == NULL)) {
 291                         rte_pktmbuf_free(mbuf);
 292                         break;
 293                 }
 294
 295                 /* get the space available for data in the mbuf */
 296                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 297                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 298                                 RTE_PKTMBUF_HEADROOM);
 299
 300                 if (packet_size <= buf_size) {
 301                         /* sze packet will fit in one mbuf, go ahead and copy */
 302                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 303                                         packet_ptr1, packet_len1);
 304                         if (packet_ptr2 != NULL) {
 305                                 rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
 306                                         uint8_t *) + packet_len1),
 307                                         packet_ptr2, packet_len2);
 308                         }
 309                         mbuf->data_len = (uint16_t)packet_size;
 310
 311                         mbuf->pkt_len = packet_size;
 312                         mbuf->port = sze_q->in_port;
 313                         bufs[num_rx] = mbuf;
 314                         num_rx++;
 315                         num_bytes += packet_size;
 316                 } else {
 317                         /*
 318                          * sze packet will not fit in one mbuf,
 319                          * scattered mode is not enabled, drop packet
 320                          */
 321                         RTE_LOG(ERR, PMD,
 322                                 "SZE segment %d bytes will not fit in one mbuf "
 323                                 "(%d bytes), scattered mode is not enabled, "
 324                                 "drop packet!!\n",
 325                                 packet_size, buf_size);
 326                         rte_pktmbuf_free(mbuf);
 327                 }
 328         }
 329
 330         sze_q->rx_pkts += num_rx;
 331         sze_q->rx_bytes += num_bytes;
 332         return num_rx;
 333 }
 334
 335 static uint16_t
 336 eth_szedata2_rx_scattered(void *queue,
 337                 struct rte_mbuf **bufs,
 338                 uint16_t nb_pkts)
 339 {
 340         unsigned int i;
 341         struct rte_mbuf *mbuf;
 342         struct szedata2_rx_queue *sze_q = queue;
 343         struct rte_pktmbuf_pool_private *mbp_priv;
 344         uint16_t num_rx = 0;
 345         uint16_t buf_size;
 346         uint16_t sg_size;
 347         uint16_t hw_size;
 348         uint16_t packet_size;
 349         uint64_t num_bytes = 0;
 350         struct szedata *sze = sze_q->sze;
 351         uint8_t *header_ptr = NULL; /* header of packet */
 352         uint8_t *packet_ptr1 = NULL;
 353         uint8_t *packet_ptr2 = NULL;
 354         uint16_t packet_len1 = 0;
 355         uint16_t packet_len2 = 0;
 356         uint16_t hw_data_align;
 357
 358         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 359                 return 0;
 360
 361         /*
 362          * Reads the given number of packets from szedata2 channel given
 363          * by queue and copies the packet data into a newly allocated mbuf
 364          * to return.
 365          */
 366         for (i = 0; i < nb_pkts; i++) {
 367                 const struct szedata_lock *ct_rx_lck_backup;
 368                 unsigned int ct_rx_rem_bytes_backup;
 369                 unsigned char *ct_rx_cur_ptr_backup;
 370
 371                 /* get the next sze packet */
 372                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 373                                 sze->ct_rx_lck->next == NULL) {
 374                         /* unlock old data */
 375                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 376                         sze->ct_rx_lck_orig = NULL;
 377                         sze->ct_rx_lck = NULL;
 378                 }
 379
 380                 /*
 381                  * Store items from sze structure which can be changed
 382                  * before mbuf allocating. Use these items in case of mbuf
 383                  * allocating failure.
 384                  */
 385                 ct_rx_lck_backup = sze->ct_rx_lck;
 386                 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 387                 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 388
 389                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 390                         /* nothing to read, lock new data */
 391                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 392                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 393
 394                         /*
 395                          * Backup items from sze structure must be updated
 396                          * after locking to contain pointers to new locks.
 397                          */
 398                         ct_rx_lck_backup = sze->ct_rx_lck;
 399                         ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 400                         ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 401
 402                         if (sze->ct_rx_lck == NULL)
 403                                 /* nothing to lock */
 404                                 break;
 405
 406                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 407                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 408
 409                         if (!sze->ct_rx_rem_bytes)
 410                                 break;
 411                 }
 412
 413                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 414                         /*
 415                          * cut in header - copy parts of header to merge buffer
 416                          */
 417                         if (sze->ct_rx_lck->next == NULL)
 418                                 break;
 419
 420                         /* copy first part of header */
 421                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 422                                         sze->ct_rx_rem_bytes);
 423
 424                         /* copy second part of header */
 425                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 426                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 427                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 428                                 sze->ct_rx_cur_ptr,
 429                                 RTE_SZE2_PACKET_HEADER_SIZE -
 430                                 sze->ct_rx_rem_bytes);
 431
 432                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 433                                 sze->ct_rx_rem_bytes;
 434                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 435                                 RTE_SZE2_PACKET_HEADER_SIZE +
 436                                 sze->ct_rx_rem_bytes;
 437
 438                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 439                 } else {
 440                         /* not cut */
 441                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 442                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 443                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 444                 }
 445
 446                 sg_size = le16toh(*((uint16_t *)header_ptr));
 447                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 448                 packet_size = sg_size -
 449                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 450
 451
 452                 /* checks if packet all right */
 453                 if (!sg_size)
 454                         errx(5, "Zero segsize");
 455
 456                 /* check sg_size and hwsize */
 457                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 458                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 459                                         "hwsize: %d", sg_size, hw_size);
 460                 }
 461
 462                 hw_data_align =
 463                         RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
 464                         hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
 465
 466                 if (sze->ct_rx_rem_bytes >=
 467                                 (uint16_t)(sg_size -
 468                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 469                         /* no cut */
 470                         /* one packet ready - go to another */
 471                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 472                         packet_len1 = packet_size;
 473                         packet_ptr2 = NULL;
 474                         packet_len2 = 0;
 475
 476                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 477                                 RTE_SZE2_PACKET_HEADER_SIZE;
 478                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 479                                 RTE_SZE2_PACKET_HEADER_SIZE;
 480                 } else {
 481                         /* cut in data */
 482                         if (sze->ct_rx_lck->next == NULL) {
 483                                 errx(6, "Need \"next\" lock, but it is "
 484                                         "missing: %u", sze->ct_rx_rem_bytes);
 485                         }
 486
 487                         /* skip hw data */
 488                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 489                                 uint16_t rem_size = hw_data_align -
 490                                         sze->ct_rx_rem_bytes;
 491
 492                                 /* MOVE to next lock */
 493                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 494                                 sze->ct_rx_cur_ptr =
 495                                         (void *)(((uint8_t *)
 496                                         (sze->ct_rx_lck->start)) + rem_size);
 497
 498                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 499                                 packet_len1 = packet_size;
 500                                 packet_ptr2 = NULL;
 501                                 packet_len2 = 0;
 502
 503                                 sze->ct_rx_cur_ptr +=
 504                                         RTE_SZE2_ALIGN8(packet_size);
 505                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 506                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 507                         } else {
 508                                 /* get pointer and length from first part */
 509                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 510                                         hw_data_align;
 511                                 packet_len1 = sze->ct_rx_rem_bytes -
 512                                         hw_data_align;
 513
 514                                 /* MOVE to next lock */
 515                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 516                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 517
 518                                 /* get pointer and length from second part */
 519                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 520                                 packet_len2 = packet_size - packet_len1;
 521
 522                                 sze->ct_rx_cur_ptr +=
 523                                         RTE_SZE2_ALIGN8(packet_size) -
 524                                         packet_len1;
 525                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 526                                         (RTE_SZE2_ALIGN8(packet_size) -
 527                                          packet_len1);
 528                         }
 529                 }
 530
 531                 if (unlikely(packet_ptr1 == NULL))
 532                         break;
 533
 534                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 535
 536                 if (unlikely(mbuf == NULL)) {
 537                         /*
 538                          * Restore items from sze structure to state after
 539                          * unlocking (eventually locking).
 540                          */
 541                         sze->ct_rx_lck = ct_rx_lck_backup;
 542                         sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
 543                         sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
 544                         break;
 545                 }
 546
 547                 /* get the space available for data in the mbuf */
 548                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 549                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 550                                 RTE_PKTMBUF_HEADROOM);
 551
 552                 if (packet_size <= buf_size) {
 553                         /* sze packet will fit in one mbuf, go ahead and copy */
 554                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 555                                         packet_ptr1, packet_len1);
 556                         if (packet_ptr2 != NULL) {
 557                                 rte_memcpy((void *)
 558                                         (rte_pktmbuf_mtod(mbuf, uint8_t *) +
 559                                         packet_len1), packet_ptr2, packet_len2);
 560                         }
 561                         mbuf->data_len = (uint16_t)packet_size;
 562                 } else {
 563                         /*
 564                          * sze packet will not fit in one mbuf,
 565                          * scatter packet into more mbufs
 566                          */
 567                         struct rte_mbuf *m = mbuf;
 568                         uint16_t len = rte_pktmbuf_tailroom(mbuf);
 569
 570                         /* copy first part of packet */
 571                         /* fill first mbuf */
 572                         rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
 573                                 len);
 574                         packet_len1 -= len;
 575                         packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 576
 577                         while (packet_len1 > 0) {
 578                                 /* fill new mbufs */
 579                                 m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
 580
 581                                 if (unlikely(m->next == NULL)) {
 582                                         rte_pktmbuf_free(mbuf);
 583                                         /*
 584                                          * Restore items from sze structure
 585                                          * to state after unlocking (eventually
 586                                          * locking).
 587                                          */
 588                                         sze->ct_rx_lck = ct_rx_lck_backup;
 589                                         sze->ct_rx_rem_bytes =
 590                                                 ct_rx_rem_bytes_backup;
 591                                         sze->ct_rx_cur_ptr =
 592                                                 ct_rx_cur_ptr_backup;
 593                                         goto finish;
 594                                 }
 595
 596                                 m = m->next;
 597
 598                                 len = RTE_MIN(rte_pktmbuf_tailroom(m),
 599                                         packet_len1);
 600                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 601                                         packet_ptr1, len);
 602
 603                                 (mbuf->nb_segs)++;
 604                                 packet_len1 -= len;
 605                                 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 606                         }
 607
 608                         if (packet_ptr2 != NULL) {
 609                                 /* copy second part of packet, if exists */
 610                                 /* fill the rest of currently last mbuf */
 611                                 len = rte_pktmbuf_tailroom(m);
 612                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 613                                         packet_ptr2, len);
 614                                 packet_len2 -= len;
 615                                 packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
 616
 617                                 while (packet_len2 > 0) {
 618                                         /* fill new mbufs */
 619                                         m->next = rte_pktmbuf_alloc(
 620                                                         sze_q->mb_pool);
 621
 622                                         if (unlikely(m->next == NULL)) {
 623                                                 rte_pktmbuf_free(mbuf);
 624                                                 /*
 625                                                  * Restore items from sze
 626                                                  * structure to state after
 627                                                  * unlocking (eventually
 628                                                  * locking).
 629                                                  */
 630                                                 sze->ct_rx_lck =
 631                                                         ct_rx_lck_backup;
 632                                                 sze->ct_rx_rem_bytes =
 633                                                         ct_rx_rem_bytes_backup;
 634                                                 sze->ct_rx_cur_ptr =
 635                                                         ct_rx_cur_ptr_backup;
 636                                                 goto finish;
 637                                         }
 638
 639                                         m = m->next;
 640
 641                                         len = RTE_MIN(rte_pktmbuf_tailroom(m),
 642                                                 packet_len2);
 643                                         rte_memcpy(
 644                                                 rte_pktmbuf_append(mbuf, len),
 645                                                 packet_ptr2, len);
 646
 647                                         (mbuf->nb_segs)++;
 648                                         packet_len2 -= len;
 649                                         packet_ptr2 = ((uint8_t *)packet_ptr2) +
 650                                                 len;
 651                                 }
 652                         }
 653                 }
 654                 mbuf->pkt_len = packet_size;
 655                 mbuf->port = sze_q->in_port;
 656                 bufs[num_rx] = mbuf;
 657                 num_rx++;
 658                 num_bytes += packet_size;
 659         }
 660
 661 finish:
 662         sze_q->rx_pkts += num_rx;
 663         sze_q->rx_bytes += num_bytes;
 664         return num_rx;
 665 }
 666
 667 static uint16_t
 668 eth_szedata2_tx(void *queue,
 669                 struct rte_mbuf **bufs,
 670                 uint16_t nb_pkts)
 671 {
 672         struct rte_mbuf *mbuf;
 673         struct szedata2_tx_queue *sze_q = queue;
 674         uint16_t num_tx = 0;
 675         uint64_t num_bytes = 0;
 676
 677         const struct szedata_lock *lck;
 678         uint32_t lock_size;
 679         uint32_t lock_size2;
 680         void *dst;
 681         uint32_t pkt_len;
 682         uint32_t hwpkt_len;
 683         uint32_t unlock_size;
 684         uint32_t rem_len;
 685         uint8_t mbuf_segs;
 686         uint16_t pkt_left = nb_pkts;
 687
 688         if (sze_q->sze == NULL || nb_pkts == 0)
 689                 return 0;
 690
 691         while (pkt_left > 0) {
 692                 unlock_size = 0;
 693                 lck = szedata_tx_lock_data(sze_q->sze,
 694                         RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
 695                         sze_q->tx_channel);
 696                 if (lck == NULL)
 697                         continue;
 698
 699                 dst = lck->start;
 700                 lock_size = lck->len;
 701                 lock_size2 = lck->next ? lck->next->len : 0;
 702
 703 next_packet:
 704                 mbuf = bufs[nb_pkts - pkt_left];
 705
 706                 pkt_len = mbuf->pkt_len;
 707                 mbuf_segs = mbuf->nb_segs;
 708
 709                 hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 710                         RTE_SZE2_ALIGN8(pkt_len);
 711
 712                 if (lock_size + lock_size2 < hwpkt_len) {
 713                         szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 714                         continue;
 715                 }
 716
 717                 num_bytes += pkt_len;
 718
 719                 if (lock_size > hwpkt_len) {
 720                         void *tmp_dst;
 721
 722                         rem_len = 0;
 723
 724                         /* write packet length at first 2 bytes in 8B header */
 725                         *((uint16_t *)dst) = htole16(
 726                                         RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 727                                         pkt_len);
 728                         *(((uint16_t *)dst) + 1) = htole16(0);
 729
 730                         /* copy packet from mbuf */
 731                         tmp_dst = ((uint8_t *)(dst)) +
 732                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 733                         if (mbuf_segs == 1) {
 734                                 /*
 735                                  * non-scattered packet,
 736                                  * transmit from one mbuf
 737                                  */
 738                                 rte_memcpy(tmp_dst,
 739                                         rte_pktmbuf_mtod(mbuf, const void *),
 740                                         pkt_len);
 741                         } else {
 742                                 /* scattered packet, transmit from more mbufs */
 743                                 struct rte_mbuf *m = mbuf;
 744                                 while (m) {
 745                                         rte_memcpy(tmp_dst,
 746                                                 rte_pktmbuf_mtod(m,
 747                                                 const void *),
 748                                                 m->data_len);
 749                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 750                                                 m->data_len;
 751                                         m = m->next;
 752                                 }
 753                         }
 754
 755
 756                         dst = ((uint8_t *)dst) + hwpkt_len;
 757                         unlock_size += hwpkt_len;
 758                         lock_size -= hwpkt_len;
 759
 760                         rte_pktmbuf_free(mbuf);
 761                         num_tx++;
 762                         pkt_left--;
 763                         if (pkt_left == 0) {
 764                                 szedata_tx_unlock_data(sze_q->sze, lck,
 765                                         unlock_size);
 766                                 break;
 767                         }
 768                         goto next_packet;
 769                 } else if (lock_size + lock_size2 >= hwpkt_len) {
 770                         void *tmp_dst;
 771                         uint16_t write_len;
 772
 773                         /* write packet length at first 2 bytes in 8B header */
 774                         *((uint16_t *)dst) =
 775                                 htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 776                                         pkt_len);
 777                         *(((uint16_t *)dst) + 1) = htole16(0);
 778
 779                         /*
 780                          * If the raw packet (pkt_len) is smaller than lock_size
 781                          * get the correct length for memcpy
 782                          */
 783                         write_len =
 784                                 pkt_len < lock_size -
 785                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
 786                                 pkt_len :
 787                                 lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 788
 789                         rem_len = hwpkt_len - lock_size;
 790
 791                         tmp_dst = ((uint8_t *)(dst)) +
 792                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 793                         if (mbuf_segs == 1) {
 794                                 /*
 795                                  * non-scattered packet,
 796                                  * transmit from one mbuf
 797                                  */
 798                                 /* copy part of packet to first area */
 799                                 rte_memcpy(tmp_dst,
 800                                         rte_pktmbuf_mtod(mbuf, const void *),
 801                                         write_len);
 802
 803                                 if (lck->next)
 804                                         dst = lck->next->start;
 805
 806                                 /* copy part of packet to second area */
 807                                 rte_memcpy(dst,
 808                                         (const void *)(rte_pktmbuf_mtod(mbuf,
 809                                                         const uint8_t *) +
 810                                         write_len), pkt_len - write_len);
 811                         } else {
 812                                 /* scattered packet, transmit from more mbufs */
 813                                 struct rte_mbuf *m = mbuf;
 814                                 uint16_t written = 0;
 815                                 uint16_t to_write = 0;
 816                                 bool new_mbuf = true;
 817                                 uint16_t write_off = 0;
 818
 819                                 /* copy part of packet to first area */
 820                                 while (m && written < write_len) {
 821                                         to_write = RTE_MIN(m->data_len,
 822                                                         write_len - written);
 823                                         rte_memcpy(tmp_dst,
 824                                                 rte_pktmbuf_mtod(m,
 825                                                         const void *),
 826                                                 to_write);
 827
 828                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 829                                                 to_write;
 830                                         if (m->data_len <= write_len -
 831                                                         written) {
 832                                                 m = m->next;
 833                                                 new_mbuf = true;
 834                                         } else {
 835                                                 new_mbuf = false;
 836                                         }
 837                                         written += to_write;
 838                                 }
 839
 840                                 if (lck->next)
 841                                         dst = lck->next->start;
 842
 843                                 tmp_dst = dst;
 844                                 written = 0;
 845                                 write_off = new_mbuf ? 0 : to_write;
 846
 847                                 /* copy part of packet to second area */
 848                                 while (m && written < pkt_len - write_len) {
 849                                         rte_memcpy(tmp_dst, (const void *)
 850                                                 (rte_pktmbuf_mtod(m,
 851                                                 uint8_t *) + write_off),
 852                                                 m->data_len - write_off);
 853
 854                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 855                                                 (m->data_len - write_off);
 856                                         written += m->data_len - write_off;
 857                                         m = m->next;
 858                                         write_off = 0;
 859                                 }
 860                         }
 861
 862                         dst = ((uint8_t *)dst) + rem_len;
 863                         unlock_size += hwpkt_len;
 864                         lock_size = lock_size2 - rem_len;
 865                         lock_size2 = 0;
 866
 867                         rte_pktmbuf_free(mbuf);
 868                         num_tx++;
 869                 }
 870
 871                 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 872                 pkt_left--;
 873         }
 874
 875         sze_q->tx_pkts += num_tx;
 876         sze_q->err_pkts += nb_pkts - num_tx;
 877         sze_q->tx_bytes += num_bytes;
 878         return num_tx;
 879 }
 880
 881 static int
 882 eth_rx_queue_start(struct rte_eth_dev *dev, uint16_t rxq_id)
 883 {
 884         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 885         int ret;
 886         struct pmd_internals *internals = (struct pmd_internals *)
 887                 dev->data->dev_private;
 888
 889         if (rxq->sze == NULL) {
 890                 uint32_t rx = 1 << rxq->rx_channel;
 891                 uint32_t tx = 0;
 892                 rxq->sze = szedata_open(internals->sze_dev);
 893                 if (rxq->sze == NULL)
 894                         return -EINVAL;
 895                 ret = szedata_subscribe3(rxq->sze, &rx, &tx);
 896                 if (ret != 0 || rx == 0)
 897                         goto err;
 898         }
 899
 900         ret = szedata_start(rxq->sze);
 901         if (ret != 0)
 902                 goto err;
 903         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 904         return 0;
 905
 906 err:
 907         szedata_close(rxq->sze);
 908         rxq->sze = NULL;
 909         return -EINVAL;
 910 }
 911
 912 static int
 913 eth_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rxq_id)
 914 {
 915         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 916
 917         if (rxq->sze != NULL) {
 918                 szedata_close(rxq->sze);
 919                 rxq->sze = NULL;
 920         }
 921
 922         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 923         return 0;
 924 }
 925
 926 static int
 927 eth_tx_queue_start(struct rte_eth_dev *dev, uint16_t txq_id)
 928 {
 929         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 930         int ret;
 931         struct pmd_internals *internals = (struct pmd_internals *)
 932                 dev->data->dev_private;
 933
 934         if (txq->sze == NULL) {
 935                 uint32_t rx = 0;
 936                 uint32_t tx = 1 << txq->tx_channel;
 937                 txq->sze = szedata_open(internals->sze_dev);
 938                 if (txq->sze == NULL)
 939                         return -EINVAL;
 940                 ret = szedata_subscribe3(txq->sze, &rx, &tx);
 941                 if (ret != 0 || tx == 0)
 942                         goto err;
 943         }
 944
 945         ret = szedata_start(txq->sze);
 946         if (ret != 0)
 947                 goto err;
 948         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 949         return 0;
 950
 951 err:
 952         szedata_close(txq->sze);
 953         txq->sze = NULL;
 954         return -EINVAL;
 955 }
 956
 957 static int
 958 eth_tx_queue_stop(struct rte_eth_dev *dev, uint16_t txq_id)
 959 {
 960         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 961
 962         if (txq->sze != NULL) {
 963                 szedata_close(txq->sze);
 964                 txq->sze = NULL;
 965         }
 966
 967         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 968         return 0;
 969 }
 970
 971 static int
 972 eth_dev_start(struct rte_eth_dev *dev)
 973 {
 974         int ret;
 975         uint16_t i;
 976         uint16_t nb_rx = dev->data->nb_rx_queues;
 977         uint16_t nb_tx = dev->data->nb_tx_queues;
 978
 979         for (i = 0; i < nb_rx; i++) {
 980                 ret = eth_rx_queue_start(dev, i);
 981                 if (ret != 0)
 982                         goto err_rx;
 983         }
 984
 985         for (i = 0; i < nb_tx; i++) {
 986                 ret = eth_tx_queue_start(dev, i);
 987                 if (ret != 0)
 988                         goto err_tx;
 989         }
 990
 991         return 0;
 992
 993 err_tx:
 994         for (i = 0; i < nb_tx; i++)
 995                 eth_tx_queue_stop(dev, i);
 996 err_rx:
 997         for (i = 0; i < nb_rx; i++)
 998                 eth_rx_queue_stop(dev, i);
 999         return ret;
1000 }
1001
1002 static void
1003 eth_dev_stop(struct rte_eth_dev *dev)
1004 {
1005         uint16_t i;
1006         uint16_t nb_rx = dev->data->nb_rx_queues;
1007         uint16_t nb_tx = dev->data->nb_tx_queues;
1008
1009         for (i = 0; i < nb_tx; i++)
1010                 eth_tx_queue_stop(dev, i);
1011
1012         for (i = 0; i < nb_rx; i++)
1013                 eth_rx_queue_stop(dev, i);
1014 }
1015
1016 static int
1017 eth_dev_configure(struct rte_eth_dev *dev)
1018 {
1019         struct rte_eth_dev_data *data = dev->data;
1020         if (data->dev_conf.rxmode.enable_scatter == 1) {
1021                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1022                 data->scattered_rx = 1;
1023         } else {
1024                 dev->rx_pkt_burst = eth_szedata2_rx;
1025                 data->scattered_rx = 0;
1026         }
1027         return 0;
1028 }
1029
1030 static void
1031 eth_dev_info(struct rte_eth_dev *dev,
1032                 struct rte_eth_dev_info *dev_info)
1033 {
1034         struct pmd_internals *internals = dev->data->dev_private;
1035         dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1036         dev_info->if_index = 0;
1037         dev_info->max_mac_addrs = 1;
1038         dev_info->max_rx_pktlen = (uint32_t)-1;
1039         dev_info->max_rx_queues = internals->max_rx_queues;
1040         dev_info->max_tx_queues = internals->max_tx_queues;
1041         dev_info->min_rx_bufsize = 0;
1042         dev_info->speed_capa = ETH_LINK_SPEED_100G;
1043 }
1044
1045 static void
1046 eth_stats_get(struct rte_eth_dev *dev,
1047                 struct rte_eth_stats *stats)
1048 {
1049         uint16_t i;
1050         uint16_t nb_rx = dev->data->nb_rx_queues;
1051         uint16_t nb_tx = dev->data->nb_tx_queues;
1052         uint64_t rx_total = 0;
1053         uint64_t tx_total = 0;
1054         uint64_t tx_err_total = 0;
1055         uint64_t rx_total_bytes = 0;
1056         uint64_t tx_total_bytes = 0;
1057         const struct pmd_internals *internals = dev->data->dev_private;
1058
1059         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_rx; i++) {
1060                 stats->q_ipackets[i] = internals->rx_queue[i].rx_pkts;
1061                 stats->q_ibytes[i] = internals->rx_queue[i].rx_bytes;
1062                 rx_total += stats->q_ipackets[i];
1063                 rx_total_bytes += stats->q_ibytes[i];
1064         }
1065
1066         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_tx; i++) {
1067                 stats->q_opackets[i] = internals->tx_queue[i].tx_pkts;
1068                 stats->q_obytes[i] = internals->tx_queue[i].tx_bytes;
1069                 stats->q_errors[i] = internals->tx_queue[i].err_pkts;
1070                 tx_total += stats->q_opackets[i];
1071                 tx_total_bytes += stats->q_obytes[i];
1072                 tx_err_total += stats->q_errors[i];
1073         }
1074
1075         stats->ipackets = rx_total;
1076         stats->opackets = tx_total;
1077         stats->ibytes = rx_total_bytes;
1078         stats->obytes = tx_total_bytes;
1079         stats->oerrors = tx_err_total;
1080 }
1081
1082 static void
1083 eth_stats_reset(struct rte_eth_dev *dev)
1084 {
1085         uint16_t i;
1086         uint16_t nb_rx = dev->data->nb_rx_queues;
1087         uint16_t nb_tx = dev->data->nb_tx_queues;
1088         struct pmd_internals *internals = dev->data->dev_private;
1089
1090         for (i = 0; i < nb_rx; i++) {
1091                 internals->rx_queue[i].rx_pkts = 0;
1092                 internals->rx_queue[i].rx_bytes = 0;
1093                 internals->rx_queue[i].err_pkts = 0;
1094         }
1095         for (i = 0; i < nb_tx; i++) {
1096                 internals->tx_queue[i].tx_pkts = 0;
1097                 internals->tx_queue[i].tx_bytes = 0;
1098                 internals->tx_queue[i].err_pkts = 0;
1099         }
1100 }
1101
1102 static void
1103 eth_rx_queue_release(void *q)
1104 {
1105         struct szedata2_rx_queue *rxq = (struct szedata2_rx_queue *)q;
1106         if (rxq->sze != NULL) {
1107                 szedata_close(rxq->sze);
1108                 rxq->sze = NULL;
1109         }
1110 }
1111
1112 static void
1113 eth_tx_queue_release(void *q)
1114 {
1115         struct szedata2_tx_queue *txq = (struct szedata2_tx_queue *)q;
1116         if (txq->sze != NULL) {
1117                 szedata_close(txq->sze);
1118                 txq->sze = NULL;
1119         }
1120 }
1121
1122 static void
1123 eth_dev_close(struct rte_eth_dev *dev)
1124 {
1125         uint16_t i;
1126         uint16_t nb_rx = dev->data->nb_rx_queues;
1127         uint16_t nb_tx = dev->data->nb_tx_queues;
1128
1129         eth_dev_stop(dev);
1130
1131         for (i = 0; i < nb_rx; i++) {
1132                 eth_rx_queue_release(dev->data->rx_queues[i]);
1133                 dev->data->rx_queues[i] = NULL;
1134         }
1135         dev->data->nb_rx_queues = 0;
1136         for (i = 0; i < nb_tx; i++) {
1137                 eth_tx_queue_release(dev->data->tx_queues[i]);
1138                 dev->data->tx_queues[i] = NULL;
1139         }
1140         dev->data->nb_tx_queues = 0;
1141 }
1142
1143 /**
1144  * Function takes value from first IBUF status register.
1145  * Values in IBUF and OBUF should be same.
1146  *
1147  * @param internals
1148  *     Pointer to device private structure.
1149  * @return
1150  *     Link speed constant.
1151  */
1152 static inline enum szedata2_link_speed
1153 get_link_speed(const struct pmd_internals *internals)
1154 {
1155         const volatile struct szedata2_ibuf *ibuf =
1156                 ibuf_ptr_by_index(internals->pci_rsc, 0);
1157         uint32_t speed = (szedata2_read32(&ibuf->ibuf_st) & 0x70) >> 4;
1158         switch (speed) {
1159         case 0x03:
1160                 return SZEDATA2_LINK_SPEED_10G;
1161         case 0x04:
1162                 return SZEDATA2_LINK_SPEED_40G;
1163         case 0x05:
1164                 return SZEDATA2_LINK_SPEED_100G;
1165         default:
1166                 return SZEDATA2_LINK_SPEED_DEFAULT;
1167         }
1168 }
1169
1170 static int
1171 eth_link_update(struct rte_eth_dev *dev,
1172                 int wait_to_complete __rte_unused)
1173 {
1174         struct rte_eth_link link;
1175         struct rte_eth_link *link_ptr = &link;
1176         struct rte_eth_link *dev_link = &dev->data->dev_link;
1177         struct pmd_internals *internals = (struct pmd_internals *)
1178                 dev->data->dev_private;
1179         const volatile struct szedata2_ibuf *ibuf;
1180         uint32_t i;
1181         bool link_is_up = false;
1182
1183         switch (get_link_speed(internals)) {
1184         case SZEDATA2_LINK_SPEED_10G:
1185                 link.link_speed = ETH_SPEED_NUM_10G;
1186                 break;
1187         case SZEDATA2_LINK_SPEED_40G:
1188                 link.link_speed = ETH_SPEED_NUM_40G;
1189                 break;
1190         case SZEDATA2_LINK_SPEED_100G:
1191                 link.link_speed = ETH_SPEED_NUM_100G;
1192                 break;
1193         default:
1194                 link.link_speed = ETH_SPEED_NUM_10G;
1195                 break;
1196         }
1197
1198         /* szedata2 uses only full duplex */
1199         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1200
1201         for (i = 0; i < szedata2_ibuf_count; i++) {
1202                 ibuf = ibuf_ptr_by_index(internals->pci_rsc, i);
1203                 /*
1204                  * Link is considered up if at least one ibuf is enabled
1205                  * and up.
1206                  */
1207                 if (ibuf_is_enabled(ibuf) && ibuf_is_link_up(ibuf)) {
1208                         link_is_up = true;
1209                         break;
1210                 }
1211         }
1212
1213         link.link_status = (link_is_up) ? ETH_LINK_UP : ETH_LINK_DOWN;
1214
1215         link.link_autoneg = ETH_LINK_SPEED_FIXED;
1216
1217         rte_atomic64_cmpset((uint64_t *)dev_link, *(uint64_t *)dev_link,
1218                         *(uint64_t *)link_ptr);
1219
1220         return 0;
1221 }
1222
1223 static int
1224 eth_dev_set_link_up(struct rte_eth_dev *dev)
1225 {
1226         struct pmd_internals *internals = (struct pmd_internals *)
1227                 dev->data->dev_private;
1228         uint32_t i;
1229
1230         for (i = 0; i < szedata2_ibuf_count; i++)
1231                 ibuf_enable(ibuf_ptr_by_index(internals->pci_rsc, i));
1232         for (i = 0; i < szedata2_obuf_count; i++)
1233                 obuf_enable(obuf_ptr_by_index(internals->pci_rsc, i));
1234         return 0;
1235 }
1236
1237 static int
1238 eth_dev_set_link_down(struct rte_eth_dev *dev)
1239 {
1240         struct pmd_internals *internals = (struct pmd_internals *)
1241                 dev->data->dev_private;
1242         uint32_t i;
1243
1244         for (i = 0; i < szedata2_ibuf_count; i++)
1245                 ibuf_disable(ibuf_ptr_by_index(internals->pci_rsc, i));
1246         for (i = 0; i < szedata2_obuf_count; i++)
1247                 obuf_disable(obuf_ptr_by_index(internals->pci_rsc, i));
1248         return 0;
1249 }
1250
1251 static int
1252 eth_rx_queue_setup(struct rte_eth_dev *dev,
1253                 uint16_t rx_queue_id,
1254                 uint16_t nb_rx_desc __rte_unused,
1255                 unsigned int socket_id __rte_unused,
1256                 const struct rte_eth_rxconf *rx_conf __rte_unused,
1257                 struct rte_mempool *mb_pool)
1258 {
1259         struct pmd_internals *internals = dev->data->dev_private;
1260         struct szedata2_rx_queue *rxq = &internals->rx_queue[rx_queue_id];
1261         int ret;
1262         uint32_t rx = 1 << rx_queue_id;
1263         uint32_t tx = 0;
1264
1265         rxq->sze = szedata_open(internals->sze_dev);
1266         if (rxq->sze == NULL)
1267                 return -EINVAL;
1268         ret = szedata_subscribe3(rxq->sze, &rx, &tx);
1269         if (ret != 0 || rx == 0) {
1270                 szedata_close(rxq->sze);
1271                 rxq->sze = NULL;
1272                 return -EINVAL;
1273         }
1274         rxq->rx_channel = rx_queue_id;
1275         rxq->in_port = dev->data->port_id;
1276         rxq->mb_pool = mb_pool;
1277         rxq->rx_pkts = 0;
1278         rxq->rx_bytes = 0;
1279         rxq->err_pkts = 0;
1280
1281         dev->data->rx_queues[rx_queue_id] = rxq;
1282         return 0;
1283 }
1284
1285 static int
1286 eth_tx_queue_setup(struct rte_eth_dev *dev,
1287                 uint16_t tx_queue_id,
1288                 uint16_t nb_tx_desc __rte_unused,
1289                 unsigned int socket_id __rte_unused,
1290                 const struct rte_eth_txconf *tx_conf __rte_unused)
1291 {
1292         struct pmd_internals *internals = dev->data->dev_private;
1293         struct szedata2_tx_queue *txq = &internals->tx_queue[tx_queue_id];
1294         int ret;
1295         uint32_t rx = 0;
1296         uint32_t tx = 1 << tx_queue_id;
1297
1298         txq->sze = szedata_open(internals->sze_dev);
1299         if (txq->sze == NULL)
1300                 return -EINVAL;
1301         ret = szedata_subscribe3(txq->sze, &rx, &tx);
1302         if (ret != 0 || tx == 0) {
1303                 szedata_close(txq->sze);
1304                 txq->sze = NULL;
1305                 return -EINVAL;
1306         }
1307         txq->tx_channel = tx_queue_id;
1308         txq->tx_pkts = 0;
1309         txq->tx_bytes = 0;
1310         txq->err_pkts = 0;
1311
1312         dev->data->tx_queues[tx_queue_id] = txq;
1313         return 0;
1314 }
1315
1316 static void
1317 eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1318                 struct ether_addr *mac_addr __rte_unused)
1319 {
1320 }
1321
1322 static void
1323 eth_promiscuous_enable(struct rte_eth_dev *dev)
1324 {
1325         struct pmd_internals *internals = (struct pmd_internals *)
1326                 dev->data->dev_private;
1327         uint32_t i;
1328
1329         for (i = 0; i < szedata2_ibuf_count; i++) {
1330                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1331                                 SZEDATA2_MAC_CHMODE_PROMISC);
1332         }
1333 }
1334
1335 static void
1336 eth_promiscuous_disable(struct rte_eth_dev *dev)
1337 {
1338         struct pmd_internals *internals = (struct pmd_internals *)
1339                 dev->data->dev_private;
1340         uint32_t i;
1341
1342         for (i = 0; i < szedata2_ibuf_count; i++) {
1343                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1344                                 SZEDATA2_MAC_CHMODE_ONLY_VALID);
1345         }
1346 }
1347
1348 static void
1349 eth_allmulticast_enable(struct rte_eth_dev *dev)
1350 {
1351         struct pmd_internals *internals = (struct pmd_internals *)
1352                 dev->data->dev_private;
1353         uint32_t i;
1354
1355         for (i = 0; i < szedata2_ibuf_count; i++) {
1356                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1357                                 SZEDATA2_MAC_CHMODE_ALL_MULTICAST);
1358         }
1359 }
1360
1361 static void
1362 eth_allmulticast_disable(struct rte_eth_dev *dev)
1363 {
1364         struct pmd_internals *internals = (struct pmd_internals *)
1365                 dev->data->dev_private;
1366         uint32_t i;
1367
1368         for (i = 0; i < szedata2_ibuf_count; i++) {
1369                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1370                                 SZEDATA2_MAC_CHMODE_ONLY_VALID);
1371         }
1372 }
1373
1374 static const struct eth_dev_ops ops = {
1375         .dev_start          = eth_dev_start,
1376         .dev_stop           = eth_dev_stop,
1377         .dev_set_link_up    = eth_dev_set_link_up,
1378         .dev_set_link_down  = eth_dev_set_link_down,
1379         .dev_close          = eth_dev_close,
1380         .dev_configure      = eth_dev_configure,
1381         .dev_infos_get      = eth_dev_info,
1382         .promiscuous_enable   = eth_promiscuous_enable,
1383         .promiscuous_disable  = eth_promiscuous_disable,
1384         .allmulticast_enable  = eth_allmulticast_enable,
1385         .allmulticast_disable = eth_allmulticast_disable,
1386         .rx_queue_start     = eth_rx_queue_start,
1387         .rx_queue_stop      = eth_rx_queue_stop,
1388         .tx_queue_start     = eth_tx_queue_start,
1389         .tx_queue_stop      = eth_tx_queue_stop,
1390         .rx_queue_setup     = eth_rx_queue_setup,
1391         .tx_queue_setup     = eth_tx_queue_setup,
1392         .rx_queue_release   = eth_rx_queue_release,
1393         .tx_queue_release   = eth_tx_queue_release,
1394         .link_update        = eth_link_update,
1395         .stats_get          = eth_stats_get,
1396         .stats_reset        = eth_stats_reset,
1397         .mac_addr_set       = eth_mac_addr_set,
1398 };
1399
1400 /*
1401  * This function goes through sysfs and looks for an index of szedata2
1402  * device file (/dev/szedataIIX, where X is the index).
1403  *
1404  * @return
1405  *           0 on success
1406  *          -1 on error
1407  */
1408 static int
1409 get_szedata2_index(const struct rte_pci_addr *pcislot_addr, uint32_t *index)
1410 {
1411         DIR *dir;
1412         struct dirent *entry;
1413         int ret;
1414         uint32_t tmp_index;
1415         FILE *fd;
1416         char pcislot_path[PATH_MAX];
1417         uint32_t domain;
1418         uint32_t bus;
1419         uint32_t devid;
1420         uint32_t function;
1421
1422         dir = opendir("/sys/class/combo");
1423         if (dir == NULL)
1424                 return -1;
1425
1426         /*
1427          * Iterate through all combosixX directories.
1428          * When the value in /sys/class/combo/combosixX/device/pcislot
1429          * file is the location of the ethernet device dev, "X" is the
1430          * index of the device.
1431          */
1432         while ((entry = readdir(dir)) != NULL) {
1433                 ret = sscanf(entry->d_name, "combosix%u", &tmp_index);
1434                 if (ret != 1)
1435                         continue;
1436
1437                 snprintf(pcislot_path, PATH_MAX,
1438                         "/sys/class/combo/combosix%u/device/pcislot",
1439                         tmp_index);
1440
1441                 fd = fopen(pcislot_path, "r");
1442                 if (fd == NULL)
1443                         continue;
1444
1445                 ret = fscanf(fd, "%4" PRIx16 ":%2" PRIx8 ":%2" PRIx8 ".%" PRIx8,
1446                                 &domain, &bus, &devid, &function);
1447                 fclose(fd);
1448                 if (ret != 4)
1449                         continue;
1450
1451                 if (pcislot_addr->domain == domain &&
1452                                 pcislot_addr->bus == bus &&
1453                                 pcislot_addr->devid == devid &&
1454                                 pcislot_addr->function == function) {
1455                         *index = tmp_index;
1456                         closedir(dir);
1457                         return 0;
1458                 }
1459         }
1460
1461         closedir(dir);
1462         return -1;
1463 }
1464
1465 static int
1466 rte_szedata2_eth_dev_init(struct rte_eth_dev *dev)
1467 {
1468         struct rte_eth_dev_data *data = dev->data;
1469         struct pmd_internals *internals = (struct pmd_internals *)
1470                 data->dev_private;
1471         struct szedata *szedata_temp;
1472         int ret;
1473         uint32_t szedata2_index;
1474         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1475         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1476         struct rte_mem_resource *pci_rsc =
1477                 &pci_dev->mem_resource[PCI_RESOURCE_NUMBER];
1478         char rsc_filename[PATH_MAX];
1479         void *pci_resource_ptr = NULL;
1480         int fd;
1481
1482         RTE_LOG(INFO, PMD, "Initializing szedata2 device (" PCI_PRI_FMT ")\n",
1483                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1484                         pci_addr->function);
1485
1486         /* Get index of szedata2 device file and create path to device file */
1487         ret = get_szedata2_index(pci_addr, &szedata2_index);
1488         if (ret != 0) {
1489                 RTE_LOG(ERR, PMD, "Failed to get szedata2 device index!\n");
1490                 return -ENODEV;
1491         }
1492         snprintf(internals->sze_dev, PATH_MAX, SZEDATA2_DEV_PATH_FMT,
1493                         szedata2_index);
1494
1495         RTE_LOG(INFO, PMD, "SZEDATA2 path: %s\n", internals->sze_dev);
1496
1497         /*
1498          * Get number of available DMA RX and TX channels, which is maximum
1499          * number of queues that can be created and store it in private device
1500          * data structure.
1501          */
1502         szedata_temp = szedata_open(internals->sze_dev);
1503         if (szedata_temp == NULL) {
1504                 RTE_LOG(ERR, PMD, "szedata_open(): failed to open %s",
1505                                 internals->sze_dev);
1506                 return -EINVAL;
1507         }
1508         internals->max_rx_queues = szedata_ifaces_available(szedata_temp,
1509                         SZE2_DIR_RX);
1510         internals->max_tx_queues = szedata_ifaces_available(szedata_temp,
1511                         SZE2_DIR_TX);
1512         szedata_close(szedata_temp);
1513
1514         RTE_LOG(INFO, PMD, "Available DMA channels RX: %u TX: %u\n",
1515                         internals->max_rx_queues, internals->max_tx_queues);
1516
1517         /* Set rx, tx burst functions */
1518         if (data->dev_conf.rxmode.enable_scatter == 1 ||
1519                 data->scattered_rx == 1) {
1520                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1521                 data->scattered_rx = 1;
1522         } else {
1523                 dev->rx_pkt_burst = eth_szedata2_rx;
1524                 data->scattered_rx = 0;
1525         }
1526         dev->tx_pkt_burst = eth_szedata2_tx;
1527
1528         /* Set function callbacks for Ethernet API */
1529         dev->dev_ops = &ops;
1530
1531         rte_eth_copy_pci_info(dev, pci_dev);
1532
1533         /* mmap pci resource0 file to rte_mem_resource structure */
1534         if (pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr ==
1535                         0) {
1536                 RTE_LOG(ERR, PMD, "Missing resource%u file\n",
1537                                 PCI_RESOURCE_NUMBER);
1538                 return -EINVAL;
1539         }
1540         snprintf(rsc_filename, PATH_MAX,
1541                 "%s/" PCI_PRI_FMT "/resource%u", pci_get_sysfs_path(),
1542                 pci_addr->domain, pci_addr->bus,
1543                 pci_addr->devid, pci_addr->function, PCI_RESOURCE_NUMBER);
1544         fd = open(rsc_filename, O_RDWR);
1545         if (fd < 0) {
1546                 RTE_LOG(ERR, PMD, "Could not open file %s\n", rsc_filename);
1547                 return -EINVAL;
1548         }
1549
1550         pci_resource_ptr = mmap(0,
1551                         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len,
1552                         PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1553         close(fd);
1554         if (pci_resource_ptr == NULL) {
1555                 RTE_LOG(ERR, PMD, "Could not mmap file %s (fd = %d)\n",
1556                                 rsc_filename, fd);
1557                 return -EINVAL;
1558         }
1559         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr = pci_resource_ptr;
1560         internals->pci_rsc = pci_rsc;
1561
1562         RTE_LOG(DEBUG, PMD, "resource%u phys_addr = 0x%llx len = %llu "
1563                         "virt addr = %llx\n", PCI_RESOURCE_NUMBER,
1564                         (unsigned long long)pci_rsc->phys_addr,
1565                         (unsigned long long)pci_rsc->len,
1566                         (unsigned long long)pci_rsc->addr);
1567
1568         /* Get link state */
1569         eth_link_update(dev, 0);
1570
1571         /* Allocate space for one mac address */
1572         data->mac_addrs = rte_zmalloc(data->name, sizeof(struct ether_addr),
1573                         RTE_CACHE_LINE_SIZE);
1574         if (data->mac_addrs == NULL) {
1575                 RTE_LOG(ERR, PMD, "Could not alloc space for MAC address!\n");
1576                 munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1577                        pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1578                 return -EINVAL;
1579         }
1580
1581         ether_addr_copy(&eth_addr, data->mac_addrs);
1582
1583         /* At initial state COMBO card is in promiscuous mode so disable it */
1584         eth_promiscuous_disable(dev);
1585
1586         RTE_LOG(INFO, PMD, "szedata2 device ("
1587                         PCI_PRI_FMT ") successfully initialized\n",
1588                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1589                         pci_addr->function);
1590
1591         return 0;
1592 }
1593
1594 static int
1595 rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev)
1596 {
1597         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1598         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1599
1600         rte_free(dev->data->mac_addrs);
1601         dev->data->mac_addrs = NULL;
1602         munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1603                pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1604
1605         RTE_LOG(INFO, PMD, "szedata2 device ("
1606                         PCI_PRI_FMT ") successfully uninitialized\n",
1607                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1608                         pci_addr->function);
1609
1610         return 0;
1611 }
1612
1613 static const struct rte_pci_id rte_szedata2_pci_id_table[] = {
1614         {
1615                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1616                                 PCI_DEVICE_ID_NETCOPE_COMBO80G)
1617         },
1618         {
1619                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1620                                 PCI_DEVICE_ID_NETCOPE_COMBO100G)
1621         },
1622         {
1623                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1624                                 PCI_DEVICE_ID_NETCOPE_COMBO100G2)
1625         },
1626         {
1627                 .vendor_id = 0,
1628         }
1629 };
1630
1631 static int szedata2_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1632         struct rte_pci_device *pci_dev)
1633 {
1634         return rte_eth_dev_pci_generic_probe(pci_dev,
1635                 sizeof(struct pmd_internals), rte_szedata2_eth_dev_init);
1636 }
1637
1638 static int szedata2_eth_pci_remove(struct rte_pci_device *pci_dev)
1639 {
1640         return rte_eth_dev_pci_generic_remove(pci_dev,
1641                 rte_szedata2_eth_dev_uninit);
1642 }
1643
1644 static struct rte_pci_driver szedata2_eth_driver = {
1645         .id_table = rte_szedata2_pci_id_table,
1646         .probe = szedata2_eth_pci_probe,
1647         .remove = szedata2_eth_pci_remove,
1648 };
1649
1650 RTE_PMD_REGISTER_PCI(RTE_SZEDATA2_DRIVER_NAME, szedata2_eth_driver);
1651 RTE_PMD_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table);
1652 RTE_PMD_REGISTER_KMOD_DEP(RTE_SZEDATA2_DRIVER_NAME,
1653         "* combo6core & combov3 & szedata2 & szedata2_cv3");