drivers/net/szedata2/rte_eth_szedata2.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) 2015 - 2016 CESNET
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of CESNET nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <unistd.h>
  36 #include <stdbool.h>
  37 #include <err.h>
  38 #include <sys/types.h>
  39 #include <dirent.h>
  40 #include <sys/stat.h>
  41 #include <fcntl.h>
  42 #include <sys/mman.h>
  43
  44 #include <libsze2.h>
  45
  46 #include <rte_mbuf.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_ethdev_pci.h>
  49 #include <rte_malloc.h>
  50 #include <rte_memcpy.h>
  51 #include <rte_kvargs.h>
  52 #include <rte_dev.h>
  53 #include <rte_atomic.h>
  54
  55 #include "rte_eth_szedata2.h"
  56 #include "szedata2_iobuf.h"
  57
  58 #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
  59 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
  60 #define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
  61
  62 /**
  63  * size of szedata2_packet header with alignment
  64  */
  65 #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
  66
  67 #define RTE_SZEDATA2_DRIVER_NAME net_szedata2
  68
  69 #define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u"
  70
  71 struct szedata2_rx_queue {
  72         struct szedata *sze;
  73         uint8_t rx_channel;
  74         uint16_t in_port;
  75         struct rte_mempool *mb_pool;
  76         volatile uint64_t rx_pkts;
  77         volatile uint64_t rx_bytes;
  78         volatile uint64_t err_pkts;
  79 };
  80
  81 struct szedata2_tx_queue {
  82         struct szedata *sze;
  83         uint8_t tx_channel;
  84         volatile uint64_t tx_pkts;
  85         volatile uint64_t tx_bytes;
  86         volatile uint64_t err_pkts;
  87 };
  88
  89 struct pmd_internals {
  90         struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
  91         struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
  92         uint16_t max_rx_queues;
  93         uint16_t max_tx_queues;
  94         char sze_dev[PATH_MAX];
  95         struct rte_mem_resource *pci_rsc;
  96 };
  97
  98 static struct ether_addr eth_addr = {
  99         .addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
 100 };
 101
 102 static uint16_t
 103 eth_szedata2_rx(void *queue,
 104                 struct rte_mbuf **bufs,
 105                 uint16_t nb_pkts)
 106 {
 107         unsigned int i;
 108         struct rte_mbuf *mbuf;
 109         struct szedata2_rx_queue *sze_q = queue;
 110         struct rte_pktmbuf_pool_private *mbp_priv;
 111         uint16_t num_rx = 0;
 112         uint16_t buf_size;
 113         uint16_t sg_size;
 114         uint16_t hw_size;
 115         uint16_t packet_size;
 116         uint64_t num_bytes = 0;
 117         struct szedata *sze = sze_q->sze;
 118         uint8_t *header_ptr = NULL; /* header of packet */
 119         uint8_t *packet_ptr1 = NULL;
 120         uint8_t *packet_ptr2 = NULL;
 121         uint16_t packet_len1 = 0;
 122         uint16_t packet_len2 = 0;
 123         uint16_t hw_data_align;
 124
 125         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 126                 return 0;
 127
 128         /*
 129          * Reads the given number of packets from szedata2 channel given
 130          * by queue and copies the packet data into a newly allocated mbuf
 131          * to return.
 132          */
 133         for (i = 0; i < nb_pkts; i++) {
 134                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 135
 136                 if (unlikely(mbuf == NULL))
 137                         break;
 138
 139                 /* get the next sze packet */
 140                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 141                                 sze->ct_rx_lck->next == NULL) {
 142                         /* unlock old data */
 143                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 144                         sze->ct_rx_lck_orig = NULL;
 145                         sze->ct_rx_lck = NULL;
 146                 }
 147
 148                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 149                         /* nothing to read, lock new data */
 150                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 151                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 152
 153                         if (sze->ct_rx_lck == NULL) {
 154                                 /* nothing to lock */
 155                                 rte_pktmbuf_free(mbuf);
 156                                 break;
 157                         }
 158
 159                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 160                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 161
 162                         if (!sze->ct_rx_rem_bytes) {
 163                                 rte_pktmbuf_free(mbuf);
 164                                 break;
 165                         }
 166                 }
 167
 168                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 169                         /*
 170                          * cut in header
 171                          * copy parts of header to merge buffer
 172                          */
 173                         if (sze->ct_rx_lck->next == NULL) {
 174                                 rte_pktmbuf_free(mbuf);
 175                                 break;
 176                         }
 177
 178                         /* copy first part of header */
 179                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 180                                         sze->ct_rx_rem_bytes);
 181
 182                         /* copy second part of header */
 183                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 184                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 185                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 186                                 sze->ct_rx_cur_ptr,
 187                                 RTE_SZE2_PACKET_HEADER_SIZE -
 188                                 sze->ct_rx_rem_bytes);
 189
 190                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 191                                 sze->ct_rx_rem_bytes;
 192                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 193                                 RTE_SZE2_PACKET_HEADER_SIZE +
 194                                 sze->ct_rx_rem_bytes;
 195
 196                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 197                 } else {
 198                         /* not cut */
 199                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 200                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 201                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 202                 }
 203
 204                 sg_size = le16toh(*((uint16_t *)header_ptr));
 205                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 206                 packet_size = sg_size -
 207                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 208
 209
 210                 /* checks if packet all right */
 211                 if (!sg_size)
 212                         errx(5, "Zero segsize");
 213
 214                 /* check sg_size and hwsize */
 215                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 216                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 217                                 "hwsize: %d", sg_size, hw_size);
 218                 }
 219
 220                 hw_data_align =
 221                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
 222                         RTE_SZE2_PACKET_HEADER_SIZE;
 223
 224                 if (sze->ct_rx_rem_bytes >=
 225                                 (uint16_t)(sg_size -
 226                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 227                         /* no cut */
 228                         /* one packet ready - go to another */
 229                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 230                         packet_len1 = packet_size;
 231                         packet_ptr2 = NULL;
 232                         packet_len2 = 0;
 233
 234                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 235                                 RTE_SZE2_PACKET_HEADER_SIZE;
 236                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 237                                 RTE_SZE2_PACKET_HEADER_SIZE;
 238                 } else {
 239                         /* cut in data */
 240                         if (sze->ct_rx_lck->next == NULL) {
 241                                 errx(6, "Need \"next\" lock, "
 242                                         "but it is missing: %u",
 243                                         sze->ct_rx_rem_bytes);
 244                         }
 245
 246                         /* skip hw data */
 247                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 248                                 uint16_t rem_size = hw_data_align -
 249                                         sze->ct_rx_rem_bytes;
 250
 251                                 /* MOVE to next lock */
 252                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 253                                 sze->ct_rx_cur_ptr =
 254                                         (void *)(((uint8_t *)
 255                                         (sze->ct_rx_lck->start)) + rem_size);
 256
 257                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 258                                 packet_len1 = packet_size;
 259                                 packet_ptr2 = NULL;
 260                                 packet_len2 = 0;
 261
 262                                 sze->ct_rx_cur_ptr +=
 263                                         RTE_SZE2_ALIGN8(packet_size);
 264                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 265                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 266                         } else {
 267                                 /* get pointer and length from first part */
 268                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 269                                         hw_data_align;
 270                                 packet_len1 = sze->ct_rx_rem_bytes -
 271                                         hw_data_align;
 272
 273                                 /* MOVE to next lock */
 274                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 275                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 276
 277                                 /* get pointer and length from second part */
 278                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 279                                 packet_len2 = packet_size - packet_len1;
 280
 281                                 sze->ct_rx_cur_ptr +=
 282                                         RTE_SZE2_ALIGN8(packet_size) -
 283                                         packet_len1;
 284                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 285                                         (RTE_SZE2_ALIGN8(packet_size) -
 286                                          packet_len1);
 287                         }
 288                 }
 289
 290                 if (unlikely(packet_ptr1 == NULL)) {
 291                         rte_pktmbuf_free(mbuf);
 292                         break;
 293                 }
 294
 295                 /* get the space available for data in the mbuf */
 296                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 297                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 298                                 RTE_PKTMBUF_HEADROOM);
 299
 300                 if (packet_size <= buf_size) {
 301                         /* sze packet will fit in one mbuf, go ahead and copy */
 302                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 303                                         packet_ptr1, packet_len1);
 304                         if (packet_ptr2 != NULL) {
 305                                 rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
 306                                         uint8_t *) + packet_len1),
 307                                         packet_ptr2, packet_len2);
 308                         }
 309                         mbuf->data_len = (uint16_t)packet_size;
 310
 311                         mbuf->pkt_len = packet_size;
 312                         mbuf->port = sze_q->in_port;
 313                         bufs[num_rx] = mbuf;
 314                         num_rx++;
 315                         num_bytes += packet_size;
 316                 } else {
 317                         /*
 318                          * sze packet will not fit in one mbuf,
 319                          * scattered mode is not enabled, drop packet
 320                          */
 321                         RTE_LOG(ERR, PMD,
 322                                 "SZE segment %d bytes will not fit in one mbuf "
 323                                 "(%d bytes), scattered mode is not enabled, "
 324                                 "drop packet!!\n",
 325                                 packet_size, buf_size);
 326                         rte_pktmbuf_free(mbuf);
 327                 }
 328         }
 329
 330         sze_q->rx_pkts += num_rx;
 331         sze_q->rx_bytes += num_bytes;
 332         return num_rx;
 333 }
 334
 335 static uint16_t
 336 eth_szedata2_rx_scattered(void *queue,
 337                 struct rte_mbuf **bufs,
 338                 uint16_t nb_pkts)
 339 {
 340         unsigned int i;
 341         struct rte_mbuf *mbuf;
 342         struct szedata2_rx_queue *sze_q = queue;
 343         struct rte_pktmbuf_pool_private *mbp_priv;
 344         uint16_t num_rx = 0;
 345         uint16_t buf_size;
 346         uint16_t sg_size;
 347         uint16_t hw_size;
 348         uint16_t packet_size;
 349         uint64_t num_bytes = 0;
 350         struct szedata *sze = sze_q->sze;
 351         uint8_t *header_ptr = NULL; /* header of packet */
 352         uint8_t *packet_ptr1 = NULL;
 353         uint8_t *packet_ptr2 = NULL;
 354         uint16_t packet_len1 = 0;
 355         uint16_t packet_len2 = 0;
 356         uint16_t hw_data_align;
 357
 358         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 359                 return 0;
 360
 361         /*
 362          * Reads the given number of packets from szedata2 channel given
 363          * by queue and copies the packet data into a newly allocated mbuf
 364          * to return.
 365          */
 366         for (i = 0; i < nb_pkts; i++) {
 367                 const struct szedata_lock *ct_rx_lck_backup;
 368                 unsigned int ct_rx_rem_bytes_backup;
 369                 unsigned char *ct_rx_cur_ptr_backup;
 370
 371                 /* get the next sze packet */
 372                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 373                                 sze->ct_rx_lck->next == NULL) {
 374                         /* unlock old data */
 375                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 376                         sze->ct_rx_lck_orig = NULL;
 377                         sze->ct_rx_lck = NULL;
 378                 }
 379
 380                 /*
 381                  * Store items from sze structure which can be changed
 382                  * before mbuf allocating. Use these items in case of mbuf
 383                  * allocating failure.
 384                  */
 385                 ct_rx_lck_backup = sze->ct_rx_lck;
 386                 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 387                 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 388
 389                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 390                         /* nothing to read, lock new data */
 391                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 392                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 393
 394                         /*
 395                          * Backup items from sze structure must be updated
 396                          * after locking to contain pointers to new locks.
 397                          */
 398                         ct_rx_lck_backup = sze->ct_rx_lck;
 399                         ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 400                         ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 401
 402                         if (sze->ct_rx_lck == NULL)
 403                                 /* nothing to lock */
 404                                 break;
 405
 406                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 407                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 408
 409                         if (!sze->ct_rx_rem_bytes)
 410                                 break;
 411                 }
 412
 413                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 414                         /*
 415                          * cut in header - copy parts of header to merge buffer
 416                          */
 417                         if (sze->ct_rx_lck->next == NULL)
 418                                 break;
 419
 420                         /* copy first part of header */
 421                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 422                                         sze->ct_rx_rem_bytes);
 423
 424                         /* copy second part of header */
 425                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 426                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 427                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 428                                 sze->ct_rx_cur_ptr,
 429                                 RTE_SZE2_PACKET_HEADER_SIZE -
 430                                 sze->ct_rx_rem_bytes);
 431
 432                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 433                                 sze->ct_rx_rem_bytes;
 434                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 435                                 RTE_SZE2_PACKET_HEADER_SIZE +
 436                                 sze->ct_rx_rem_bytes;
 437
 438                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 439                 } else {
 440                         /* not cut */
 441                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 442                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 443                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 444                 }
 445
 446                 sg_size = le16toh(*((uint16_t *)header_ptr));
 447                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 448                 packet_size = sg_size -
 449                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 450
 451
 452                 /* checks if packet all right */
 453                 if (!sg_size)
 454                         errx(5, "Zero segsize");
 455
 456                 /* check sg_size and hwsize */
 457                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 458                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 459                                         "hwsize: %d", sg_size, hw_size);
 460                 }
 461
 462                 hw_data_align =
 463                         RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
 464                         hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
 465
 466                 if (sze->ct_rx_rem_bytes >=
 467                                 (uint16_t)(sg_size -
 468                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 469                         /* no cut */
 470                         /* one packet ready - go to another */
 471                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 472                         packet_len1 = packet_size;
 473                         packet_ptr2 = NULL;
 474                         packet_len2 = 0;
 475
 476                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 477                                 RTE_SZE2_PACKET_HEADER_SIZE;
 478                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 479                                 RTE_SZE2_PACKET_HEADER_SIZE;
 480                 } else {
 481                         /* cut in data */
 482                         if (sze->ct_rx_lck->next == NULL) {
 483                                 errx(6, "Need \"next\" lock, but it is "
 484                                         "missing: %u", sze->ct_rx_rem_bytes);
 485                         }
 486
 487                         /* skip hw data */
 488                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 489                                 uint16_t rem_size = hw_data_align -
 490                                         sze->ct_rx_rem_bytes;
 491
 492                                 /* MOVE to next lock */
 493                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 494                                 sze->ct_rx_cur_ptr =
 495                                         (void *)(((uint8_t *)
 496                                         (sze->ct_rx_lck->start)) + rem_size);
 497
 498                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 499                                 packet_len1 = packet_size;
 500                                 packet_ptr2 = NULL;
 501                                 packet_len2 = 0;
 502
 503                                 sze->ct_rx_cur_ptr +=
 504                                         RTE_SZE2_ALIGN8(packet_size);
 505                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 506                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 507                         } else {
 508                                 /* get pointer and length from first part */
 509                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 510                                         hw_data_align;
 511                                 packet_len1 = sze->ct_rx_rem_bytes -
 512                                         hw_data_align;
 513
 514                                 /* MOVE to next lock */
 515                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 516                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 517
 518                                 /* get pointer and length from second part */
 519                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 520                                 packet_len2 = packet_size - packet_len1;
 521
 522                                 sze->ct_rx_cur_ptr +=
 523                                         RTE_SZE2_ALIGN8(packet_size) -
 524                                         packet_len1;
 525                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 526                                         (RTE_SZE2_ALIGN8(packet_size) -
 527                                          packet_len1);
 528                         }
 529                 }
 530
 531                 if (unlikely(packet_ptr1 == NULL))
 532                         break;
 533
 534                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 535
 536                 if (unlikely(mbuf == NULL)) {
 537                         /*
 538                          * Restore items from sze structure to state after
 539                          * unlocking (eventually locking).
 540                          */
 541                         sze->ct_rx_lck = ct_rx_lck_backup;
 542                         sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
 543                         sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
 544                         break;
 545                 }
 546
 547                 /* get the space available for data in the mbuf */
 548                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 549                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 550                                 RTE_PKTMBUF_HEADROOM);
 551
 552                 if (packet_size <= buf_size) {
 553                         /* sze packet will fit in one mbuf, go ahead and copy */
 554                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 555                                         packet_ptr1, packet_len1);
 556                         if (packet_ptr2 != NULL) {
 557                                 rte_memcpy((void *)
 558                                         (rte_pktmbuf_mtod(mbuf, uint8_t *) +
 559                                         packet_len1), packet_ptr2, packet_len2);
 560                         }
 561                         mbuf->data_len = (uint16_t)packet_size;
 562                 } else {
 563                         /*
 564                          * sze packet will not fit in one mbuf,
 565                          * scatter packet into more mbufs
 566                          */
 567                         struct rte_mbuf *m = mbuf;
 568                         uint16_t len = rte_pktmbuf_tailroom(mbuf);
 569
 570                         /* copy first part of packet */
 571                         /* fill first mbuf */
 572                         rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
 573                                 len);
 574                         packet_len1 -= len;
 575                         packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 576
 577                         while (packet_len1 > 0) {
 578                                 /* fill new mbufs */
 579                                 m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
 580
 581                                 if (unlikely(m->next == NULL)) {
 582                                         rte_pktmbuf_free(mbuf);
 583                                         /*
 584                                          * Restore items from sze structure
 585                                          * to state after unlocking (eventually
 586                                          * locking).
 587                                          */
 588                                         sze->ct_rx_lck = ct_rx_lck_backup;
 589                                         sze->ct_rx_rem_bytes =
 590                                                 ct_rx_rem_bytes_backup;
 591                                         sze->ct_rx_cur_ptr =
 592                                                 ct_rx_cur_ptr_backup;
 593                                         goto finish;
 594                                 }
 595
 596                                 m = m->next;
 597
 598                                 len = RTE_MIN(rte_pktmbuf_tailroom(m),
 599                                         packet_len1);
 600                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 601                                         packet_ptr1, len);
 602
 603                                 (mbuf->nb_segs)++;
 604                                 packet_len1 -= len;
 605                                 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 606                         }
 607
 608                         if (packet_ptr2 != NULL) {
 609                                 /* copy second part of packet, if exists */
 610                                 /* fill the rest of currently last mbuf */
 611                                 len = rte_pktmbuf_tailroom(m);
 612                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 613                                         packet_ptr2, len);
 614                                 packet_len2 -= len;
 615                                 packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
 616
 617                                 while (packet_len2 > 0) {
 618                                         /* fill new mbufs */
 619                                         m->next = rte_pktmbuf_alloc(
 620                                                         sze_q->mb_pool);
 621
 622                                         if (unlikely(m->next == NULL)) {
 623                                                 rte_pktmbuf_free(mbuf);
 624                                                 /*
 625                                                  * Restore items from sze
 626                                                  * structure to state after
 627                                                  * unlocking (eventually
 628                                                  * locking).
 629                                                  */
 630                                                 sze->ct_rx_lck =
 631                                                         ct_rx_lck_backup;
 632                                                 sze->ct_rx_rem_bytes =
 633                                                         ct_rx_rem_bytes_backup;
 634                                                 sze->ct_rx_cur_ptr =
 635                                                         ct_rx_cur_ptr_backup;
 636                                                 goto finish;
 637                                         }
 638
 639                                         m = m->next;
 640
 641                                         len = RTE_MIN(rte_pktmbuf_tailroom(m),
 642                                                 packet_len2);
 643                                         rte_memcpy(
 644                                                 rte_pktmbuf_append(mbuf, len),
 645                                                 packet_ptr2, len);
 646
 647                                         (mbuf->nb_segs)++;
 648                                         packet_len2 -= len;
 649                                         packet_ptr2 = ((uint8_t *)packet_ptr2) +
 650                                                 len;
 651                                 }
 652                         }
 653                 }
 654                 mbuf->pkt_len = packet_size;
 655                 mbuf->port = sze_q->in_port;
 656                 bufs[num_rx] = mbuf;
 657                 num_rx++;
 658                 num_bytes += packet_size;
 659         }
 660
 661 finish:
 662         sze_q->rx_pkts += num_rx;
 663         sze_q->rx_bytes += num_bytes;
 664         return num_rx;
 665 }
 666
 667 static uint16_t
 668 eth_szedata2_tx(void *queue,
 669                 struct rte_mbuf **bufs,
 670                 uint16_t nb_pkts)
 671 {
 672         struct rte_mbuf *mbuf;
 673         struct szedata2_tx_queue *sze_q = queue;
 674         uint16_t num_tx = 0;
 675         uint64_t num_bytes = 0;
 676
 677         const struct szedata_lock *lck;
 678         uint32_t lock_size;
 679         uint32_t lock_size2;
 680         void *dst;
 681         uint32_t pkt_len;
 682         uint32_t hwpkt_len;
 683         uint32_t unlock_size;
 684         uint32_t rem_len;
 685         uint16_t mbuf_segs;
 686         uint16_t pkt_left = nb_pkts;
 687
 688         if (sze_q->sze == NULL || nb_pkts == 0)
 689                 return 0;
 690
 691         while (pkt_left > 0) {
 692                 unlock_size = 0;
 693                 lck = szedata_tx_lock_data(sze_q->sze,
 694                         RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
 695                         sze_q->tx_channel);
 696                 if (lck == NULL)
 697                         continue;
 698
 699                 dst = lck->start;
 700                 lock_size = lck->len;
 701                 lock_size2 = lck->next ? lck->next->len : 0;
 702
 703 next_packet:
 704                 mbuf = bufs[nb_pkts - pkt_left];
 705
 706                 pkt_len = mbuf->pkt_len;
 707                 mbuf_segs = mbuf->nb_segs;
 708
 709                 hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 710                         RTE_SZE2_ALIGN8(pkt_len);
 711
 712                 if (lock_size + lock_size2 < hwpkt_len) {
 713                         szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 714                         continue;
 715                 }
 716
 717                 num_bytes += pkt_len;
 718
 719                 if (lock_size > hwpkt_len) {
 720                         void *tmp_dst;
 721
 722                         rem_len = 0;
 723
 724                         /* write packet length at first 2 bytes in 8B header */
 725                         *((uint16_t *)dst) = htole16(
 726                                         RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 727                                         pkt_len);
 728                         *(((uint16_t *)dst) + 1) = htole16(0);
 729
 730                         /* copy packet from mbuf */
 731                         tmp_dst = ((uint8_t *)(dst)) +
 732                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 733                         if (mbuf_segs == 1) {
 734                                 /*
 735                                  * non-scattered packet,
 736                                  * transmit from one mbuf
 737                                  */
 738                                 rte_memcpy(tmp_dst,
 739                                         rte_pktmbuf_mtod(mbuf, const void *),
 740                                         pkt_len);
 741                         } else {
 742                                 /* scattered packet, transmit from more mbufs */
 743                                 struct rte_mbuf *m = mbuf;
 744                                 while (m) {
 745                                         rte_memcpy(tmp_dst,
 746                                                 rte_pktmbuf_mtod(m,
 747                                                 const void *),
 748                                                 m->data_len);
 749                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 750                                                 m->data_len;
 751                                         m = m->next;
 752                                 }
 753                         }
 754
 755
 756                         dst = ((uint8_t *)dst) + hwpkt_len;
 757                         unlock_size += hwpkt_len;
 758                         lock_size -= hwpkt_len;
 759
 760                         rte_pktmbuf_free(mbuf);
 761                         num_tx++;
 762                         pkt_left--;
 763                         if (pkt_left == 0) {
 764                                 szedata_tx_unlock_data(sze_q->sze, lck,
 765                                         unlock_size);
 766                                 break;
 767                         }
 768                         goto next_packet;
 769                 } else if (lock_size + lock_size2 >= hwpkt_len) {
 770                         void *tmp_dst;
 771                         uint16_t write_len;
 772
 773                         /* write packet length at first 2 bytes in 8B header */
 774                         *((uint16_t *)dst) =
 775                                 htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 776                                         pkt_len);
 777                         *(((uint16_t *)dst) + 1) = htole16(0);
 778
 779                         /*
 780                          * If the raw packet (pkt_len) is smaller than lock_size
 781                          * get the correct length for memcpy
 782                          */
 783                         write_len =
 784                                 pkt_len < lock_size -
 785                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
 786                                 pkt_len :
 787                                 lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 788
 789                         rem_len = hwpkt_len - lock_size;
 790
 791                         tmp_dst = ((uint8_t *)(dst)) +
 792                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 793                         if (mbuf_segs == 1) {
 794                                 /*
 795                                  * non-scattered packet,
 796                                  * transmit from one mbuf
 797                                  */
 798                                 /* copy part of packet to first area */
 799                                 rte_memcpy(tmp_dst,
 800                                         rte_pktmbuf_mtod(mbuf, const void *),
 801                                         write_len);
 802
 803                                 if (lck->next)
 804                                         dst = lck->next->start;
 805
 806                                 /* copy part of packet to second area */
 807                                 rte_memcpy(dst,
 808                                         (const void *)(rte_pktmbuf_mtod(mbuf,
 809                                                         const uint8_t *) +
 810                                         write_len), pkt_len - write_len);
 811                         } else {
 812                                 /* scattered packet, transmit from more mbufs */
 813                                 struct rte_mbuf *m = mbuf;
 814                                 uint16_t written = 0;
 815                                 uint16_t to_write = 0;
 816                                 bool new_mbuf = true;
 817                                 uint16_t write_off = 0;
 818
 819                                 /* copy part of packet to first area */
 820                                 while (m && written < write_len) {
 821                                         to_write = RTE_MIN(m->data_len,
 822                                                         write_len - written);
 823                                         rte_memcpy(tmp_dst,
 824                                                 rte_pktmbuf_mtod(m,
 825                                                         const void *),
 826                                                 to_write);
 827
 828                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 829                                                 to_write;
 830                                         if (m->data_len <= write_len -
 831                                                         written) {
 832                                                 m = m->next;
 833                                                 new_mbuf = true;
 834                                         } else {
 835                                                 new_mbuf = false;
 836                                         }
 837                                         written += to_write;
 838                                 }
 839
 840                                 if (lck->next)
 841                                         dst = lck->next->start;
 842
 843                                 tmp_dst = dst;
 844                                 written = 0;
 845                                 write_off = new_mbuf ? 0 : to_write;
 846
 847                                 /* copy part of packet to second area */
 848                                 while (m && written < pkt_len - write_len) {
 849                                         rte_memcpy(tmp_dst, (const void *)
 850                                                 (rte_pktmbuf_mtod(m,
 851                                                 uint8_t *) + write_off),
 852                                                 m->data_len - write_off);
 853
 854                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 855                                                 (m->data_len - write_off);
 856                                         written += m->data_len - write_off;
 857                                         m = m->next;
 858                                         write_off = 0;
 859                                 }
 860                         }
 861
 862                         dst = ((uint8_t *)dst) + rem_len;
 863                         unlock_size += hwpkt_len;
 864                         lock_size = lock_size2 - rem_len;
 865                         lock_size2 = 0;
 866
 867                         rte_pktmbuf_free(mbuf);
 868                         num_tx++;
 869                 }
 870
 871                 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 872                 pkt_left--;
 873         }
 874
 875         sze_q->tx_pkts += num_tx;
 876         sze_q->err_pkts += nb_pkts - num_tx;
 877         sze_q->tx_bytes += num_bytes;
 878         return num_tx;
 879 }
 880
 881 static int
 882 eth_rx_queue_start(struct rte_eth_dev *dev, uint16_t rxq_id)
 883 {
 884         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 885         int ret;
 886         struct pmd_internals *internals = (struct pmd_internals *)
 887                 dev->data->dev_private;
 888
 889         if (rxq->sze == NULL) {
 890                 uint32_t rx = 1 << rxq->rx_channel;
 891                 uint32_t tx = 0;
 892                 rxq->sze = szedata_open(internals->sze_dev);
 893                 if (rxq->sze == NULL)
 894                         return -EINVAL;
 895                 ret = szedata_subscribe3(rxq->sze, &rx, &tx);
 896                 if (ret != 0 || rx == 0)
 897                         goto err;
 898         }
 899
 900         ret = szedata_start(rxq->sze);
 901         if (ret != 0)
 902                 goto err;
 903         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 904         return 0;
 905
 906 err:
 907         szedata_close(rxq->sze);
 908         rxq->sze = NULL;
 909         return -EINVAL;
 910 }
 911
 912 static int
 913 eth_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rxq_id)
 914 {
 915         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 916
 917         if (rxq->sze != NULL) {
 918                 szedata_close(rxq->sze);
 919                 rxq->sze = NULL;
 920         }
 921
 922         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 923         return 0;
 924 }
 925
 926 static int
 927 eth_tx_queue_start(struct rte_eth_dev *dev, uint16_t txq_id)
 928 {
 929         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 930         int ret;
 931         struct pmd_internals *internals = (struct pmd_internals *)
 932                 dev->data->dev_private;
 933
 934         if (txq->sze == NULL) {
 935                 uint32_t rx = 0;
 936                 uint32_t tx = 1 << txq->tx_channel;
 937                 txq->sze = szedata_open(internals->sze_dev);
 938                 if (txq->sze == NULL)
 939                         return -EINVAL;
 940                 ret = szedata_subscribe3(txq->sze, &rx, &tx);
 941                 if (ret != 0 || tx == 0)
 942                         goto err;
 943         }
 944
 945         ret = szedata_start(txq->sze);
 946         if (ret != 0)
 947                 goto err;
 948         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 949         return 0;
 950
 951 err:
 952         szedata_close(txq->sze);
 953         txq->sze = NULL;
 954         return -EINVAL;
 955 }
 956
 957 static int
 958 eth_tx_queue_stop(struct rte_eth_dev *dev, uint16_t txq_id)
 959 {
 960         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 961
 962         if (txq->sze != NULL) {
 963                 szedata_close(txq->sze);
 964                 txq->sze = NULL;
 965         }
 966
 967         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 968         return 0;
 969 }
 970
 971 static int
 972 eth_dev_start(struct rte_eth_dev *dev)
 973 {
 974         int ret;
 975         uint16_t i;
 976         uint16_t nb_rx = dev->data->nb_rx_queues;
 977         uint16_t nb_tx = dev->data->nb_tx_queues;
 978
 979         for (i = 0; i < nb_rx; i++) {
 980                 ret = eth_rx_queue_start(dev, i);
 981                 if (ret != 0)
 982                         goto err_rx;
 983         }
 984
 985         for (i = 0; i < nb_tx; i++) {
 986                 ret = eth_tx_queue_start(dev, i);
 987                 if (ret != 0)
 988                         goto err_tx;
 989         }
 990
 991         return 0;
 992
 993 err_tx:
 994         for (i = 0; i < nb_tx; i++)
 995                 eth_tx_queue_stop(dev, i);
 996 err_rx:
 997         for (i = 0; i < nb_rx; i++)
 998                 eth_rx_queue_stop(dev, i);
 999         return ret;
1000 }
1001
1002 static void
1003 eth_dev_stop(struct rte_eth_dev *dev)
1004 {
1005         uint16_t i;
1006         uint16_t nb_rx = dev->data->nb_rx_queues;
1007         uint16_t nb_tx = dev->data->nb_tx_queues;
1008
1009         for (i = 0; i < nb_tx; i++)
1010                 eth_tx_queue_stop(dev, i);
1011
1012         for (i = 0; i < nb_rx; i++)
1013                 eth_rx_queue_stop(dev, i);
1014 }
1015
1016 static int
1017 eth_dev_configure(struct rte_eth_dev *dev)
1018 {
1019         struct rte_eth_dev_data *data = dev->data;
1020         if (data->dev_conf.rxmode.enable_scatter == 1) {
1021                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1022                 data->scattered_rx = 1;
1023         } else {
1024                 dev->rx_pkt_burst = eth_szedata2_rx;
1025                 data->scattered_rx = 0;
1026         }
1027         return 0;
1028 }
1029
1030 static void
1031 eth_dev_info(struct rte_eth_dev *dev,
1032                 struct rte_eth_dev_info *dev_info)
1033 {
1034         struct pmd_internals *internals = dev->data->dev_private;
1035         dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1036         dev_info->if_index = 0;
1037         dev_info->max_mac_addrs = 1;
1038         dev_info->max_rx_pktlen = (uint32_t)-1;
1039         dev_info->max_rx_queues = internals->max_rx_queues;
1040         dev_info->max_tx_queues = internals->max_tx_queues;
1041         dev_info->min_rx_bufsize = 0;
1042         dev_info->speed_capa = ETH_LINK_SPEED_100G;
1043 }
1044
1045 static int
1046 eth_stats_get(struct rte_eth_dev *dev,
1047                 struct rte_eth_stats *stats)
1048 {
1049         uint16_t i;
1050         uint16_t nb_rx = dev->data->nb_rx_queues;
1051         uint16_t nb_tx = dev->data->nb_tx_queues;
1052         uint64_t rx_total = 0;
1053         uint64_t tx_total = 0;
1054         uint64_t tx_err_total = 0;
1055         uint64_t rx_total_bytes = 0;
1056         uint64_t tx_total_bytes = 0;
1057         const struct pmd_internals *internals = dev->data->dev_private;
1058
1059         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_rx; i++) {
1060                 stats->q_ipackets[i] = internals->rx_queue[i].rx_pkts;
1061                 stats->q_ibytes[i] = internals->rx_queue[i].rx_bytes;
1062                 rx_total += stats->q_ipackets[i];
1063                 rx_total_bytes += stats->q_ibytes[i];
1064         }
1065
1066         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_tx; i++) {
1067                 stats->q_opackets[i] = internals->tx_queue[i].tx_pkts;
1068                 stats->q_obytes[i] = internals->tx_queue[i].tx_bytes;
1069                 stats->q_errors[i] = internals->tx_queue[i].err_pkts;
1070                 tx_total += stats->q_opackets[i];
1071                 tx_total_bytes += stats->q_obytes[i];
1072                 tx_err_total += stats->q_errors[i];
1073         }
1074
1075         stats->ipackets = rx_total;
1076         stats->opackets = tx_total;
1077         stats->ibytes = rx_total_bytes;
1078         stats->obytes = tx_total_bytes;
1079         stats->oerrors = tx_err_total;
1080
1081         return 0;
1082 }
1083
1084 static void
1085 eth_stats_reset(struct rte_eth_dev *dev)
1086 {
1087         uint16_t i;
1088         uint16_t nb_rx = dev->data->nb_rx_queues;
1089         uint16_t nb_tx = dev->data->nb_tx_queues;
1090         struct pmd_internals *internals = dev->data->dev_private;
1091
1092         for (i = 0; i < nb_rx; i++) {
1093                 internals->rx_queue[i].rx_pkts = 0;
1094                 internals->rx_queue[i].rx_bytes = 0;
1095                 internals->rx_queue[i].err_pkts = 0;
1096         }
1097         for (i = 0; i < nb_tx; i++) {
1098                 internals->tx_queue[i].tx_pkts = 0;
1099                 internals->tx_queue[i].tx_bytes = 0;
1100                 internals->tx_queue[i].err_pkts = 0;
1101         }
1102 }
1103
1104 static void
1105 eth_rx_queue_release(void *q)
1106 {
1107         struct szedata2_rx_queue *rxq = (struct szedata2_rx_queue *)q;
1108         if (rxq->sze != NULL) {
1109                 szedata_close(rxq->sze);
1110                 rxq->sze = NULL;
1111         }
1112 }
1113
1114 static void
1115 eth_tx_queue_release(void *q)
1116 {
1117         struct szedata2_tx_queue *txq = (struct szedata2_tx_queue *)q;
1118         if (txq->sze != NULL) {
1119                 szedata_close(txq->sze);
1120                 txq->sze = NULL;
1121         }
1122 }
1123
1124 static void
1125 eth_dev_close(struct rte_eth_dev *dev)
1126 {
1127         uint16_t i;
1128         uint16_t nb_rx = dev->data->nb_rx_queues;
1129         uint16_t nb_tx = dev->data->nb_tx_queues;
1130
1131         eth_dev_stop(dev);
1132
1133         for (i = 0; i < nb_rx; i++) {
1134                 eth_rx_queue_release(dev->data->rx_queues[i]);
1135                 dev->data->rx_queues[i] = NULL;
1136         }
1137         dev->data->nb_rx_queues = 0;
1138         for (i = 0; i < nb_tx; i++) {
1139                 eth_tx_queue_release(dev->data->tx_queues[i]);
1140                 dev->data->tx_queues[i] = NULL;
1141         }
1142         dev->data->nb_tx_queues = 0;
1143 }
1144
1145 /**
1146  * Function takes value from first IBUF status register.
1147  * Values in IBUF and OBUF should be same.
1148  *
1149  * @param internals
1150  *     Pointer to device private structure.
1151  * @return
1152  *     Link speed constant.
1153  */
1154 static inline enum szedata2_link_speed
1155 get_link_speed(const struct pmd_internals *internals)
1156 {
1157         const volatile struct szedata2_ibuf *ibuf =
1158                 ibuf_ptr_by_index(internals->pci_rsc, 0);
1159         uint32_t speed = (szedata2_read32(&ibuf->ibuf_st) & 0x70) >> 4;
1160         switch (speed) {
1161         case 0x03:
1162                 return SZEDATA2_LINK_SPEED_10G;
1163         case 0x04:
1164                 return SZEDATA2_LINK_SPEED_40G;
1165         case 0x05:
1166                 return SZEDATA2_LINK_SPEED_100G;
1167         default:
1168                 return SZEDATA2_LINK_SPEED_DEFAULT;
1169         }
1170 }
1171
1172 static int
1173 eth_link_update(struct rte_eth_dev *dev,
1174                 int wait_to_complete __rte_unused)
1175 {
1176         struct rte_eth_link link;
1177         struct rte_eth_link *link_ptr = &link;
1178         struct rte_eth_link *dev_link = &dev->data->dev_link;
1179         struct pmd_internals *internals = (struct pmd_internals *)
1180                 dev->data->dev_private;
1181         const volatile struct szedata2_ibuf *ibuf;
1182         uint32_t i;
1183         bool link_is_up = false;
1184
1185         switch (get_link_speed(internals)) {
1186         case SZEDATA2_LINK_SPEED_10G:
1187                 link.link_speed = ETH_SPEED_NUM_10G;
1188                 break;
1189         case SZEDATA2_LINK_SPEED_40G:
1190                 link.link_speed = ETH_SPEED_NUM_40G;
1191                 break;
1192         case SZEDATA2_LINK_SPEED_100G:
1193                 link.link_speed = ETH_SPEED_NUM_100G;
1194                 break;
1195         default:
1196                 link.link_speed = ETH_SPEED_NUM_10G;
1197                 break;
1198         }
1199
1200         /* szedata2 uses only full duplex */
1201         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1202
1203         for (i = 0; i < szedata2_ibuf_count; i++) {
1204                 ibuf = ibuf_ptr_by_index(internals->pci_rsc, i);
1205                 /*
1206                  * Link is considered up if at least one ibuf is enabled
1207                  * and up.
1208                  */
1209                 if (ibuf_is_enabled(ibuf) && ibuf_is_link_up(ibuf)) {
1210                         link_is_up = true;
1211                         break;
1212                 }
1213         }
1214
1215         link.link_status = (link_is_up) ? ETH_LINK_UP : ETH_LINK_DOWN;
1216
1217         link.link_autoneg = ETH_LINK_FIXED;
1218
1219         rte_atomic64_cmpset((uint64_t *)dev_link, *(uint64_t *)dev_link,
1220                         *(uint64_t *)link_ptr);
1221
1222         return 0;
1223 }
1224
1225 static int
1226 eth_dev_set_link_up(struct rte_eth_dev *dev)
1227 {
1228         struct pmd_internals *internals = (struct pmd_internals *)
1229                 dev->data->dev_private;
1230         uint32_t i;
1231
1232         for (i = 0; i < szedata2_ibuf_count; i++)
1233                 ibuf_enable(ibuf_ptr_by_index(internals->pci_rsc, i));
1234         for (i = 0; i < szedata2_obuf_count; i++)
1235                 obuf_enable(obuf_ptr_by_index(internals->pci_rsc, i));
1236         return 0;
1237 }
1238
1239 static int
1240 eth_dev_set_link_down(struct rte_eth_dev *dev)
1241 {
1242         struct pmd_internals *internals = (struct pmd_internals *)
1243                 dev->data->dev_private;
1244         uint32_t i;
1245
1246         for (i = 0; i < szedata2_ibuf_count; i++)
1247                 ibuf_disable(ibuf_ptr_by_index(internals->pci_rsc, i));
1248         for (i = 0; i < szedata2_obuf_count; i++)
1249                 obuf_disable(obuf_ptr_by_index(internals->pci_rsc, i));
1250         return 0;
1251 }
1252
1253 static int
1254 eth_rx_queue_setup(struct rte_eth_dev *dev,
1255                 uint16_t rx_queue_id,
1256                 uint16_t nb_rx_desc __rte_unused,
1257                 unsigned int socket_id __rte_unused,
1258                 const struct rte_eth_rxconf *rx_conf __rte_unused,
1259                 struct rte_mempool *mb_pool)
1260 {
1261         struct pmd_internals *internals = dev->data->dev_private;
1262         struct szedata2_rx_queue *rxq = &internals->rx_queue[rx_queue_id];
1263         int ret;
1264         uint32_t rx = 1 << rx_queue_id;
1265         uint32_t tx = 0;
1266
1267         rxq->sze = szedata_open(internals->sze_dev);
1268         if (rxq->sze == NULL)
1269                 return -EINVAL;
1270         ret = szedata_subscribe3(rxq->sze, &rx, &tx);
1271         if (ret != 0 || rx == 0) {
1272                 szedata_close(rxq->sze);
1273                 rxq->sze = NULL;
1274                 return -EINVAL;
1275         }
1276         rxq->rx_channel = rx_queue_id;
1277         rxq->in_port = dev->data->port_id;
1278         rxq->mb_pool = mb_pool;
1279         rxq->rx_pkts = 0;
1280         rxq->rx_bytes = 0;
1281         rxq->err_pkts = 0;
1282
1283         dev->data->rx_queues[rx_queue_id] = rxq;
1284         return 0;
1285 }
1286
1287 static int
1288 eth_tx_queue_setup(struct rte_eth_dev *dev,
1289                 uint16_t tx_queue_id,
1290                 uint16_t nb_tx_desc __rte_unused,
1291                 unsigned int socket_id __rte_unused,
1292                 const struct rte_eth_txconf *tx_conf __rte_unused)
1293 {
1294         struct pmd_internals *internals = dev->data->dev_private;
1295         struct szedata2_tx_queue *txq = &internals->tx_queue[tx_queue_id];
1296         int ret;
1297         uint32_t rx = 0;
1298         uint32_t tx = 1 << tx_queue_id;
1299
1300         txq->sze = szedata_open(internals->sze_dev);
1301         if (txq->sze == NULL)
1302                 return -EINVAL;
1303         ret = szedata_subscribe3(txq->sze, &rx, &tx);
1304         if (ret != 0 || tx == 0) {
1305                 szedata_close(txq->sze);
1306                 txq->sze = NULL;
1307                 return -EINVAL;
1308         }
1309         txq->tx_channel = tx_queue_id;
1310         txq->tx_pkts = 0;
1311         txq->tx_bytes = 0;
1312         txq->err_pkts = 0;
1313
1314         dev->data->tx_queues[tx_queue_id] = txq;
1315         return 0;
1316 }
1317
1318 static void
1319 eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1320                 struct ether_addr *mac_addr __rte_unused)
1321 {
1322 }
1323
1324 static void
1325 eth_promiscuous_enable(struct rte_eth_dev *dev)
1326 {
1327         struct pmd_internals *internals = (struct pmd_internals *)
1328                 dev->data->dev_private;
1329         uint32_t i;
1330
1331         for (i = 0; i < szedata2_ibuf_count; i++) {
1332                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1333                                 SZEDATA2_MAC_CHMODE_PROMISC);
1334         }
1335 }
1336
1337 static void
1338 eth_promiscuous_disable(struct rte_eth_dev *dev)
1339 {
1340         struct pmd_internals *internals = (struct pmd_internals *)
1341                 dev->data->dev_private;
1342         uint32_t i;
1343
1344         for (i = 0; i < szedata2_ibuf_count; i++) {
1345                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1346                                 SZEDATA2_MAC_CHMODE_ONLY_VALID);
1347         }
1348 }
1349
1350 static void
1351 eth_allmulticast_enable(struct rte_eth_dev *dev)
1352 {
1353         struct pmd_internals *internals = (struct pmd_internals *)
1354                 dev->data->dev_private;
1355         uint32_t i;
1356
1357         for (i = 0; i < szedata2_ibuf_count; i++) {
1358                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1359                                 SZEDATA2_MAC_CHMODE_ALL_MULTICAST);
1360         }
1361 }
1362
1363 static void
1364 eth_allmulticast_disable(struct rte_eth_dev *dev)
1365 {
1366         struct pmd_internals *internals = (struct pmd_internals *)
1367                 dev->data->dev_private;
1368         uint32_t i;
1369
1370         for (i = 0; i < szedata2_ibuf_count; i++) {
1371                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1372                                 SZEDATA2_MAC_CHMODE_ONLY_VALID);
1373         }
1374 }
1375
1376 static const struct eth_dev_ops ops = {
1377         .dev_start          = eth_dev_start,
1378         .dev_stop           = eth_dev_stop,
1379         .dev_set_link_up    = eth_dev_set_link_up,
1380         .dev_set_link_down  = eth_dev_set_link_down,
1381         .dev_close          = eth_dev_close,
1382         .dev_configure      = eth_dev_configure,
1383         .dev_infos_get      = eth_dev_info,
1384         .promiscuous_enable   = eth_promiscuous_enable,
1385         .promiscuous_disable  = eth_promiscuous_disable,
1386         .allmulticast_enable  = eth_allmulticast_enable,
1387         .allmulticast_disable = eth_allmulticast_disable,
1388         .rx_queue_start     = eth_rx_queue_start,
1389         .rx_queue_stop      = eth_rx_queue_stop,
1390         .tx_queue_start     = eth_tx_queue_start,
1391         .tx_queue_stop      = eth_tx_queue_stop,
1392         .rx_queue_setup     = eth_rx_queue_setup,
1393         .tx_queue_setup     = eth_tx_queue_setup,
1394         .rx_queue_release   = eth_rx_queue_release,
1395         .tx_queue_release   = eth_tx_queue_release,
1396         .link_update        = eth_link_update,
1397         .stats_get          = eth_stats_get,
1398         .stats_reset        = eth_stats_reset,
1399         .mac_addr_set       = eth_mac_addr_set,
1400 };
1401
1402 /*
1403  * This function goes through sysfs and looks for an index of szedata2
1404  * device file (/dev/szedataIIX, where X is the index).
1405  *
1406  * @return
1407  *           0 on success
1408  *          -1 on error
1409  */
1410 static int
1411 get_szedata2_index(const struct rte_pci_addr *pcislot_addr, uint32_t *index)
1412 {
1413         DIR *dir;
1414         struct dirent *entry;
1415         int ret;
1416         uint32_t tmp_index;
1417         FILE *fd;
1418         char pcislot_path[PATH_MAX];
1419         uint32_t domain;
1420         uint32_t bus;
1421         uint32_t devid;
1422         uint32_t function;
1423
1424         dir = opendir("/sys/class/combo");
1425         if (dir == NULL)
1426                 return -1;
1427
1428         /*
1429          * Iterate through all combosixX directories.
1430          * When the value in /sys/class/combo/combosixX/device/pcislot
1431          * file is the location of the ethernet device dev, "X" is the
1432          * index of the device.
1433          */
1434         while ((entry = readdir(dir)) != NULL) {
1435                 ret = sscanf(entry->d_name, "combosix%u", &tmp_index);
1436                 if (ret != 1)
1437                         continue;
1438
1439                 snprintf(pcislot_path, PATH_MAX,
1440                         "/sys/class/combo/combosix%u/device/pcislot",
1441                         tmp_index);
1442
1443                 fd = fopen(pcislot_path, "r");
1444                 if (fd == NULL)
1445                         continue;
1446
1447                 ret = fscanf(fd, "%4" PRIx16 ":%2" PRIx8 ":%2" PRIx8 ".%" PRIx8,
1448                                 &domain, &bus, &devid, &function);
1449                 fclose(fd);
1450                 if (ret != 4)
1451                         continue;
1452
1453                 if (pcislot_addr->domain == domain &&
1454                                 pcislot_addr->bus == bus &&
1455                                 pcislot_addr->devid == devid &&
1456                                 pcislot_addr->function == function) {
1457                         *index = tmp_index;
1458                         closedir(dir);
1459                         return 0;
1460                 }
1461         }
1462
1463         closedir(dir);
1464         return -1;
1465 }
1466
1467 static int
1468 rte_szedata2_eth_dev_init(struct rte_eth_dev *dev)
1469 {
1470         struct rte_eth_dev_data *data = dev->data;
1471         struct pmd_internals *internals = (struct pmd_internals *)
1472                 data->dev_private;
1473         struct szedata *szedata_temp;
1474         int ret;
1475         uint32_t szedata2_index;
1476         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1477         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1478         struct rte_mem_resource *pci_rsc =
1479                 &pci_dev->mem_resource[PCI_RESOURCE_NUMBER];
1480         char rsc_filename[PATH_MAX];
1481         void *pci_resource_ptr = NULL;
1482         int fd;
1483
1484         RTE_LOG(INFO, PMD, "Initializing szedata2 device (" PCI_PRI_FMT ")\n",
1485                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1486                         pci_addr->function);
1487
1488         /* Get index of szedata2 device file and create path to device file */
1489         ret = get_szedata2_index(pci_addr, &szedata2_index);
1490         if (ret != 0) {
1491                 RTE_LOG(ERR, PMD, "Failed to get szedata2 device index!\n");
1492                 return -ENODEV;
1493         }
1494         snprintf(internals->sze_dev, PATH_MAX, SZEDATA2_DEV_PATH_FMT,
1495                         szedata2_index);
1496
1497         RTE_LOG(INFO, PMD, "SZEDATA2 path: %s\n", internals->sze_dev);
1498
1499         /*
1500          * Get number of available DMA RX and TX channels, which is maximum
1501          * number of queues that can be created and store it in private device
1502          * data structure.
1503          */
1504         szedata_temp = szedata_open(internals->sze_dev);
1505         if (szedata_temp == NULL) {
1506                 RTE_LOG(ERR, PMD, "szedata_open(): failed to open %s",
1507                                 internals->sze_dev);
1508                 return -EINVAL;
1509         }
1510         internals->max_rx_queues = szedata_ifaces_available(szedata_temp,
1511                         SZE2_DIR_RX);
1512         internals->max_tx_queues = szedata_ifaces_available(szedata_temp,
1513                         SZE2_DIR_TX);
1514         szedata_close(szedata_temp);
1515
1516         RTE_LOG(INFO, PMD, "Available DMA channels RX: %u TX: %u\n",
1517                         internals->max_rx_queues, internals->max_tx_queues);
1518
1519         /* Set rx, tx burst functions */
1520         if (data->dev_conf.rxmode.enable_scatter == 1 ||
1521                 data->scattered_rx == 1) {
1522                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1523                 data->scattered_rx = 1;
1524         } else {
1525                 dev->rx_pkt_burst = eth_szedata2_rx;
1526                 data->scattered_rx = 0;
1527         }
1528         dev->tx_pkt_burst = eth_szedata2_tx;
1529
1530         /* Set function callbacks for Ethernet API */
1531         dev->dev_ops = &ops;
1532
1533         rte_eth_copy_pci_info(dev, pci_dev);
1534
1535         /* mmap pci resource0 file to rte_mem_resource structure */
1536         if (pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr ==
1537                         0) {
1538                 RTE_LOG(ERR, PMD, "Missing resource%u file\n",
1539                                 PCI_RESOURCE_NUMBER);
1540                 return -EINVAL;
1541         }
1542         snprintf(rsc_filename, PATH_MAX,
1543                 "%s/" PCI_PRI_FMT "/resource%u", rte_pci_get_sysfs_path(),
1544                 pci_addr->domain, pci_addr->bus,
1545                 pci_addr->devid, pci_addr->function, PCI_RESOURCE_NUMBER);
1546         fd = open(rsc_filename, O_RDWR);
1547         if (fd < 0) {
1548                 RTE_LOG(ERR, PMD, "Could not open file %s\n", rsc_filename);
1549                 return -EINVAL;
1550         }
1551
1552         pci_resource_ptr = mmap(0,
1553                         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len,
1554                         PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1555         close(fd);
1556         if (pci_resource_ptr == MAP_FAILED) {
1557                 RTE_LOG(ERR, PMD, "Could not mmap file %s (fd = %d)\n",
1558                                 rsc_filename, fd);
1559                 return -EINVAL;
1560         }
1561         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr = pci_resource_ptr;
1562         internals->pci_rsc = pci_rsc;
1563
1564         RTE_LOG(DEBUG, PMD, "resource%u phys_addr = 0x%llx len = %llu "
1565                         "virt addr = %llx\n", PCI_RESOURCE_NUMBER,
1566                         (unsigned long long)pci_rsc->phys_addr,
1567                         (unsigned long long)pci_rsc->len,
1568                         (unsigned long long)pci_rsc->addr);
1569
1570         /* Get link state */
1571         eth_link_update(dev, 0);
1572
1573         /* Allocate space for one mac address */
1574         data->mac_addrs = rte_zmalloc(data->name, sizeof(struct ether_addr),
1575                         RTE_CACHE_LINE_SIZE);
1576         if (data->mac_addrs == NULL) {
1577                 RTE_LOG(ERR, PMD, "Could not alloc space for MAC address!\n");
1578                 munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1579                        pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1580                 return -EINVAL;
1581         }
1582
1583         ether_addr_copy(&eth_addr, data->mac_addrs);
1584
1585         /* At initial state COMBO card is in promiscuous mode so disable it */
1586         eth_promiscuous_disable(dev);
1587
1588         RTE_LOG(INFO, PMD, "szedata2 device ("
1589                         PCI_PRI_FMT ") successfully initialized\n",
1590                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1591                         pci_addr->function);
1592
1593         return 0;
1594 }
1595
1596 static int
1597 rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev)
1598 {
1599         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1600         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1601
1602         rte_free(dev->data->mac_addrs);
1603         dev->data->mac_addrs = NULL;
1604         munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1605                pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1606
1607         RTE_LOG(INFO, PMD, "szedata2 device ("
1608                         PCI_PRI_FMT ") successfully uninitialized\n",
1609                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1610                         pci_addr->function);
1611
1612         return 0;
1613 }
1614
1615 static const struct rte_pci_id rte_szedata2_pci_id_table[] = {
1616         {
1617                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1618                                 PCI_DEVICE_ID_NETCOPE_COMBO80G)
1619         },
1620         {
1621                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1622                                 PCI_DEVICE_ID_NETCOPE_COMBO100G)
1623         },
1624         {
1625                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1626                                 PCI_DEVICE_ID_NETCOPE_COMBO100G2)
1627         },
1628         {
1629                 .vendor_id = 0,
1630         }
1631 };
1632
1633 static int szedata2_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1634         struct rte_pci_device *pci_dev)
1635 {
1636         return rte_eth_dev_pci_generic_probe(pci_dev,
1637                 sizeof(struct pmd_internals), rte_szedata2_eth_dev_init);
1638 }
1639
1640 static int szedata2_eth_pci_remove(struct rte_pci_device *pci_dev)
1641 {
1642         return rte_eth_dev_pci_generic_remove(pci_dev,
1643                 rte_szedata2_eth_dev_uninit);
1644 }
1645
1646 static struct rte_pci_driver szedata2_eth_driver = {
1647         .id_table = rte_szedata2_pci_id_table,
1648         .probe = szedata2_eth_pci_probe,
1649         .remove = szedata2_eth_pci_remove,
1650 };
1651
1652 RTE_PMD_REGISTER_PCI(RTE_SZEDATA2_DRIVER_NAME, szedata2_eth_driver);
1653 RTE_PMD_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table);
1654 RTE_PMD_REGISTER_KMOD_DEP(RTE_SZEDATA2_DRIVER_NAME,
1655         "* combo6core & combov3 & szedata2 & szedata2_cv3");