drivers/net/bonding/rte_eth_bond_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33 #include <stdlib.h>
  34 #include <netinet/in.h>
  35
  36 #include <rte_mbuf.h>
  37 #include <rte_malloc.h>
  38 #include <rte_ethdev.h>
  39 #include <rte_ethdev_vdev.h>
  40 #include <rte_tcp.h>
  41 #include <rte_udp.h>
  42 #include <rte_ip.h>
  43 #include <rte_ip_frag.h>
  44 #include <rte_devargs.h>
  45 #include <rte_kvargs.h>
  46 #include <rte_vdev.h>
  47 #include <rte_alarm.h>
  48 #include <rte_cycles.h>
  49
  50 #include "rte_eth_bond.h"
  51 #include "rte_eth_bond_private.h"
  52 #include "rte_eth_bond_8023ad_private.h"
  53
  54 #define REORDER_PERIOD_MS 10
  55 #define DEFAULT_POLLING_INTERVAL_10_MS (10)
  56
  57 #define HASH_L4_PORTS(h) ((h)->src_port ^ (h)->dst_port)
  58
  59 /* Table for statistics in mode 5 TLB */
  60 static uint64_t tlb_last_obytets[RTE_MAX_ETHPORTS];
  61
  62 static inline size_t
  63 get_vlan_offset(struct ether_hdr *eth_hdr, uint16_t *proto)
  64 {
  65         size_t vlan_offset = 0;
  66
  67         if (rte_cpu_to_be_16(ETHER_TYPE_VLAN) == *proto) {
  68                 struct vlan_hdr *vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1);
  69
  70                 vlan_offset = sizeof(struct vlan_hdr);
  71                 *proto = vlan_hdr->eth_proto;
  72
  73                 if (rte_cpu_to_be_16(ETHER_TYPE_VLAN) == *proto) {
  74                         vlan_hdr = vlan_hdr + 1;
  75                         *proto = vlan_hdr->eth_proto;
  76                         vlan_offset += sizeof(struct vlan_hdr);
  77                 }
  78         }
  79         return vlan_offset;
  80 }
  81
  82 static uint16_t
  83 bond_ethdev_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
  84 {
  85         struct bond_dev_private *internals;
  86
  87         uint16_t num_rx_slave = 0;
  88         uint16_t num_rx_total = 0;
  89
  90         int i;
  91
  92         /* Cast to structure, containing bonded device's port id and queue id */
  93         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
  94
  95         internals = bd_rx_q->dev_private;
  96
  97
  98         for (i = 0; i < internals->active_slave_count && nb_pkts; i++) {
  99                 /* Offset of pointer to *bufs increases as packets are received
 100                  * from other slaves */
 101                 num_rx_slave = rte_eth_rx_burst(internals->active_slaves[i],
 102                                 bd_rx_q->queue_id, bufs + num_rx_total, nb_pkts);
 103                 if (num_rx_slave) {
 104                         num_rx_total += num_rx_slave;
 105                         nb_pkts -= num_rx_slave;
 106                 }
 107         }
 108
 109         return num_rx_total;
 110 }
 111
 112 static uint16_t
 113 bond_ethdev_rx_burst_active_backup(void *queue, struct rte_mbuf **bufs,
 114                 uint16_t nb_pkts)
 115 {
 116         struct bond_dev_private *internals;
 117
 118         /* Cast to structure, containing bonded device's port id and queue id */
 119         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
 120
 121         internals = bd_rx_q->dev_private;
 122
 123         return rte_eth_rx_burst(internals->current_primary_port,
 124                         bd_rx_q->queue_id, bufs, nb_pkts);
 125 }
 126
 127 static inline uint8_t
 128 is_lacp_packets(uint16_t ethertype, uint8_t subtype, uint16_t vlan_tci)
 129 {
 130         const uint16_t ether_type_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW);
 131
 132         return !vlan_tci && (ethertype == ether_type_slow_be &&
 133                 (subtype == SLOW_SUBTYPE_MARKER || subtype == SLOW_SUBTYPE_LACP));
 134 }
 135
 136 static uint16_t
 137 bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 138                 uint16_t nb_pkts)
 139 {
 140         /* Cast to structure, containing bonded device's port id and queue id */
 141         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
 142         struct bond_dev_private *internals = bd_rx_q->dev_private;
 143         struct ether_addr bond_mac;
 144
 145         struct ether_hdr *hdr;
 146
 147         const uint16_t ether_type_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW);
 148         uint16_t num_rx_total = 0;      /* Total number of received packets */
 149         uint8_t slaves[RTE_MAX_ETHPORTS];
 150         uint8_t slave_count, idx;
 151
 152         uint8_t collecting;  /* current slave collecting status */
 153         const uint8_t promisc = internals->promiscuous_en;
 154         uint8_t i, j, k;
 155         uint8_t subtype;
 156
 157         rte_eth_macaddr_get(internals->port_id, &bond_mac);
 158         /* Copy slave list to protect against slave up/down changes during tx
 159          * bursting */
 160         slave_count = internals->active_slave_count;
 161         memcpy(slaves, internals->active_slaves,
 162                         sizeof(internals->active_slaves[0]) * slave_count);
 163
 164         idx = internals->active_slave;
 165         if (idx >= slave_count) {
 166                 internals->active_slave = 0;
 167                 idx = 0;
 168         }
 169         for (i = 0; i < slave_count && num_rx_total < nb_pkts; i++) {
 170                 j = num_rx_total;
 171                 collecting = ACTOR_STATE(&mode_8023ad_ports[slaves[idx]],
 172                                          COLLECTING);
 173
 174                 /* Read packets from this slave */
 175                 num_rx_total += rte_eth_rx_burst(slaves[idx], bd_rx_q->queue_id,
 176                                 &bufs[num_rx_total], nb_pkts - num_rx_total);
 177
 178                 for (k = j; k < 2 && k < num_rx_total; k++)
 179                         rte_prefetch0(rte_pktmbuf_mtod(bufs[k], void *));
 180
 181                 /* Handle slow protocol packets. */
 182                 while (j < num_rx_total) {
 183                         if (j + 3 < num_rx_total)
 184                                 rte_prefetch0(rte_pktmbuf_mtod(bufs[j + 3], void *));
 185
 186                         hdr = rte_pktmbuf_mtod(bufs[j], struct ether_hdr *);
 187                         subtype = ((struct slow_protocol_frame *)hdr)->slow_protocol.subtype;
 188
 189                         /* Remove packet from array if it is slow packet or slave is not
 190                          * in collecting state or bondign interface is not in promiscus
 191                          * mode and packet address does not match. */
 192                         if (unlikely(is_lacp_packets(hdr->ether_type, subtype, bufs[j]->vlan_tci) ||
 193                                 !collecting || (!promisc &&
 194                                         !is_multicast_ether_addr(&hdr->d_addr) &&
 195                                         !is_same_ether_addr(&bond_mac, &hdr->d_addr)))) {
 196
 197                                 if (hdr->ether_type == ether_type_slow_be) {
 198                                         bond_mode_8023ad_handle_slow_pkt(
 199                                             internals, slaves[idx], bufs[j]);
 200                                 } else
 201                                         rte_pktmbuf_free(bufs[j]);
 202
 203                                 /* Packet is managed by mode 4 or dropped, shift the array */
 204                                 num_rx_total--;
 205                                 if (j < num_rx_total) {
 206                                         memmove(&bufs[j], &bufs[j + 1], sizeof(bufs[0]) *
 207                                                 (num_rx_total - j));
 208                                 }
 209                         } else
 210                                 j++;
 211                 }
 212                 if (unlikely(++idx == slave_count))
 213                         idx = 0;
 214         }
 215
 216         internals->active_slave = idx;
 217         return num_rx_total;
 218 }
 219
 220 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 221 uint32_t burstnumberRX;
 222 uint32_t burstnumberTX;
 223
 224 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 225
 226 static void
 227 arp_op_name(uint16_t arp_op, char *buf)
 228 {
 229         switch (arp_op) {
 230         case ARP_OP_REQUEST:
 231                 snprintf(buf, sizeof("ARP Request"), "%s", "ARP Request");
 232                 return;
 233         case ARP_OP_REPLY:
 234                 snprintf(buf, sizeof("ARP Reply"), "%s", "ARP Reply");
 235                 return;
 236         case ARP_OP_REVREQUEST:
 237                 snprintf(buf, sizeof("Reverse ARP Request"), "%s",
 238                                 "Reverse ARP Request");
 239                 return;
 240         case ARP_OP_REVREPLY:
 241                 snprintf(buf, sizeof("Reverse ARP Reply"), "%s",
 242                                 "Reverse ARP Reply");
 243                 return;
 244         case ARP_OP_INVREQUEST:
 245                 snprintf(buf, sizeof("Peer Identify Request"), "%s",
 246                                 "Peer Identify Request");
 247                 return;
 248         case ARP_OP_INVREPLY:
 249                 snprintf(buf, sizeof("Peer Identify Reply"), "%s",
 250                                 "Peer Identify Reply");
 251                 return;
 252         default:
 253                 break;
 254         }
 255         snprintf(buf, sizeof("Unknown"), "%s", "Unknown");
 256         return;
 257 }
 258 #endif
 259 #define MaxIPv4String   16
 260 static void
 261 ipv4_addr_to_dot(uint32_t be_ipv4_addr, char *buf, uint8_t buf_size)
 262 {
 263         uint32_t ipv4_addr;
 264
 265         ipv4_addr = rte_be_to_cpu_32(be_ipv4_addr);
 266         snprintf(buf, buf_size, "%d.%d.%d.%d", (ipv4_addr >> 24) & 0xFF,
 267                 (ipv4_addr >> 16) & 0xFF, (ipv4_addr >> 8) & 0xFF,
 268                 ipv4_addr & 0xFF);
 269 }
 270
 271 #define MAX_CLIENTS_NUMBER      128
 272 uint8_t active_clients;
 273 struct client_stats_t {
 274         uint8_t port;
 275         uint32_t ipv4_addr;
 276         uint32_t ipv4_rx_packets;
 277         uint32_t ipv4_tx_packets;
 278 };
 279 struct client_stats_t client_stats[MAX_CLIENTS_NUMBER];
 280
 281 static void
 282 update_client_stats(uint32_t addr, uint8_t port, uint32_t *TXorRXindicator)
 283 {
 284         int i = 0;
 285
 286         for (; i < MAX_CLIENTS_NUMBER; i++)     {
 287                 if ((client_stats[i].ipv4_addr == addr) && (client_stats[i].port == port))      {
 288                         /* Just update RX packets number for this client */
 289                         if (TXorRXindicator == &burstnumberRX)
 290                                 client_stats[i].ipv4_rx_packets++;
 291                         else
 292                                 client_stats[i].ipv4_tx_packets++;
 293                         return;
 294                 }
 295         }
 296         /* We have a new client. Insert him to the table, and increment stats */
 297         if (TXorRXindicator == &burstnumberRX)
 298                 client_stats[active_clients].ipv4_rx_packets++;
 299         else
 300                 client_stats[active_clients].ipv4_tx_packets++;
 301         client_stats[active_clients].ipv4_addr = addr;
 302         client_stats[active_clients].port = port;
 303         active_clients++;
 304
 305 }
 306
 307 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 308 #define MODE6_DEBUG(info, src_ip, dst_ip, eth_h, arp_op, port, burstnumber)     \
 309                 RTE_LOG(DEBUG, PMD, \
 310                 "%s " \
 311                 "port:%d " \
 312                 "SrcMAC:%02X:%02X:%02X:%02X:%02X:%02X " \
 313                 "SrcIP:%s " \
 314                 "DstMAC:%02X:%02X:%02X:%02X:%02X:%02X " \
 315                 "DstIP:%s " \
 316                 "%s " \
 317                 "%d\n", \
 318                 info, \
 319                 port, \
 320                 eth_h->s_addr.addr_bytes[0], \
 321                 eth_h->s_addr.addr_bytes[1], \
 322                 eth_h->s_addr.addr_bytes[2], \
 323                 eth_h->s_addr.addr_bytes[3], \
 324                 eth_h->s_addr.addr_bytes[4], \
 325                 eth_h->s_addr.addr_bytes[5], \
 326                 src_ip, \
 327                 eth_h->d_addr.addr_bytes[0], \
 328                 eth_h->d_addr.addr_bytes[1], \
 329                 eth_h->d_addr.addr_bytes[2], \
 330                 eth_h->d_addr.addr_bytes[3], \
 331                 eth_h->d_addr.addr_bytes[4], \
 332                 eth_h->d_addr.addr_bytes[5], \
 333                 dst_ip, \
 334                 arp_op, \
 335                 ++burstnumber)
 336 #endif
 337
 338 static void
 339 mode6_debug(const char __attribute__((unused)) *info, struct ether_hdr *eth_h,
 340                 uint8_t port, uint32_t __attribute__((unused)) *burstnumber)
 341 {
 342         struct ipv4_hdr *ipv4_h;
 343 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 344         struct arp_hdr *arp_h;
 345         char dst_ip[16];
 346         char ArpOp[24];
 347         char buf[16];
 348 #endif
 349         char src_ip[16];
 350
 351         uint16_t ether_type = eth_h->ether_type;
 352         uint16_t offset = get_vlan_offset(eth_h, &ether_type);
 353
 354 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 355         snprintf(buf, 16, "%s", info);
 356 #endif
 357
 358         if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) {
 359                 ipv4_h = (struct ipv4_hdr *)((char *)(eth_h + 1) + offset);
 360                 ipv4_addr_to_dot(ipv4_h->src_addr, src_ip, MaxIPv4String);
 361 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 362                 ipv4_addr_to_dot(ipv4_h->dst_addr, dst_ip, MaxIPv4String);
 363                 MODE6_DEBUG(buf, src_ip, dst_ip, eth_h, "", port, *burstnumber);
 364 #endif
 365                 update_client_stats(ipv4_h->src_addr, port, burstnumber);
 366         }
 367 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 368         else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_ARP)) {
 369                 arp_h = (struct arp_hdr *)((char *)(eth_h + 1) + offset);
 370                 ipv4_addr_to_dot(arp_h->arp_data.arp_sip, src_ip, MaxIPv4String);
 371                 ipv4_addr_to_dot(arp_h->arp_data.arp_tip, dst_ip, MaxIPv4String);
 372                 arp_op_name(rte_be_to_cpu_16(arp_h->arp_op), ArpOp);
 373                 MODE6_DEBUG(buf, src_ip, dst_ip, eth_h, ArpOp, port, *burstnumber);
 374         }
 375 #endif
 376 }
 377 #endif
 378
 379 static uint16_t
 380 bond_ethdev_rx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 381 {
 382         struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
 383         struct bond_dev_private *internals = bd_tx_q->dev_private;
 384         struct ether_hdr *eth_h;
 385         uint16_t ether_type, offset;
 386         uint16_t nb_recv_pkts;
 387         int i;
 388
 389         nb_recv_pkts = bond_ethdev_rx_burst(queue, bufs, nb_pkts);
 390
 391         for (i = 0; i < nb_recv_pkts; i++) {
 392                 eth_h = rte_pktmbuf_mtod(bufs[i], struct ether_hdr *);
 393                 ether_type = eth_h->ether_type;
 394                 offset = get_vlan_offset(eth_h, &ether_type);
 395
 396                 if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_ARP)) {
 397 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 398                         mode6_debug("RX ARP:", eth_h, bufs[i]->port, &burstnumberRX);
 399 #endif
 400                         bond_mode_alb_arp_recv(eth_h, offset, internals);
 401                 }
 402 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 403                 else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4))
 404                         mode6_debug("RX IPv4:", eth_h, bufs[i]->port, &burstnumberRX);
 405 #endif
 406         }
 407
 408         return nb_recv_pkts;
 409 }
 410
 411 static uint16_t
 412 bond_ethdev_tx_burst_round_robin(void *queue, struct rte_mbuf **bufs,
 413                 uint16_t nb_pkts)
 414 {
 415         struct bond_dev_private *internals;
 416         struct bond_tx_queue *bd_tx_q;
 417
 418         struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts];
 419         uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
 420
 421         uint8_t num_of_slaves;
 422         uint8_t slaves[RTE_MAX_ETHPORTS];
 423
 424         uint16_t num_tx_total = 0, num_tx_slave;
 425
 426         static int slave_idx = 0;
 427         int i, cslave_idx = 0, tx_fail_total = 0;
 428
 429         bd_tx_q = (struct bond_tx_queue *)queue;
 430         internals = bd_tx_q->dev_private;
 431
 432         /* Copy slave list to protect against slave up/down changes during tx
 433          * bursting */
 434         num_of_slaves = internals->active_slave_count;
 435         memcpy(slaves, internals->active_slaves,
 436                         sizeof(internals->active_slaves[0]) * num_of_slaves);
 437
 438         if (num_of_slaves < 1)
 439                 return num_tx_total;
 440
 441         /* Populate slaves mbuf with which packets are to be sent on it  */
 442         for (i = 0; i < nb_pkts; i++) {
 443                 cslave_idx = (slave_idx + i) % num_of_slaves;
 444                 slave_bufs[cslave_idx][(slave_nb_pkts[cslave_idx])++] = bufs[i];
 445         }
 446
 447         /* increment current slave index so the next call to tx burst starts on the
 448          * next slave */
 449         slave_idx = ++cslave_idx;
 450
 451         /* Send packet burst on each slave device */
 452         for (i = 0; i < num_of_slaves; i++) {
 453                 if (slave_nb_pkts[i] > 0) {
 454                         num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
 455                                         slave_bufs[i], slave_nb_pkts[i]);
 456
 457                         /* if tx burst fails move packets to end of bufs */
 458                         if (unlikely(num_tx_slave < slave_nb_pkts[i])) {
 459                                 int tx_fail_slave = slave_nb_pkts[i] - num_tx_slave;
 460
 461                                 tx_fail_total += tx_fail_slave;
 462
 463                                 memcpy(&bufs[nb_pkts - tx_fail_total],
 464                                                 &slave_bufs[i][num_tx_slave],
 465                                                 tx_fail_slave * sizeof(bufs[0]));
 466                         }
 467                         num_tx_total += num_tx_slave;
 468                 }
 469         }
 470
 471         return num_tx_total;
 472 }
 473
 474 static uint16_t
 475 bond_ethdev_tx_burst_active_backup(void *queue,
 476                 struct rte_mbuf **bufs, uint16_t nb_pkts)
 477 {
 478         struct bond_dev_private *internals;
 479         struct bond_tx_queue *bd_tx_q;
 480
 481         bd_tx_q = (struct bond_tx_queue *)queue;
 482         internals = bd_tx_q->dev_private;
 483
 484         if (internals->active_slave_count < 1)
 485                 return 0;
 486
 487         return rte_eth_tx_burst(internals->current_primary_port, bd_tx_q->queue_id,
 488                         bufs, nb_pkts);
 489 }
 490
 491 static inline uint16_t
 492 ether_hash(struct ether_hdr *eth_hdr)
 493 {
 494         unaligned_uint16_t *word_src_addr =
 495                 (unaligned_uint16_t *)eth_hdr->s_addr.addr_bytes;
 496         unaligned_uint16_t *word_dst_addr =
 497                 (unaligned_uint16_t *)eth_hdr->d_addr.addr_bytes;
 498
 499         return (word_src_addr[0] ^ word_dst_addr[0]) ^
 500                         (word_src_addr[1] ^ word_dst_addr[1]) ^
 501                         (word_src_addr[2] ^ word_dst_addr[2]);
 502 }
 503
 504 static inline uint32_t
 505 ipv4_hash(struct ipv4_hdr *ipv4_hdr)
 506 {
 507         return ipv4_hdr->src_addr ^ ipv4_hdr->dst_addr;
 508 }
 509
 510 static inline uint32_t
 511 ipv6_hash(struct ipv6_hdr *ipv6_hdr)
 512 {
 513         unaligned_uint32_t *word_src_addr =
 514                 (unaligned_uint32_t *)&(ipv6_hdr->src_addr[0]);
 515         unaligned_uint32_t *word_dst_addr =
 516                 (unaligned_uint32_t *)&(ipv6_hdr->dst_addr[0]);
 517
 518         return (word_src_addr[0] ^ word_dst_addr[0]) ^
 519                         (word_src_addr[1] ^ word_dst_addr[1]) ^
 520                         (word_src_addr[2] ^ word_dst_addr[2]) ^
 521                         (word_src_addr[3] ^ word_dst_addr[3]);
 522 }
 523
 524 uint16_t
 525 xmit_l2_hash(const struct rte_mbuf *buf, uint8_t slave_count)
 526 {
 527         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(buf, struct ether_hdr *);
 528
 529         uint32_t hash = ether_hash(eth_hdr);
 530
 531         return (hash ^= hash >> 8) % slave_count;
 532 }
 533
 534 uint16_t
 535 xmit_l23_hash(const struct rte_mbuf *buf, uint8_t slave_count)
 536 {
 537         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(buf, struct ether_hdr *);
 538         uint16_t proto = eth_hdr->ether_type;
 539         size_t vlan_offset = get_vlan_offset(eth_hdr, &proto);
 540         uint32_t hash, l3hash = 0;
 541
 542         hash = ether_hash(eth_hdr);
 543
 544         if (rte_cpu_to_be_16(ETHER_TYPE_IPv4) == proto) {
 545                 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *)
 546                                 ((char *)(eth_hdr + 1) + vlan_offset);
 547                 l3hash = ipv4_hash(ipv4_hdr);
 548
 549         } else if (rte_cpu_to_be_16(ETHER_TYPE_IPv6) == proto) {
 550                 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)
 551                                 ((char *)(eth_hdr + 1) + vlan_offset);
 552                 l3hash = ipv6_hash(ipv6_hdr);
 553         }
 554
 555         hash = hash ^ l3hash;
 556         hash ^= hash >> 16;
 557         hash ^= hash >> 8;
 558
 559         return hash % slave_count;
 560 }
 561
 562 uint16_t
 563 xmit_l34_hash(const struct rte_mbuf *buf, uint8_t slave_count)
 564 {
 565         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(buf, struct ether_hdr *);
 566         uint16_t proto = eth_hdr->ether_type;
 567         size_t vlan_offset = get_vlan_offset(eth_hdr, &proto);
 568
 569         struct udp_hdr *udp_hdr = NULL;
 570         struct tcp_hdr *tcp_hdr = NULL;
 571         uint32_t hash, l3hash = 0, l4hash = 0;
 572
 573         if (rte_cpu_to_be_16(ETHER_TYPE_IPv4) == proto) {
 574                 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *)
 575                                 ((char *)(eth_hdr + 1) + vlan_offset);
 576                 size_t ip_hdr_offset;
 577
 578                 l3hash = ipv4_hash(ipv4_hdr);
 579
 580                 /* there is no L4 header in fragmented packet */
 581                 if (likely(rte_ipv4_frag_pkt_is_fragmented(ipv4_hdr) == 0)) {
 582                         ip_hdr_offset = (ipv4_hdr->version_ihl & IPV4_HDR_IHL_MASK) *
 583                                         IPV4_IHL_MULTIPLIER;
 584
 585                         if (ipv4_hdr->next_proto_id == IPPROTO_TCP) {
 586                                 tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr +
 587                                                 ip_hdr_offset);
 588                                 l4hash = HASH_L4_PORTS(tcp_hdr);
 589                         } else if (ipv4_hdr->next_proto_id == IPPROTO_UDP) {
 590                                 udp_hdr = (struct udp_hdr *)((char *)ipv4_hdr +
 591                                                 ip_hdr_offset);
 592                                 l4hash = HASH_L4_PORTS(udp_hdr);
 593                         }
 594                 }
 595         } else if  (rte_cpu_to_be_16(ETHER_TYPE_IPv6) == proto) {
 596                 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)
 597                                 ((char *)(eth_hdr + 1) + vlan_offset);
 598                 l3hash = ipv6_hash(ipv6_hdr);
 599
 600                 if (ipv6_hdr->proto == IPPROTO_TCP) {
 601                         tcp_hdr = (struct tcp_hdr *)(ipv6_hdr + 1);
 602                         l4hash = HASH_L4_PORTS(tcp_hdr);
 603                 } else if (ipv6_hdr->proto == IPPROTO_UDP) {
 604                         udp_hdr = (struct udp_hdr *)(ipv6_hdr + 1);
 605                         l4hash = HASH_L4_PORTS(udp_hdr);
 606                 }
 607         }
 608
 609         hash = l3hash ^ l4hash;
 610         hash ^= hash >> 16;
 611         hash ^= hash >> 8;
 612
 613         return hash % slave_count;
 614 }
 615
 616 struct bwg_slave {
 617         uint64_t bwg_left_int;
 618         uint64_t bwg_left_remainder;
 619         uint8_t slave;
 620 };
 621
 622 void
 623 bond_tlb_activate_slave(struct bond_dev_private *internals) {
 624         int i;
 625
 626         for (i = 0; i < internals->active_slave_count; i++) {
 627                 tlb_last_obytets[internals->active_slaves[i]] = 0;
 628         }
 629 }
 630
 631 static int
 632 bandwidth_cmp(const void *a, const void *b)
 633 {
 634         const struct bwg_slave *bwg_a = a;
 635         const struct bwg_slave *bwg_b = b;
 636         int64_t diff = (int64_t)bwg_b->bwg_left_int - (int64_t)bwg_a->bwg_left_int;
 637         int64_t diff2 = (int64_t)bwg_b->bwg_left_remainder -
 638                         (int64_t)bwg_a->bwg_left_remainder;
 639         if (diff > 0)
 640                 return 1;
 641         else if (diff < 0)
 642                 return -1;
 643         else if (diff2 > 0)
 644                 return 1;
 645         else if (diff2 < 0)
 646                 return -1;
 647         else
 648                 return 0;
 649 }
 650
 651 static void
 652 bandwidth_left(uint8_t port_id, uint64_t load, uint8_t update_idx,
 653                 struct bwg_slave *bwg_slave)
 654 {
 655         struct rte_eth_link link_status;
 656
 657         rte_eth_link_get_nowait(port_id, &link_status);
 658         uint64_t link_bwg = link_status.link_speed * 1000000ULL / 8;
 659         if (link_bwg == 0)
 660                 return;
 661         link_bwg = link_bwg * (update_idx+1) * REORDER_PERIOD_MS;
 662         bwg_slave->bwg_left_int = (link_bwg - 1000*load) / link_bwg;
 663         bwg_slave->bwg_left_remainder = (link_bwg - 1000*load) % link_bwg;
 664 }
 665
 666 static void
 667 bond_ethdev_update_tlb_slave_cb(void *arg)
 668 {
 669         struct bond_dev_private *internals = arg;
 670         struct rte_eth_stats slave_stats;
 671         struct bwg_slave bwg_array[RTE_MAX_ETHPORTS];
 672         uint8_t slave_count;
 673         uint64_t tx_bytes;
 674
 675         uint8_t update_stats = 0;
 676         uint8_t i, slave_id;
 677
 678         internals->slave_update_idx++;
 679
 680
 681         if (internals->slave_update_idx >= REORDER_PERIOD_MS)
 682                 update_stats = 1;
 683
 684         for (i = 0; i < internals->active_slave_count; i++) {
 685                 slave_id = internals->active_slaves[i];
 686                 rte_eth_stats_get(slave_id, &slave_stats);
 687                 tx_bytes = slave_stats.obytes - tlb_last_obytets[slave_id];
 688                 bandwidth_left(slave_id, tx_bytes,
 689                                 internals->slave_update_idx, &bwg_array[i]);
 690                 bwg_array[i].slave = slave_id;
 691
 692                 if (update_stats) {
 693                         tlb_last_obytets[slave_id] = slave_stats.obytes;
 694                 }
 695         }
 696
 697         if (update_stats == 1)
 698                 internals->slave_update_idx = 0;
 699
 700         slave_count = i;
 701         qsort(bwg_array, slave_count, sizeof(bwg_array[0]), bandwidth_cmp);
 702         for (i = 0; i < slave_count; i++)
 703                 internals->tlb_slaves_order[i] = bwg_array[i].slave;
 704
 705         rte_eal_alarm_set(REORDER_PERIOD_MS * 1000, bond_ethdev_update_tlb_slave_cb,
 706                         (struct bond_dev_private *)internals);
 707 }
 708
 709 static uint16_t
 710 bond_ethdev_tx_burst_tlb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 711 {
 712         struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
 713         struct bond_dev_private *internals = bd_tx_q->dev_private;
 714
 715         struct rte_eth_dev *primary_port =
 716                         &rte_eth_devices[internals->primary_port];
 717         uint16_t num_tx_total = 0;
 718         uint8_t i, j;
 719
 720         uint8_t num_of_slaves = internals->active_slave_count;
 721         uint8_t slaves[RTE_MAX_ETHPORTS];
 722
 723         struct ether_hdr *ether_hdr;
 724         struct ether_addr primary_slave_addr;
 725         struct ether_addr active_slave_addr;
 726
 727         if (num_of_slaves < 1)
 728                 return num_tx_total;
 729
 730         memcpy(slaves, internals->tlb_slaves_order,
 731                                 sizeof(internals->tlb_slaves_order[0]) * num_of_slaves);
 732
 733
 734         ether_addr_copy(primary_port->data->mac_addrs, &primary_slave_addr);
 735
 736         if (nb_pkts > 3) {
 737                 for (i = 0; i < 3; i++)
 738                         rte_prefetch0(rte_pktmbuf_mtod(bufs[i], void*));
 739         }
 740
 741         for (i = 0; i < num_of_slaves; i++) {
 742                 rte_eth_macaddr_get(slaves[i], &active_slave_addr);
 743                 for (j = num_tx_total; j < nb_pkts; j++) {
 744                         if (j + 3 < nb_pkts)
 745                                 rte_prefetch0(rte_pktmbuf_mtod(bufs[j+3], void*));
 746
 747                         ether_hdr = rte_pktmbuf_mtod(bufs[j], struct ether_hdr *);
 748                         if (is_same_ether_addr(&ether_hdr->s_addr, &primary_slave_addr))
 749                                 ether_addr_copy(&active_slave_addr, &ether_hdr->s_addr);
 750 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 751                                         mode6_debug("TX IPv4:", ether_hdr, slaves[i], &burstnumberTX);
 752 #endif
 753                 }
 754
 755                 num_tx_total += rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
 756                                 bufs + num_tx_total, nb_pkts - num_tx_total);
 757
 758                 if (num_tx_total == nb_pkts)
 759                         break;
 760         }
 761
 762         return num_tx_total;
 763 }
 764
 765 void
 766 bond_tlb_disable(struct bond_dev_private *internals)
 767 {
 768         rte_eal_alarm_cancel(bond_ethdev_update_tlb_slave_cb, internals);
 769 }
 770
 771 void
 772 bond_tlb_enable(struct bond_dev_private *internals)
 773 {
 774         bond_ethdev_update_tlb_slave_cb(internals);
 775 }
 776
 777 static uint16_t
 778 bond_ethdev_tx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 779 {
 780         struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
 781         struct bond_dev_private *internals = bd_tx_q->dev_private;
 782
 783         struct ether_hdr *eth_h;
 784         uint16_t ether_type, offset;
 785
 786         struct client_data *client_info;
 787
 788         /*
 789          * We create transmit buffers for every slave and one additional to send
 790          * through tlb. In worst case every packet will be send on one port.
 791          */
 792         struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS + 1][nb_pkts];
 793         uint16_t slave_bufs_pkts[RTE_MAX_ETHPORTS + 1] = { 0 };
 794
 795         /*
 796          * We create separate transmit buffers for update packets as they wont be
 797          * counted in num_tx_total.
 798          */
 799         struct rte_mbuf *update_bufs[RTE_MAX_ETHPORTS][ALB_HASH_TABLE_SIZE];
 800         uint16_t update_bufs_pkts[RTE_MAX_ETHPORTS] = { 0 };
 801
 802         struct rte_mbuf *upd_pkt;
 803         size_t pkt_size;
 804
 805         uint16_t num_send, num_not_send = 0;
 806         uint16_t num_tx_total = 0;
 807         uint8_t slave_idx;
 808
 809         int i, j;
 810
 811         /* Search tx buffer for ARP packets and forward them to alb */
 812         for (i = 0; i < nb_pkts; i++) {
 813                 eth_h = rte_pktmbuf_mtod(bufs[i], struct ether_hdr *);
 814                 ether_type = eth_h->ether_type;
 815                 offset = get_vlan_offset(eth_h, &ether_type);
 816
 817                 if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_ARP)) {
 818                         slave_idx = bond_mode_alb_arp_xmit(eth_h, offset, internals);
 819
 820                         /* Change src mac in eth header */
 821                         rte_eth_macaddr_get(slave_idx, &eth_h->s_addr);
 822
 823                         /* Add packet to slave tx buffer */
 824                         slave_bufs[slave_idx][slave_bufs_pkts[slave_idx]] = bufs[i];
 825                         slave_bufs_pkts[slave_idx]++;
 826                 } else {
 827                         /* If packet is not ARP, send it with TLB policy */
 828                         slave_bufs[RTE_MAX_ETHPORTS][slave_bufs_pkts[RTE_MAX_ETHPORTS]] =
 829                                         bufs[i];
 830                         slave_bufs_pkts[RTE_MAX_ETHPORTS]++;
 831                 }
 832         }
 833
 834         /* Update connected client ARP tables */
 835         if (internals->mode6.ntt) {
 836                 for (i = 0; i < ALB_HASH_TABLE_SIZE; i++) {
 837                         client_info = &internals->mode6.client_table[i];
 838
 839                         if (client_info->in_use) {
 840                                 /* Allocate new packet to send ARP update on current slave */
 841                                 upd_pkt = rte_pktmbuf_alloc(internals->mode6.mempool);
 842                                 if (upd_pkt == NULL) {
 843                                         RTE_LOG(ERR, PMD, "Failed to allocate ARP packet from pool\n");
 844                                         continue;
 845                                 }
 846                                 pkt_size = sizeof(struct ether_hdr) + sizeof(struct arp_hdr)
 847                                                 + client_info->vlan_count * sizeof(struct vlan_hdr);
 848                                 upd_pkt->data_len = pkt_size;
 849                                 upd_pkt->pkt_len = pkt_size;
 850
 851                                 slave_idx = bond_mode_alb_arp_upd(client_info, upd_pkt,
 852                                                 internals);
 853
 854                                 /* Add packet to update tx buffer */
 855                                 update_bufs[slave_idx][update_bufs_pkts[slave_idx]] = upd_pkt;
 856                                 update_bufs_pkts[slave_idx]++;
 857                         }
 858                 }
 859                 internals->mode6.ntt = 0;
 860         }
 861
 862         /* Send ARP packets on proper slaves */
 863         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 864                 if (slave_bufs_pkts[i] > 0) {
 865                         num_send = rte_eth_tx_burst(i, bd_tx_q->queue_id,
 866                                         slave_bufs[i], slave_bufs_pkts[i]);
 867                         for (j = 0; j < slave_bufs_pkts[i] - num_send; j++) {
 868                                 bufs[nb_pkts - 1 - num_not_send - j] =
 869                                                 slave_bufs[i][nb_pkts - 1 - j];
 870                         }
 871
 872                         num_tx_total += num_send;
 873                         num_not_send += slave_bufs_pkts[i] - num_send;
 874
 875 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 876         /* Print TX stats including update packets */
 877                         for (j = 0; j < slave_bufs_pkts[i]; j++) {
 878                                 eth_h = rte_pktmbuf_mtod(slave_bufs[i][j], struct ether_hdr *);
 879                                 mode6_debug("TX ARP:", eth_h, i, &burstnumberTX);
 880                         }
 881 #endif
 882                 }
 883         }
 884
 885         /* Send update packets on proper slaves */
 886         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 887                 if (update_bufs_pkts[i] > 0) {
 888                         num_send = rte_eth_tx_burst(i, bd_tx_q->queue_id, update_bufs[i],
 889                                         update_bufs_pkts[i]);
 890                         for (j = num_send; j < update_bufs_pkts[i]; j++) {
 891                                 rte_pktmbuf_free(update_bufs[i][j]);
 892                         }
 893 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 894                         for (j = 0; j < update_bufs_pkts[i]; j++) {
 895                                 eth_h = rte_pktmbuf_mtod(update_bufs[i][j], struct ether_hdr *);
 896                                 mode6_debug("TX ARPupd:", eth_h, i, &burstnumberTX);
 897                         }
 898 #endif
 899                 }
 900         }
 901
 902         /* Send non-ARP packets using tlb policy */
 903         if (slave_bufs_pkts[RTE_MAX_ETHPORTS] > 0) {
 904                 num_send = bond_ethdev_tx_burst_tlb(queue,
 905                                 slave_bufs[RTE_MAX_ETHPORTS],
 906                                 slave_bufs_pkts[RTE_MAX_ETHPORTS]);
 907
 908                 for (j = 0; j < slave_bufs_pkts[RTE_MAX_ETHPORTS]; j++) {
 909                         bufs[nb_pkts - 1 - num_not_send - j] =
 910                                         slave_bufs[RTE_MAX_ETHPORTS][nb_pkts - 1 - j];
 911                 }
 912
 913                 num_tx_total += num_send;
 914         }
 915
 916         return num_tx_total;
 917 }
 918
 919 static uint16_t
 920 bond_ethdev_tx_burst_balance(void *queue, struct rte_mbuf **bufs,
 921                 uint16_t nb_pkts)
 922 {
 923         struct bond_dev_private *internals;
 924         struct bond_tx_queue *bd_tx_q;
 925
 926         uint8_t num_of_slaves;
 927         uint8_t slaves[RTE_MAX_ETHPORTS];
 928
 929         uint16_t num_tx_total = 0, num_tx_slave = 0, tx_fail_total = 0;
 930
 931         int i, op_slave_id;
 932
 933         struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts];
 934         uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
 935
 936         bd_tx_q = (struct bond_tx_queue *)queue;
 937         internals = bd_tx_q->dev_private;
 938
 939         /* Copy slave list to protect against slave up/down changes during tx
 940          * bursting */
 941         num_of_slaves = internals->active_slave_count;
 942         memcpy(slaves, internals->active_slaves,
 943                         sizeof(internals->active_slaves[0]) * num_of_slaves);
 944
 945         if (num_of_slaves < 1)
 946                 return num_tx_total;
 947
 948         /* Populate slaves mbuf with the packets which are to be sent on it  */
 949         for (i = 0; i < nb_pkts; i++) {
 950                 /* Select output slave using hash based on xmit policy */
 951                 op_slave_id = internals->xmit_hash(bufs[i], num_of_slaves);
 952
 953                 /* Populate slave mbuf arrays with mbufs for that slave */
 954                 slave_bufs[op_slave_id][slave_nb_pkts[op_slave_id]++] = bufs[i];
 955         }
 956
 957         /* Send packet burst on each slave device */
 958         for (i = 0; i < num_of_slaves; i++) {
 959                 if (slave_nb_pkts[i] > 0) {
 960                         num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
 961                                         slave_bufs[i], slave_nb_pkts[i]);
 962
 963                         /* if tx burst fails move packets to end of bufs */
 964                         if (unlikely(num_tx_slave < slave_nb_pkts[i])) {
 965                                 int slave_tx_fail_count = slave_nb_pkts[i] - num_tx_slave;
 966
 967                                 tx_fail_total += slave_tx_fail_count;
 968                                 memcpy(&bufs[nb_pkts - tx_fail_total],
 969                                                 &slave_bufs[i][num_tx_slave],
 970                                                 slave_tx_fail_count * sizeof(bufs[0]));
 971                         }
 972
 973                         num_tx_total += num_tx_slave;
 974                 }
 975         }
 976
 977         return num_tx_total;
 978 }
 979
 980 static uint16_t
 981 bond_ethdev_tx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 982                 uint16_t nb_pkts)
 983 {
 984         struct bond_dev_private *internals;
 985         struct bond_tx_queue *bd_tx_q;
 986
 987         uint8_t num_of_slaves;
 988         uint8_t slaves[RTE_MAX_ETHPORTS];
 989          /* positions in slaves, not ID */
 990         uint8_t distributing_offsets[RTE_MAX_ETHPORTS];
 991         uint8_t distributing_count;
 992
 993         uint16_t num_tx_slave, num_tx_total = 0, num_tx_fail_total = 0;
 994         uint16_t i, j, op_slave_idx;
 995         const uint16_t buffs_size = nb_pkts + BOND_MODE_8023AX_SLAVE_TX_PKTS + 1;
 996
 997         /* Allocate additional packets in case 8023AD mode. */
 998         struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][buffs_size];
 999         void *slow_pkts[BOND_MODE_8023AX_SLAVE_TX_PKTS] = { NULL };
1000
1001         /* Total amount of packets in slave_bufs */
1002         uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
1003         /* Slow packets placed in each slave */
1004         uint8_t slave_slow_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
1005
1006         bd_tx_q = (struct bond_tx_queue *)queue;
1007         internals = bd_tx_q->dev_private;
1008
1009         /* Copy slave list to protect against slave up/down changes during tx
1010          * bursting */
1011         num_of_slaves = internals->active_slave_count;
1012         if (num_of_slaves < 1)
1013                 return num_tx_total;
1014
1015         memcpy(slaves, internals->active_slaves, sizeof(slaves[0]) * num_of_slaves);
1016
1017         distributing_count = 0;
1018         for (i = 0; i < num_of_slaves; i++) {
1019                 struct port *port = &mode_8023ad_ports[slaves[i]];
1020
1021                 slave_slow_nb_pkts[i] = rte_ring_dequeue_burst(port->tx_ring,
1022                                 slow_pkts, BOND_MODE_8023AX_SLAVE_TX_PKTS,
1023                                 NULL);
1024                 slave_nb_pkts[i] = slave_slow_nb_pkts[i];
1025
1026                 for (j = 0; j < slave_slow_nb_pkts[i]; j++)
1027                         slave_bufs[i][j] = slow_pkts[j];
1028
1029                 if (ACTOR_STATE(port, DISTRIBUTING))
1030                         distributing_offsets[distributing_count++] = i;
1031         }
1032
1033         if (likely(distributing_count > 0)) {
1034                 /* Populate slaves mbuf with the packets which are to be sent on it */
1035                 for (i = 0; i < nb_pkts; i++) {
1036                         /* Select output slave using hash based on xmit policy */
1037                         op_slave_idx = internals->xmit_hash(bufs[i], distributing_count);
1038
1039                         /* Populate slave mbuf arrays with mbufs for that slave. Use only
1040                          * slaves that are currently distributing. */
1041                         uint8_t slave_offset = distributing_offsets[op_slave_idx];
1042                         slave_bufs[slave_offset][slave_nb_pkts[slave_offset]] = bufs[i];
1043                         slave_nb_pkts[slave_offset]++;
1044                 }
1045         }
1046
1047         /* Send packet burst on each slave device */
1048         for (i = 0; i < num_of_slaves; i++) {
1049                 if (slave_nb_pkts[i] == 0)
1050                         continue;
1051
1052                 num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
1053                                 slave_bufs[i], slave_nb_pkts[i]);
1054
1055                 /* If tx burst fails drop slow packets */
1056                 for ( ; num_tx_slave < slave_slow_nb_pkts[i]; num_tx_slave++)
1057                         rte_pktmbuf_free(slave_bufs[i][num_tx_slave]);
1058
1059                 num_tx_total += num_tx_slave - slave_slow_nb_pkts[i];
1060                 num_tx_fail_total += slave_nb_pkts[i] - num_tx_slave;
1061
1062                 /* If tx burst fails move packets to end of bufs */
1063                 if (unlikely(num_tx_slave < slave_nb_pkts[i])) {
1064                         uint16_t j = nb_pkts - num_tx_fail_total;
1065                         for ( ; num_tx_slave < slave_nb_pkts[i]; j++, num_tx_slave++)
1066                                 bufs[j] = slave_bufs[i][num_tx_slave];
1067                 }
1068         }
1069
1070         return num_tx_total;
1071 }
1072
1073 static uint16_t
1074 bond_ethdev_tx_burst_broadcast(void *queue, struct rte_mbuf **bufs,
1075                 uint16_t nb_pkts)
1076 {
1077         struct bond_dev_private *internals;
1078         struct bond_tx_queue *bd_tx_q;
1079
1080         uint8_t tx_failed_flag = 0, num_of_slaves;
1081         uint8_t slaves[RTE_MAX_ETHPORTS];
1082
1083         uint16_t max_nb_of_tx_pkts = 0;
1084
1085         int slave_tx_total[RTE_MAX_ETHPORTS];
1086         int i, most_successful_tx_slave = -1;
1087
1088         bd_tx_q = (struct bond_tx_queue *)queue;
1089         internals = bd_tx_q->dev_private;
1090
1091         /* Copy slave list to protect against slave up/down changes during tx
1092          * bursting */
1093         num_of_slaves = internals->active_slave_count;
1094         memcpy(slaves, internals->active_slaves,
1095                         sizeof(internals->active_slaves[0]) * num_of_slaves);
1096
1097         if (num_of_slaves < 1)
1098                 return 0;
1099
1100         /* Increment reference count on mbufs */
1101         for (i = 0; i < nb_pkts; i++)
1102                 rte_mbuf_refcnt_update(bufs[i], num_of_slaves - 1);
1103
1104         /* Transmit burst on each active slave */
1105         for (i = 0; i < num_of_slaves; i++) {
1106                 slave_tx_total[i] = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
1107                                         bufs, nb_pkts);
1108
1109                 if (unlikely(slave_tx_total[i] < nb_pkts))
1110                         tx_failed_flag = 1;
1111
1112                 /* record the value and slave index for the slave which transmits the
1113                  * maximum number of packets */
1114                 if (slave_tx_total[i] > max_nb_of_tx_pkts) {
1115                         max_nb_of_tx_pkts = slave_tx_total[i];
1116                         most_successful_tx_slave = i;
1117                 }
1118         }
1119
1120         /* if slaves fail to transmit packets from burst, the calling application
1121          * is not expected to know about multiple references to packets so we must
1122          * handle failures of all packets except those of the most successful slave
1123          */
1124         if (unlikely(tx_failed_flag))
1125                 for (i = 0; i < num_of_slaves; i++)
1126                         if (i != most_successful_tx_slave)
1127                                 while (slave_tx_total[i] < nb_pkts)
1128                                         rte_pktmbuf_free(bufs[slave_tx_total[i]++]);
1129
1130         return max_nb_of_tx_pkts;
1131 }
1132
1133 void
1134 link_properties_set(struct rte_eth_dev *bonded_eth_dev,
1135                 struct rte_eth_link *slave_dev_link)
1136 {
1137         struct rte_eth_link *bonded_dev_link = &bonded_eth_dev->data->dev_link;
1138         struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1139
1140         if (slave_dev_link->link_status &&
1141                 bonded_eth_dev->data->dev_started) {
1142                 bonded_dev_link->link_duplex = slave_dev_link->link_duplex;
1143                 bonded_dev_link->link_speed = slave_dev_link->link_speed;
1144
1145                 internals->link_props_set = 1;
1146         }
1147 }
1148
1149 void
1150 link_properties_reset(struct rte_eth_dev *bonded_eth_dev)
1151 {
1152         struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1153
1154         memset(&(bonded_eth_dev->data->dev_link), 0,
1155                         sizeof(bonded_eth_dev->data->dev_link));
1156
1157         internals->link_props_set = 0;
1158 }
1159
1160 int
1161 link_properties_valid(struct rte_eth_link *bonded_dev_link,
1162                 struct rte_eth_link *slave_dev_link)
1163 {
1164         if (bonded_dev_link->link_duplex != slave_dev_link->link_duplex ||
1165                 bonded_dev_link->link_speed !=  slave_dev_link->link_speed)
1166                 return -1;
1167
1168         return 0;
1169 }
1170
1171 int
1172 mac_address_get(struct rte_eth_dev *eth_dev, struct ether_addr *dst_mac_addr)
1173 {
1174         struct ether_addr *mac_addr;
1175
1176         if (eth_dev == NULL) {
1177                 RTE_LOG(ERR, PMD, "%s: NULL pointer eth_dev specified\n", __func__);
1178                 return -1;
1179         }
1180
1181         if (dst_mac_addr == NULL) {
1182                 RTE_LOG(ERR, PMD, "%s: NULL pointer MAC specified\n", __func__);
1183                 return -1;
1184         }
1185
1186         mac_addr = eth_dev->data->mac_addrs;
1187
1188         ether_addr_copy(mac_addr, dst_mac_addr);
1189         return 0;
1190 }
1191
1192 int
1193 mac_address_set(struct rte_eth_dev *eth_dev, struct ether_addr *new_mac_addr)
1194 {
1195         struct ether_addr *mac_addr;
1196
1197         if (eth_dev == NULL) {
1198                 RTE_BOND_LOG(ERR, "NULL pointer eth_dev specified");
1199                 return -1;
1200         }
1201
1202         if (new_mac_addr == NULL) {
1203                 RTE_BOND_LOG(ERR, "NULL pointer MAC specified");
1204                 return -1;
1205         }
1206
1207         mac_addr = eth_dev->data->mac_addrs;
1208
1209         /* If new MAC is different to current MAC then update */
1210         if (memcmp(mac_addr, new_mac_addr, sizeof(*mac_addr)) != 0)
1211                 memcpy(mac_addr, new_mac_addr, sizeof(*mac_addr));
1212
1213         return 0;
1214 }
1215
1216 int
1217 mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev)
1218 {
1219         struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1220         int i;
1221
1222         /* Update slave devices MAC addresses */
1223         if (internals->slave_count < 1)
1224                 return -1;
1225
1226         switch (internals->mode) {
1227         case BONDING_MODE_ROUND_ROBIN:
1228         case BONDING_MODE_BALANCE:
1229         case BONDING_MODE_BROADCAST:
1230                 for (i = 0; i < internals->slave_count; i++) {
1231                         if (mac_address_set(&rte_eth_devices[internals->slaves[i].port_id],
1232                                         bonded_eth_dev->data->mac_addrs)) {
1233                                 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1234                                                 internals->slaves[i].port_id);
1235                                 return -1;
1236                         }
1237                 }
1238                 break;
1239         case BONDING_MODE_8023AD:
1240                 bond_mode_8023ad_mac_address_update(bonded_eth_dev);
1241                 break;
1242         case BONDING_MODE_ACTIVE_BACKUP:
1243         case BONDING_MODE_TLB:
1244         case BONDING_MODE_ALB:
1245         default:
1246                 for (i = 0; i < internals->slave_count; i++) {
1247                         if (internals->slaves[i].port_id ==
1248                                         internals->current_primary_port) {
1249                                 if (mac_address_set(&rte_eth_devices[internals->primary_port],
1250                                                 bonded_eth_dev->data->mac_addrs)) {
1251                                         RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1252                                                         internals->current_primary_port);
1253                                         return -1;
1254                                 }
1255                         } else {
1256                                 if (mac_address_set(
1257                                                 &rte_eth_devices[internals->slaves[i].port_id],
1258                                                 &internals->slaves[i].persisted_mac_addr)) {
1259                                         RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1260                                                         internals->slaves[i].port_id);
1261                                         return -1;
1262                                 }
1263                         }
1264                 }
1265         }
1266
1267         return 0;
1268 }
1269
1270 int
1271 bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode)
1272 {
1273         struct bond_dev_private *internals;
1274
1275         internals = eth_dev->data->dev_private;
1276
1277         switch (mode) {
1278         case BONDING_MODE_ROUND_ROBIN:
1279                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_round_robin;
1280                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1281                 break;
1282         case BONDING_MODE_ACTIVE_BACKUP:
1283                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_active_backup;
1284                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup;
1285                 break;
1286         case BONDING_MODE_BALANCE:
1287                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_balance;
1288                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1289                 break;
1290         case BONDING_MODE_BROADCAST:
1291                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_broadcast;
1292                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1293                 break;
1294         case BONDING_MODE_8023AD:
1295                 if (bond_mode_8023ad_enable(eth_dev) != 0)
1296                         return -1;
1297
1298                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_8023ad;
1299                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_8023ad;
1300                 RTE_LOG(WARNING, PMD,
1301                                 "Using mode 4, it is necessary to do TX burst and RX burst "
1302                                 "at least every 100ms.\n");
1303                 break;
1304         case BONDING_MODE_TLB:
1305                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_tlb;
1306                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup;
1307                 break;
1308         case BONDING_MODE_ALB:
1309                 if (bond_mode_alb_enable(eth_dev) != 0)
1310                         return -1;
1311
1312                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_alb;
1313                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_alb;
1314                 break;
1315         default:
1316                 return -1;
1317         }
1318
1319         internals->mode = mode;
1320
1321         return 0;
1322 }
1323
1324 int
1325 slave_configure(struct rte_eth_dev *bonded_eth_dev,
1326                 struct rte_eth_dev *slave_eth_dev)
1327 {
1328         struct bond_rx_queue *bd_rx_q;
1329         struct bond_tx_queue *bd_tx_q;
1330
1331         int errval;
1332         uint16_t q_id;
1333
1334         /* Stop slave */
1335         rte_eth_dev_stop(slave_eth_dev->data->port_id);
1336
1337         /* Enable interrupts on slave device if supported */
1338         if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1339                 slave_eth_dev->data->dev_conf.intr_conf.lsc = 1;
1340
1341         /* If RSS is enabled for bonding, try to enable it for slaves  */
1342         if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
1343                 if (bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len
1344                                 != 0) {
1345                         slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len =
1346                                         bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len;
1347                         slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key =
1348                                         bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key;
1349                 } else {
1350                         slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1351                 }
1352
1353                 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf =
1354                                 bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
1355                 slave_eth_dev->data->dev_conf.rxmode.mq_mode =
1356                                 bonded_eth_dev->data->dev_conf.rxmode.mq_mode;
1357         }
1358
1359         slave_eth_dev->data->dev_conf.rxmode.hw_vlan_filter =
1360                         bonded_eth_dev->data->dev_conf.rxmode.hw_vlan_filter;
1361
1362         /* Configure device */
1363         errval = rte_eth_dev_configure(slave_eth_dev->data->port_id,
1364                         bonded_eth_dev->data->nb_rx_queues,
1365                         bonded_eth_dev->data->nb_tx_queues,
1366                         &(slave_eth_dev->data->dev_conf));
1367         if (errval != 0) {
1368                 RTE_BOND_LOG(ERR, "Cannot configure slave device: port %u , err (%d)",
1369                                 slave_eth_dev->data->port_id, errval);
1370                 return errval;
1371         }
1372
1373         /* Setup Rx Queues */
1374         for (q_id = 0; q_id < bonded_eth_dev->data->nb_rx_queues; q_id++) {
1375                 bd_rx_q = (struct bond_rx_queue *)bonded_eth_dev->data->rx_queues[q_id];
1376
1377                 errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id, q_id,
1378                                 bd_rx_q->nb_rx_desc,
1379                                 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1380                                 &(bd_rx_q->rx_conf), bd_rx_q->mb_pool);
1381                 if (errval != 0) {
1382                         RTE_BOND_LOG(ERR,
1383                                         "rte_eth_rx_queue_setup: port=%d queue_id %d, err (%d)",
1384                                         slave_eth_dev->data->port_id, q_id, errval);
1385                         return errval;
1386                 }
1387         }
1388
1389         /* Setup Tx Queues */
1390         for (q_id = 0; q_id < bonded_eth_dev->data->nb_tx_queues; q_id++) {
1391                 bd_tx_q = (struct bond_tx_queue *)bonded_eth_dev->data->tx_queues[q_id];
1392
1393                 errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id, q_id,
1394                                 bd_tx_q->nb_tx_desc,
1395                                 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1396                                 &bd_tx_q->tx_conf);
1397                 if (errval != 0) {
1398                         RTE_BOND_LOG(ERR,
1399                                         "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)",
1400                                         slave_eth_dev->data->port_id, q_id, errval);
1401                         return errval;
1402                 }
1403         }
1404
1405         /* Start device */
1406         errval = rte_eth_dev_start(slave_eth_dev->data->port_id);
1407         if (errval != 0) {
1408                 RTE_BOND_LOG(ERR, "rte_eth_dev_start: port=%u, err (%d)",
1409                                 slave_eth_dev->data->port_id, errval);
1410                 return -1;
1411         }
1412
1413         /* If RSS is enabled for bonding, synchronize RETA */
1414         if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) {
1415                 int i;
1416                 struct bond_dev_private *internals;
1417
1418                 internals = bonded_eth_dev->data->dev_private;
1419
1420                 for (i = 0; i < internals->slave_count; i++) {
1421                         if (internals->slaves[i].port_id == slave_eth_dev->data->port_id) {
1422                                 errval = rte_eth_dev_rss_reta_update(
1423                                                 slave_eth_dev->data->port_id,
1424                                                 &internals->reta_conf[0],
1425                                                 internals->slaves[i].reta_size);
1426                                 if (errval != 0) {
1427                                         RTE_LOG(WARNING, PMD,
1428                                                         "rte_eth_dev_rss_reta_update on slave port %d fails (err %d)."
1429                                                         " RSS Configuration for bonding may be inconsistent.\n",
1430                                                         slave_eth_dev->data->port_id, errval);
1431                                 }
1432                                 break;
1433                         }
1434                 }
1435         }
1436
1437         /* If lsc interrupt is set, check initial slave's link status */
1438         if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
1439                 slave_eth_dev->dev_ops->link_update(slave_eth_dev, 0);
1440                 bond_ethdev_lsc_event_callback(slave_eth_dev->data->port_id,
1441                         RTE_ETH_EVENT_INTR_LSC, &bonded_eth_dev->data->port_id);
1442         }
1443
1444         return 0;
1445 }
1446
1447 void
1448 slave_remove(struct bond_dev_private *internals,
1449                 struct rte_eth_dev *slave_eth_dev)
1450 {
1451         uint8_t i;
1452
1453         for (i = 0; i < internals->slave_count; i++)
1454                 if (internals->slaves[i].port_id ==
1455                                 slave_eth_dev->data->port_id)
1456                         break;
1457
1458         if (i < (internals->slave_count - 1))
1459                 memmove(&internals->slaves[i], &internals->slaves[i + 1],
1460                                 sizeof(internals->slaves[0]) *
1461                                 (internals->slave_count - i - 1));
1462
1463         internals->slave_count--;
1464
1465         /* force reconfiguration of slave interfaces */
1466         _rte_eth_dev_reset(slave_eth_dev);
1467 }
1468
1469 static void
1470 bond_ethdev_slave_link_status_change_monitor(void *cb_arg);
1471
1472 void
1473 slave_add(struct bond_dev_private *internals,
1474                 struct rte_eth_dev *slave_eth_dev)
1475 {
1476         struct bond_slave_details *slave_details =
1477                         &internals->slaves[internals->slave_count];
1478
1479         slave_details->port_id = slave_eth_dev->data->port_id;
1480         slave_details->last_link_status = 0;
1481
1482         /* Mark slave devices that don't support interrupts so we can
1483          * compensate when we start the bond
1484          */
1485         if (!(slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
1486                 slave_details->link_status_poll_enabled = 1;
1487         }
1488
1489         slave_details->link_status_wait_to_complete = 0;
1490         /* clean tlb_last_obytes when adding port for bonding device */
1491         memcpy(&(slave_details->persisted_mac_addr), slave_eth_dev->data->mac_addrs,
1492                         sizeof(struct ether_addr));
1493 }
1494
1495 void
1496 bond_ethdev_primary_set(struct bond_dev_private *internals,
1497                 uint8_t slave_port_id)
1498 {
1499         int i;
1500
1501         if (internals->active_slave_count < 1)
1502                 internals->current_primary_port = slave_port_id;
1503         else
1504                 /* Search bonded device slave ports for new proposed primary port */
1505                 for (i = 0; i < internals->active_slave_count; i++) {
1506                         if (internals->active_slaves[i] == slave_port_id)
1507                                 internals->current_primary_port = slave_port_id;
1508                 }
1509 }
1510
1511 static void
1512 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev);
1513
1514 static int
1515 bond_ethdev_start(struct rte_eth_dev *eth_dev)
1516 {
1517         struct bond_dev_private *internals;
1518         int i;
1519
1520         /* slave eth dev will be started by bonded device */
1521         if (check_for_bonded_ethdev(eth_dev)) {
1522                 RTE_BOND_LOG(ERR, "User tried to explicitly start a slave eth_dev (%d)",
1523                                 eth_dev->data->port_id);
1524                 return -1;
1525         }
1526
1527         eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
1528         eth_dev->data->dev_started = 1;
1529
1530         internals = eth_dev->data->dev_private;
1531
1532         if (internals->slave_count == 0) {
1533                 RTE_BOND_LOG(ERR, "Cannot start port since there are no slave devices");
1534                 return -1;
1535         }
1536
1537         if (internals->user_defined_mac == 0) {
1538                 struct ether_addr *new_mac_addr = NULL;
1539
1540                 for (i = 0; i < internals->slave_count; i++)
1541                         if (internals->slaves[i].port_id == internals->primary_port)
1542                                 new_mac_addr = &internals->slaves[i].persisted_mac_addr;
1543
1544                 if (new_mac_addr == NULL)
1545                         return -1;
1546
1547                 if (mac_address_set(eth_dev, new_mac_addr) != 0) {
1548                         RTE_BOND_LOG(ERR, "bonded port (%d) failed to update MAC address",
1549                                         eth_dev->data->port_id);
1550                         return -1;
1551                 }
1552         }
1553
1554         /* Update all slave devices MACs*/
1555         if (mac_address_slaves_update(eth_dev) != 0)
1556                 return -1;
1557
1558         /* If bonded device is configure in promiscuous mode then re-apply config */
1559         if (internals->promiscuous_en)
1560                 bond_ethdev_promiscuous_enable(eth_dev);
1561
1562         /* Reconfigure each slave device if starting bonded device */
1563         for (i = 0; i < internals->slave_count; i++) {
1564                 if (slave_configure(eth_dev,
1565                                 &(rte_eth_devices[internals->slaves[i].port_id])) != 0) {
1566                         RTE_BOND_LOG(ERR,
1567                                         "bonded port (%d) failed to reconfigure slave device (%d)",
1568                                         eth_dev->data->port_id, internals->slaves[i].port_id);
1569                         return -1;
1570                 }
1571                 /* We will need to poll for link status if any slave doesn't
1572                  * support interrupts
1573                  */
1574                 if (internals->slaves[i].link_status_poll_enabled)
1575                         internals->link_status_polling_enabled = 1;
1576         }
1577         /* start polling if needed */
1578         if (internals->link_status_polling_enabled) {
1579                 rte_eal_alarm_set(
1580                         internals->link_status_polling_interval_ms * 1000,
1581                         bond_ethdev_slave_link_status_change_monitor,
1582                         (void *)&rte_eth_devices[internals->port_id]);
1583         }
1584
1585         if (internals->user_defined_primary_port)
1586                 bond_ethdev_primary_set(internals, internals->primary_port);
1587
1588         if (internals->mode == BONDING_MODE_8023AD)
1589                 bond_mode_8023ad_start(eth_dev);
1590
1591         if (internals->mode == BONDING_MODE_TLB ||
1592                         internals->mode == BONDING_MODE_ALB)
1593                 bond_tlb_enable(internals);
1594
1595         return 0;
1596 }
1597
1598 static void
1599 bond_ethdev_free_queues(struct rte_eth_dev *dev)
1600 {
1601         uint8_t i;
1602
1603         if (dev->data->rx_queues != NULL) {
1604                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1605                         rte_free(dev->data->rx_queues[i]);
1606                         dev->data->rx_queues[i] = NULL;
1607                 }
1608                 dev->data->nb_rx_queues = 0;
1609         }
1610
1611         if (dev->data->tx_queues != NULL) {
1612                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1613                         rte_free(dev->data->tx_queues[i]);
1614                         dev->data->tx_queues[i] = NULL;
1615                 }
1616                 dev->data->nb_tx_queues = 0;
1617         }
1618 }
1619
1620 void
1621 bond_ethdev_stop(struct rte_eth_dev *eth_dev)
1622 {
1623         struct bond_dev_private *internals = eth_dev->data->dev_private;
1624         uint8_t i;
1625
1626         if (internals->mode == BONDING_MODE_8023AD) {
1627                 struct port *port;
1628                 void *pkt = NULL;
1629
1630                 bond_mode_8023ad_stop(eth_dev);
1631
1632                 /* Discard all messages to/from mode 4 state machines */
1633                 for (i = 0; i < internals->active_slave_count; i++) {
1634                         port = &mode_8023ad_ports[internals->active_slaves[i]];
1635
1636                         RTE_ASSERT(port->rx_ring != NULL);
1637                         while (rte_ring_dequeue(port->rx_ring, &pkt) != -ENOENT)
1638                                 rte_pktmbuf_free(pkt);
1639
1640                         RTE_ASSERT(port->tx_ring != NULL);
1641                         while (rte_ring_dequeue(port->tx_ring, &pkt) != -ENOENT)
1642                                 rte_pktmbuf_free(pkt);
1643                 }
1644         }
1645
1646         if (internals->mode == BONDING_MODE_TLB ||
1647                         internals->mode == BONDING_MODE_ALB) {
1648                 bond_tlb_disable(internals);
1649                 for (i = 0; i < internals->active_slave_count; i++)
1650                         tlb_last_obytets[internals->active_slaves[i]] = 0;
1651         }
1652
1653         internals->active_slave_count = 0;
1654         internals->link_status_polling_enabled = 0;
1655         for (i = 0; i < internals->slave_count; i++)
1656                 internals->slaves[i].last_link_status = 0;
1657
1658         eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
1659         eth_dev->data->dev_started = 0;
1660 }
1661
1662 void
1663 bond_ethdev_close(struct rte_eth_dev *dev)
1664 {
1665         struct bond_dev_private *internals = dev->data->dev_private;
1666         uint8_t bond_port_id = internals->port_id;
1667         int skipped = 0;
1668
1669         RTE_LOG(INFO, EAL, "Closing bonded device %s\n", dev->data->name);
1670         while (internals->slave_count != skipped) {
1671                 uint8_t port_id = internals->slaves[skipped].port_id;
1672
1673                 rte_eth_dev_stop(port_id);
1674
1675                 if (rte_eth_bond_slave_remove(bond_port_id, port_id) != 0) {
1676                         RTE_LOG(ERR, EAL,
1677                                 "Failed to remove port %d from bonded device "
1678                                 "%s\n", port_id, dev->data->name);
1679                         skipped++;
1680                 }
1681         }
1682         bond_ethdev_free_queues(dev);
1683         rte_bitmap_reset(internals->vlan_filter_bmp);
1684 }
1685
1686 /* forward declaration */
1687 static int bond_ethdev_configure(struct rte_eth_dev *dev);
1688
1689 static void
1690 bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1691 {
1692         struct bond_dev_private *internals = dev->data->dev_private;
1693         uint16_t max_nb_rx_queues = UINT16_MAX;
1694         uint16_t max_nb_tx_queues = UINT16_MAX;
1695
1696         dev_info->max_mac_addrs = 1;
1697
1698         dev_info->max_rx_pktlen = internals->candidate_max_rx_pktlen
1699                                   ? internals->candidate_max_rx_pktlen
1700                                   : ETHER_MAX_JUMBO_FRAME_LEN;
1701
1702         if (internals->slave_count > 0) {
1703                 /* Max number of tx/rx queues that the bonded device can
1704                  * support is the minimum values of the bonded slaves, as
1705                  * all slaves must be capable of supporting the same number
1706                  * of tx/rx queues.
1707                  */
1708                 struct rte_eth_dev_info slave_info;
1709                 uint8_t idx;
1710
1711                 for (idx = 0; idx < internals->slave_count; idx++) {
1712                         rte_eth_dev_info_get(internals->slaves[idx].port_id,
1713                                         &slave_info);
1714
1715                         if (slave_info.max_rx_queues < max_nb_rx_queues)
1716                                 max_nb_rx_queues = slave_info.max_rx_queues;
1717
1718                         if (slave_info.max_tx_queues < max_nb_tx_queues)
1719                                 max_nb_tx_queues = slave_info.max_tx_queues;
1720                 }
1721         }
1722
1723         dev_info->max_rx_queues = max_nb_rx_queues;
1724         dev_info->max_tx_queues = max_nb_tx_queues;
1725
1726         dev_info->min_rx_bufsize = 0;
1727
1728         dev_info->rx_offload_capa = internals->rx_offload_capa;
1729         dev_info->tx_offload_capa = internals->tx_offload_capa;
1730         dev_info->flow_type_rss_offloads = internals->flow_type_rss_offloads;
1731
1732         dev_info->reta_size = internals->reta_size;
1733 }
1734
1735 static int
1736 bond_ethdev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1737 {
1738         int res;
1739         uint8_t i;
1740         struct bond_dev_private *internals = dev->data->dev_private;
1741
1742         /* don't do this while a slave is being added */
1743         rte_spinlock_lock(&internals->lock);
1744
1745         if (on)
1746                 rte_bitmap_set(internals->vlan_filter_bmp, vlan_id);
1747         else
1748                 rte_bitmap_clear(internals->vlan_filter_bmp, vlan_id);
1749
1750         for (i = 0; i < internals->slave_count; i++) {
1751                 uint8_t port_id = internals->slaves[i].port_id;
1752
1753                 res = rte_eth_dev_vlan_filter(port_id, vlan_id, on);
1754                 if (res == ENOTSUP)
1755                         RTE_LOG(WARNING, PMD,
1756                                 "Setting VLAN filter on slave port %u not supported.\n",
1757                                 port_id);
1758         }
1759
1760         rte_spinlock_unlock(&internals->lock);
1761         return 0;
1762 }
1763
1764 static int
1765 bond_ethdev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1766                 uint16_t nb_rx_desc, unsigned int socket_id __rte_unused,
1767                 const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mb_pool)
1768 {
1769         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)
1770                         rte_zmalloc_socket(NULL, sizeof(struct bond_rx_queue),
1771                                         0, dev->data->numa_node);
1772         if (bd_rx_q == NULL)
1773                 return -1;
1774
1775         bd_rx_q->queue_id = rx_queue_id;
1776         bd_rx_q->dev_private = dev->data->dev_private;
1777
1778         bd_rx_q->nb_rx_desc = nb_rx_desc;
1779
1780         memcpy(&(bd_rx_q->rx_conf), rx_conf, sizeof(struct rte_eth_rxconf));
1781         bd_rx_q->mb_pool = mb_pool;
1782
1783         dev->data->rx_queues[rx_queue_id] = bd_rx_q;
1784
1785         return 0;
1786 }
1787
1788 static int
1789 bond_ethdev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1790                 uint16_t nb_tx_desc, unsigned int socket_id __rte_unused,
1791                 const struct rte_eth_txconf *tx_conf)
1792 {
1793         struct bond_tx_queue *bd_tx_q  = (struct bond_tx_queue *)
1794                         rte_zmalloc_socket(NULL, sizeof(struct bond_tx_queue),
1795                                         0, dev->data->numa_node);
1796
1797         if (bd_tx_q == NULL)
1798                 return -1;
1799
1800         bd_tx_q->queue_id = tx_queue_id;
1801         bd_tx_q->dev_private = dev->data->dev_private;
1802
1803         bd_tx_q->nb_tx_desc = nb_tx_desc;
1804         memcpy(&(bd_tx_q->tx_conf), tx_conf, sizeof(bd_tx_q->tx_conf));
1805
1806         dev->data->tx_queues[tx_queue_id] = bd_tx_q;
1807
1808         return 0;
1809 }
1810
1811 static void
1812 bond_ethdev_rx_queue_release(void *queue)
1813 {
1814         if (queue == NULL)
1815                 return;
1816
1817         rte_free(queue);
1818 }
1819
1820 static void
1821 bond_ethdev_tx_queue_release(void *queue)
1822 {
1823         if (queue == NULL)
1824                 return;
1825
1826         rte_free(queue);
1827 }
1828
1829 static void
1830 bond_ethdev_slave_link_status_change_monitor(void *cb_arg)
1831 {
1832         struct rte_eth_dev *bonded_ethdev, *slave_ethdev;
1833         struct bond_dev_private *internals;
1834
1835         /* Default value for polling slave found is true as we don't want to
1836          * disable the polling thread if we cannot get the lock */
1837         int i, polling_slave_found = 1;
1838
1839         if (cb_arg == NULL)
1840                 return;
1841
1842         bonded_ethdev = (struct rte_eth_dev *)cb_arg;
1843         internals = (struct bond_dev_private *)bonded_ethdev->data->dev_private;
1844
1845         if (!bonded_ethdev->data->dev_started ||
1846                 !internals->link_status_polling_enabled)
1847                 return;
1848
1849         /* If device is currently being configured then don't check slaves link
1850          * status, wait until next period */
1851         if (rte_spinlock_trylock(&internals->lock)) {
1852                 if (internals->slave_count > 0)
1853                         polling_slave_found = 0;
1854
1855                 for (i = 0; i < internals->slave_count; i++) {
1856                         if (!internals->slaves[i].link_status_poll_enabled)
1857                                 continue;
1858
1859                         slave_ethdev = &rte_eth_devices[internals->slaves[i].port_id];
1860                         polling_slave_found = 1;
1861
1862                         /* Update slave link status */
1863                         (*slave_ethdev->dev_ops->link_update)(slave_ethdev,
1864                                         internals->slaves[i].link_status_wait_to_complete);
1865
1866                         /* if link status has changed since last checked then call lsc
1867                          * event callback */
1868                         if (slave_ethdev->data->dev_link.link_status !=
1869                                         internals->slaves[i].last_link_status) {
1870                                 internals->slaves[i].last_link_status =
1871                                                 slave_ethdev->data->dev_link.link_status;
1872
1873                                 bond_ethdev_lsc_event_callback(internals->slaves[i].port_id,
1874                                                 RTE_ETH_EVENT_INTR_LSC,
1875                                                 &bonded_ethdev->data->port_id);
1876                         }
1877                 }
1878                 rte_spinlock_unlock(&internals->lock);
1879         }
1880
1881         if (polling_slave_found)
1882                 /* Set alarm to continue monitoring link status of slave ethdev's */
1883                 rte_eal_alarm_set(internals->link_status_polling_interval_ms * 1000,
1884                                 bond_ethdev_slave_link_status_change_monitor, cb_arg);
1885 }
1886
1887 static int
1888 bond_ethdev_link_update(struct rte_eth_dev *bonded_eth_dev,
1889                 int wait_to_complete)
1890 {
1891         struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1892
1893         if (!bonded_eth_dev->data->dev_started ||
1894                 internals->active_slave_count == 0) {
1895                 bonded_eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
1896                 return 0;
1897         } else {
1898                 struct rte_eth_dev *slave_eth_dev;
1899                 int i, link_up = 0;
1900
1901                 for (i = 0; i < internals->active_slave_count; i++) {
1902                         slave_eth_dev = &rte_eth_devices[internals->active_slaves[i]];
1903
1904                         (*slave_eth_dev->dev_ops->link_update)(slave_eth_dev,
1905                                         wait_to_complete);
1906                         if (slave_eth_dev->data->dev_link.link_status == ETH_LINK_UP) {
1907                                 link_up = 1;
1908                                 break;
1909                         }
1910                 }
1911
1912                 bonded_eth_dev->data->dev_link.link_status = link_up;
1913         }
1914
1915         return 0;
1916 }
1917
1918 static void
1919 bond_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1920 {
1921         struct bond_dev_private *internals = dev->data->dev_private;
1922         struct rte_eth_stats slave_stats;
1923         int i, j;
1924
1925         for (i = 0; i < internals->slave_count; i++) {
1926                 rte_eth_stats_get(internals->slaves[i].port_id, &slave_stats);
1927
1928                 stats->ipackets += slave_stats.ipackets;
1929                 stats->opackets += slave_stats.opackets;
1930                 stats->ibytes += slave_stats.ibytes;
1931                 stats->obytes += slave_stats.obytes;
1932                 stats->imissed += slave_stats.imissed;
1933                 stats->ierrors += slave_stats.ierrors;
1934                 stats->oerrors += slave_stats.oerrors;
1935                 stats->rx_nombuf += slave_stats.rx_nombuf;
1936
1937                 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1938                         stats->q_ipackets[j] += slave_stats.q_ipackets[j];
1939                         stats->q_opackets[j] += slave_stats.q_opackets[j];
1940                         stats->q_ibytes[j] += slave_stats.q_ibytes[j];
1941                         stats->q_obytes[j] += slave_stats.q_obytes[j];
1942                         stats->q_errors[j] += slave_stats.q_errors[j];
1943                 }
1944
1945         }
1946 }
1947
1948 static void
1949 bond_ethdev_stats_reset(struct rte_eth_dev *dev)
1950 {
1951         struct bond_dev_private *internals = dev->data->dev_private;
1952         int i;
1953
1954         for (i = 0; i < internals->slave_count; i++)
1955                 rte_eth_stats_reset(internals->slaves[i].port_id);
1956 }
1957
1958 static void
1959 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev)
1960 {
1961         struct bond_dev_private *internals = eth_dev->data->dev_private;
1962         int i;
1963
1964         internals->promiscuous_en = 1;
1965
1966         switch (internals->mode) {
1967         /* Promiscuous mode is propagated to all slaves */
1968         case BONDING_MODE_ROUND_ROBIN:
1969         case BONDING_MODE_BALANCE:
1970         case BONDING_MODE_BROADCAST:
1971                 for (i = 0; i < internals->slave_count; i++)
1972                         rte_eth_promiscuous_enable(internals->slaves[i].port_id);
1973                 break;
1974         /* In mode4 promiscus mode is managed when slave is added/removed */
1975         case BONDING_MODE_8023AD:
1976                 break;
1977         /* Promiscuous mode is propagated only to primary slave */
1978         case BONDING_MODE_ACTIVE_BACKUP:
1979         case BONDING_MODE_TLB:
1980         case BONDING_MODE_ALB:
1981         default:
1982                 rte_eth_promiscuous_enable(internals->current_primary_port);
1983         }
1984 }
1985
1986 static void
1987 bond_ethdev_promiscuous_disable(struct rte_eth_dev *dev)
1988 {
1989         struct bond_dev_private *internals = dev->data->dev_private;
1990         int i;
1991
1992         internals->promiscuous_en = 0;
1993
1994         switch (internals->mode) {
1995         /* Promiscuous mode is propagated to all slaves */
1996         case BONDING_MODE_ROUND_ROBIN:
1997         case BONDING_MODE_BALANCE:
1998         case BONDING_MODE_BROADCAST:
1999                 for (i = 0; i < internals->slave_count; i++)
2000                         rte_eth_promiscuous_disable(internals->slaves[i].port_id);
2001                 break;
2002         /* In mode4 promiscus mode is set managed when slave is added/removed */
2003         case BONDING_MODE_8023AD:
2004                 break;
2005         /* Promiscuous mode is propagated only to primary slave */
2006         case BONDING_MODE_ACTIVE_BACKUP:
2007         case BONDING_MODE_TLB:
2008         case BONDING_MODE_ALB:
2009         default:
2010                 rte_eth_promiscuous_disable(internals->current_primary_port);
2011         }
2012 }
2013
2014 static void
2015 bond_ethdev_delayed_lsc_propagation(void *arg)
2016 {
2017         if (arg == NULL)
2018                 return;
2019
2020         _rte_eth_dev_callback_process((struct rte_eth_dev *)arg,
2021                         RTE_ETH_EVENT_INTR_LSC, NULL);
2022 }
2023
2024 void
2025 bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type,
2026                 void *param)
2027 {
2028         struct rte_eth_dev *bonded_eth_dev, *slave_eth_dev;
2029         struct bond_dev_private *internals;
2030         struct rte_eth_link link;
2031
2032         int i, valid_slave = 0;
2033         uint8_t active_pos;
2034         uint8_t lsc_flag = 0;
2035
2036         if (type != RTE_ETH_EVENT_INTR_LSC || param == NULL)
2037                 return;
2038
2039         bonded_eth_dev = &rte_eth_devices[*(uint8_t *)param];
2040         slave_eth_dev = &rte_eth_devices[port_id];
2041
2042         if (check_for_bonded_ethdev(bonded_eth_dev))
2043                 return;
2044
2045         internals = bonded_eth_dev->data->dev_private;
2046
2047         /* If the device isn't started don't handle interrupts */
2048         if (!bonded_eth_dev->data->dev_started)
2049                 return;
2050
2051         /* verify that port_id is a valid slave of bonded port */
2052         for (i = 0; i < internals->slave_count; i++) {
2053                 if (internals->slaves[i].port_id == port_id) {
2054                         valid_slave = 1;
2055                         break;
2056                 }
2057         }
2058
2059         if (!valid_slave)
2060                 return;
2061
2062         /* Search for port in active port list */
2063         active_pos = find_slave_by_id(internals->active_slaves,
2064                         internals->active_slave_count, port_id);
2065
2066         rte_eth_link_get_nowait(port_id, &link);
2067         if (link.link_status) {
2068                 if (active_pos < internals->active_slave_count)
2069                         return;
2070
2071                 /* if no active slave ports then set this port to be primary port */
2072                 if (internals->active_slave_count < 1) {
2073                         /* If first active slave, then change link status */
2074                         bonded_eth_dev->data->dev_link.link_status = ETH_LINK_UP;
2075                         internals->current_primary_port = port_id;
2076                         lsc_flag = 1;
2077
2078                         mac_address_slaves_update(bonded_eth_dev);
2079
2080                         /* Inherit eth dev link properties from first active slave */
2081                         link_properties_set(bonded_eth_dev,
2082                                         &(slave_eth_dev->data->dev_link));
2083                 } else {
2084                         if (link_properties_valid(
2085                                 &bonded_eth_dev->data->dev_link, &link) != 0) {
2086                                 slave_eth_dev->data->dev_flags &=
2087                                         (~RTE_ETH_DEV_BONDED_SLAVE);
2088                                 RTE_LOG(ERR, PMD,
2089                                         "port %u invalid speed/duplex\n",
2090                                         port_id);
2091                                 return;
2092                         }
2093                 }
2094
2095                 activate_slave(bonded_eth_dev, port_id);
2096
2097                 /* If user has defined the primary port then default to using it */
2098                 if (internals->user_defined_primary_port &&
2099                                 internals->primary_port == port_id)
2100                         bond_ethdev_primary_set(internals, port_id);
2101         } else {
2102                 if (active_pos == internals->active_slave_count)
2103                         return;
2104
2105                 /* Remove from active slave list */
2106                 deactivate_slave(bonded_eth_dev, port_id);
2107
2108                 /* No active slaves, change link status to down and reset other
2109                  * link properties */
2110                 if (internals->active_slave_count < 1) {
2111                         lsc_flag = 1;
2112                         bonded_eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
2113
2114                         link_properties_reset(bonded_eth_dev);
2115                 }
2116
2117                 /* Update primary id, take first active slave from list or if none
2118                  * available set to -1 */
2119                 if (port_id == internals->current_primary_port) {
2120                         if (internals->active_slave_count > 0)
2121                                 bond_ethdev_primary_set(internals,
2122                                                 internals->active_slaves[0]);
2123                         else
2124                                 internals->current_primary_port = internals->primary_port;
2125                 }
2126         }
2127
2128         if (lsc_flag) {
2129                 /* Cancel any possible outstanding interrupts if delays are enabled */
2130                 if (internals->link_up_delay_ms > 0 ||
2131                         internals->link_down_delay_ms > 0)
2132                         rte_eal_alarm_cancel(bond_ethdev_delayed_lsc_propagation,
2133                                         bonded_eth_dev);
2134
2135                 if (bonded_eth_dev->data->dev_link.link_status) {
2136                         if (internals->link_up_delay_ms > 0)
2137                                 rte_eal_alarm_set(internals->link_up_delay_ms * 1000,
2138                                                 bond_ethdev_delayed_lsc_propagation,
2139                                                 (void *)bonded_eth_dev);
2140                         else
2141                                 _rte_eth_dev_callback_process(bonded_eth_dev,
2142                                                 RTE_ETH_EVENT_INTR_LSC, NULL);
2143
2144                 } else {
2145                         if (internals->link_down_delay_ms > 0)
2146                                 rte_eal_alarm_set(internals->link_down_delay_ms * 1000,
2147                                                 bond_ethdev_delayed_lsc_propagation,
2148                                                 (void *)bonded_eth_dev);
2149                         else
2150                                 _rte_eth_dev_callback_process(bonded_eth_dev,
2151                                                 RTE_ETH_EVENT_INTR_LSC, NULL);
2152                 }
2153         }
2154 }
2155
2156 static int
2157 bond_ethdev_rss_reta_update(struct rte_eth_dev *dev,
2158                 struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size)
2159 {
2160         unsigned i, j;
2161         int result = 0;
2162         int slave_reta_size;
2163         unsigned reta_count;
2164         struct bond_dev_private *internals = dev->data->dev_private;
2165
2166         if (reta_size != internals->reta_size)
2167                 return -EINVAL;
2168
2169          /* Copy RETA table */
2170         reta_count = reta_size / RTE_RETA_GROUP_SIZE;
2171
2172         for (i = 0; i < reta_count; i++) {
2173                 internals->reta_conf[i].mask = reta_conf[i].mask;
2174                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
2175                         if ((reta_conf[i].mask >> j) & 0x01)
2176                                 internals->reta_conf[i].reta[j] = reta_conf[i].reta[j];
2177         }
2178
2179         /* Fill rest of array */
2180         for (; i < RTE_DIM(internals->reta_conf); i += reta_count)
2181                 memcpy(&internals->reta_conf[i], &internals->reta_conf[0],
2182                                 sizeof(internals->reta_conf[0]) * reta_count);
2183
2184         /* Propagate RETA over slaves */
2185         for (i = 0; i < internals->slave_count; i++) {
2186                 slave_reta_size = internals->slaves[i].reta_size;
2187                 result = rte_eth_dev_rss_reta_update(internals->slaves[i].port_id,
2188                                 &internals->reta_conf[0], slave_reta_size);
2189                 if (result < 0)
2190                         return result;
2191         }
2192
2193         return 0;
2194 }
2195
2196 static int
2197 bond_ethdev_rss_reta_query(struct rte_eth_dev *dev,
2198                 struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size)
2199 {
2200         int i, j;
2201         struct bond_dev_private *internals = dev->data->dev_private;
2202
2203         if (reta_size != internals->reta_size)
2204                 return -EINVAL;
2205
2206          /* Copy RETA table */
2207         for (i = 0; i < reta_size / RTE_RETA_GROUP_SIZE; i++)
2208                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
2209                         if ((reta_conf[i].mask >> j) & 0x01)
2210                                 reta_conf[i].reta[j] = internals->reta_conf[i].reta[j];
2211
2212         return 0;
2213 }
2214
2215 static int
2216 bond_ethdev_rss_hash_update(struct rte_eth_dev *dev,
2217                 struct rte_eth_rss_conf *rss_conf)
2218 {
2219         int i, result = 0;
2220         struct bond_dev_private *internals = dev->data->dev_private;
2221         struct rte_eth_rss_conf bond_rss_conf;
2222
2223         memcpy(&bond_rss_conf, rss_conf, sizeof(struct rte_eth_rss_conf));
2224
2225         bond_rss_conf.rss_hf &= internals->flow_type_rss_offloads;
2226
2227         if (bond_rss_conf.rss_hf != 0)
2228                 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf = bond_rss_conf.rss_hf;
2229
2230         if (bond_rss_conf.rss_key && bond_rss_conf.rss_key_len <
2231                         sizeof(internals->rss_key)) {
2232                 if (bond_rss_conf.rss_key_len == 0)
2233                         bond_rss_conf.rss_key_len = 40;
2234                 internals->rss_key_len = bond_rss_conf.rss_key_len;
2235                 memcpy(internals->rss_key, bond_rss_conf.rss_key,
2236                                 internals->rss_key_len);
2237         }
2238
2239         for (i = 0; i < internals->slave_count; i++) {
2240                 result = rte_eth_dev_rss_hash_update(internals->slaves[i].port_id,
2241                                 &bond_rss_conf);
2242                 if (result < 0)
2243                         return result;
2244         }
2245
2246         return 0;
2247 }
2248
2249 static int
2250 bond_ethdev_rss_hash_conf_get(struct rte_eth_dev *dev,
2251                 struct rte_eth_rss_conf *rss_conf)
2252 {
2253         struct bond_dev_private *internals = dev->data->dev_private;
2254
2255         rss_conf->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
2256         rss_conf->rss_key_len = internals->rss_key_len;
2257         if (rss_conf->rss_key)
2258                 memcpy(rss_conf->rss_key, internals->rss_key, internals->rss_key_len);
2259
2260         return 0;
2261 }
2262
2263 const struct eth_dev_ops default_dev_ops = {
2264         .dev_start            = bond_ethdev_start,
2265         .dev_stop             = bond_ethdev_stop,
2266         .dev_close            = bond_ethdev_close,
2267         .dev_configure        = bond_ethdev_configure,
2268         .dev_infos_get        = bond_ethdev_info,
2269         .vlan_filter_set      = bond_ethdev_vlan_filter_set,
2270         .rx_queue_setup       = bond_ethdev_rx_queue_setup,
2271         .tx_queue_setup       = bond_ethdev_tx_queue_setup,
2272         .rx_queue_release     = bond_ethdev_rx_queue_release,
2273         .tx_queue_release     = bond_ethdev_tx_queue_release,
2274         .link_update          = bond_ethdev_link_update,
2275         .stats_get            = bond_ethdev_stats_get,
2276         .stats_reset          = bond_ethdev_stats_reset,
2277         .promiscuous_enable   = bond_ethdev_promiscuous_enable,
2278         .promiscuous_disable  = bond_ethdev_promiscuous_disable,
2279         .reta_update          = bond_ethdev_rss_reta_update,
2280         .reta_query           = bond_ethdev_rss_reta_query,
2281         .rss_hash_update      = bond_ethdev_rss_hash_update,
2282         .rss_hash_conf_get    = bond_ethdev_rss_hash_conf_get
2283 };
2284
2285 static int
2286 bond_alloc(struct rte_vdev_device *dev, uint8_t mode)
2287 {
2288         const char *name = rte_vdev_device_name(dev);
2289         uint8_t socket_id = dev->device.numa_node;
2290         struct bond_dev_private *internals = NULL;
2291         struct rte_eth_dev *eth_dev = NULL;
2292         uint32_t vlan_filter_bmp_size;
2293
2294         /* now do all data allocation - for eth_dev structure, dummy pci driver
2295          * and internal (private) data
2296          */
2297
2298         if (socket_id >= number_of_sockets()) {
2299                 RTE_BOND_LOG(ERR,
2300                                 "Invalid socket id specified to create bonded device on.");
2301                 goto err;
2302         }
2303
2304         /* reserve an ethdev entry */
2305         eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internals));
2306         if (eth_dev == NULL) {
2307                 RTE_BOND_LOG(ERR, "Unable to allocate rte_eth_dev");
2308                 goto err;
2309         }
2310
2311         internals = eth_dev->data->dev_private;
2312         eth_dev->data->nb_rx_queues = (uint16_t)1;
2313         eth_dev->data->nb_tx_queues = (uint16_t)1;
2314
2315         eth_dev->data->mac_addrs = rte_zmalloc_socket(name, ETHER_ADDR_LEN, 0,
2316                         socket_id);
2317         if (eth_dev->data->mac_addrs == NULL) {
2318                 RTE_BOND_LOG(ERR, "Unable to malloc mac_addrs");
2319                 goto err;
2320         }
2321
2322         eth_dev->dev_ops = &default_dev_ops;
2323         eth_dev->data->dev_flags = RTE_ETH_DEV_INTR_LSC |
2324                 RTE_ETH_DEV_DETACHABLE;
2325
2326         rte_spinlock_init(&internals->lock);
2327
2328         internals->port_id = eth_dev->data->port_id;
2329         internals->mode = BONDING_MODE_INVALID;
2330         internals->current_primary_port = RTE_MAX_ETHPORTS + 1;
2331         internals->balance_xmit_policy = BALANCE_XMIT_POLICY_LAYER2;
2332         internals->xmit_hash = xmit_l2_hash;
2333         internals->user_defined_mac = 0;
2334         internals->link_props_set = 0;
2335
2336         internals->link_status_polling_enabled = 0;
2337
2338         internals->link_status_polling_interval_ms =
2339                 DEFAULT_POLLING_INTERVAL_10_MS;
2340         internals->link_down_delay_ms = 0;
2341         internals->link_up_delay_ms = 0;
2342
2343         internals->slave_count = 0;
2344         internals->active_slave_count = 0;
2345         internals->rx_offload_capa = 0;
2346         internals->tx_offload_capa = 0;
2347         internals->candidate_max_rx_pktlen = 0;
2348         internals->max_rx_pktlen = 0;
2349
2350         /* Initially allow to choose any offload type */
2351         internals->flow_type_rss_offloads = ETH_RSS_PROTO_MASK;
2352
2353         memset(internals->active_slaves, 0, sizeof(internals->active_slaves));
2354         memset(internals->slaves, 0, sizeof(internals->slaves));
2355
2356         /* Set mode 4 default configuration */
2357         bond_mode_8023ad_setup(eth_dev, NULL);
2358         if (bond_ethdev_mode_set(eth_dev, mode)) {
2359                 RTE_BOND_LOG(ERR, "Failed to set bonded device %d mode too %d",
2360                                  eth_dev->data->port_id, mode);
2361                 goto err;
2362         }
2363
2364         vlan_filter_bmp_size =
2365                 rte_bitmap_get_memory_footprint(ETHER_MAX_VLAN_ID + 1);
2366         internals->vlan_filter_bmpmem = rte_malloc(name, vlan_filter_bmp_size,
2367                                                    RTE_CACHE_LINE_SIZE);
2368         if (internals->vlan_filter_bmpmem == NULL) {
2369                 RTE_BOND_LOG(ERR,
2370                              "Failed to allocate vlan bitmap for bonded device %u\n",
2371                              eth_dev->data->port_id);
2372                 goto err;
2373         }
2374
2375         internals->vlan_filter_bmp = rte_bitmap_init(ETHER_MAX_VLAN_ID + 1,
2376                         internals->vlan_filter_bmpmem, vlan_filter_bmp_size);
2377         if (internals->vlan_filter_bmp == NULL) {
2378                 RTE_BOND_LOG(ERR,
2379                              "Failed to init vlan bitmap for bonded device %u\n",
2380                              eth_dev->data->port_id);
2381                 rte_free(internals->vlan_filter_bmpmem);
2382                 goto err;
2383         }
2384
2385         return eth_dev->data->port_id;
2386
2387 err:
2388         rte_free(internals);
2389         if (eth_dev != NULL) {
2390                 rte_free(eth_dev->data->mac_addrs);
2391                 rte_eth_dev_release_port(eth_dev);
2392         }
2393         return -1;
2394 }
2395
2396 static int
2397 bond_probe(struct rte_vdev_device *dev)
2398 {
2399         const char *name;
2400         struct bond_dev_private *internals;
2401         struct rte_kvargs *kvlist;
2402         uint8_t bonding_mode, socket_id;
2403         int  arg_count, port_id;
2404
2405         if (!dev)
2406                 return -EINVAL;
2407
2408         name = rte_vdev_device_name(dev);
2409         RTE_LOG(INFO, EAL, "Initializing pmd_bond for %s\n", name);
2410
2411         kvlist = rte_kvargs_parse(rte_vdev_device_args(dev),
2412                 pmd_bond_init_valid_arguments);
2413         if (kvlist == NULL)
2414                 return -1;
2415
2416         /* Parse link bonding mode */
2417         if (rte_kvargs_count(kvlist, PMD_BOND_MODE_KVARG) == 1) {
2418                 if (rte_kvargs_process(kvlist, PMD_BOND_MODE_KVARG,
2419                                 &bond_ethdev_parse_slave_mode_kvarg,
2420                                 &bonding_mode) != 0) {
2421                         RTE_LOG(ERR, EAL, "Invalid mode for bonded device %s\n",
2422                                         name);
2423                         goto parse_error;
2424                 }
2425         } else {
2426                 RTE_LOG(ERR, EAL, "Mode must be specified only once for bonded "
2427                                 "device %s\n", name);
2428                 goto parse_error;
2429         }
2430
2431         /* Parse socket id to create bonding device on */
2432         arg_count = rte_kvargs_count(kvlist, PMD_BOND_SOCKET_ID_KVARG);
2433         if (arg_count == 1) {
2434                 if (rte_kvargs_process(kvlist, PMD_BOND_SOCKET_ID_KVARG,
2435                                 &bond_ethdev_parse_socket_id_kvarg, &socket_id)
2436                                 != 0) {
2437                         RTE_LOG(ERR, EAL, "Invalid socket Id specified for "
2438                                         "bonded device %s\n", name);
2439                         goto parse_error;
2440                 }
2441         } else if (arg_count > 1) {
2442                 RTE_LOG(ERR, EAL, "Socket Id can be specified only once for "
2443                                 "bonded device %s\n", name);
2444                 goto parse_error;
2445         } else {
2446                 socket_id = rte_socket_id();
2447         }
2448
2449         dev->device.numa_node = socket_id;
2450
2451         /* Create link bonding eth device */
2452         port_id = bond_alloc(dev, bonding_mode);
2453         if (port_id < 0) {
2454                 RTE_LOG(ERR, EAL, "Failed to create socket %s in mode %u on "
2455                                 "socket %u.\n", name, bonding_mode, socket_id);
2456                 goto parse_error;
2457         }
2458         internals = rte_eth_devices[port_id].data->dev_private;
2459         internals->kvlist = kvlist;
2460
2461         RTE_LOG(INFO, EAL, "Create bonded device %s on port %d in mode %u on "
2462                         "socket %u.\n", name, port_id, bonding_mode, socket_id);
2463         return 0;
2464
2465 parse_error:
2466         rte_kvargs_free(kvlist);
2467
2468         return -1;
2469 }
2470
2471 static int
2472 bond_remove(struct rte_vdev_device *dev)
2473 {
2474         struct rte_eth_dev *eth_dev;
2475         struct bond_dev_private *internals;
2476         const char *name;
2477
2478         if (!dev)
2479                 return -EINVAL;
2480
2481         name = rte_vdev_device_name(dev);
2482         RTE_LOG(INFO, EAL, "Uninitializing pmd_bond for %s\n", name);
2483
2484         /* now free all data allocation - for eth_dev structure,
2485          * dummy pci driver and internal (private) data
2486          */
2487
2488         /* find an ethdev entry */
2489         eth_dev = rte_eth_dev_allocated(name);
2490         if (eth_dev == NULL)
2491                 return -ENODEV;
2492
2493         RTE_ASSERT(eth_dev->device == &dev->device);
2494
2495         internals = eth_dev->data->dev_private;
2496         if (internals->slave_count != 0)
2497                 return -EBUSY;
2498
2499         if (eth_dev->data->dev_started == 1) {
2500                 bond_ethdev_stop(eth_dev);
2501                 bond_ethdev_close(eth_dev);
2502         }
2503
2504         eth_dev->dev_ops = NULL;
2505         eth_dev->rx_pkt_burst = NULL;
2506         eth_dev->tx_pkt_burst = NULL;
2507
2508         internals = eth_dev->data->dev_private;
2509         rte_bitmap_free(internals->vlan_filter_bmp);
2510         rte_free(internals->vlan_filter_bmpmem);
2511         rte_free(eth_dev->data->dev_private);
2512         rte_free(eth_dev->data->mac_addrs);
2513
2514         rte_eth_dev_release_port(eth_dev);
2515
2516         return 0;
2517 }
2518
2519 /* this part will resolve the slave portids after all the other pdev and vdev
2520  * have been allocated */
2521 static int
2522 bond_ethdev_configure(struct rte_eth_dev *dev)
2523 {
2524         char *name = dev->data->name;
2525         struct bond_dev_private *internals = dev->data->dev_private;
2526         struct rte_kvargs *kvlist = internals->kvlist;
2527         int arg_count;
2528         uint8_t port_id = dev - rte_eth_devices;
2529
2530         static const uint8_t default_rss_key[40] = {
2531                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 0x41, 0x67, 0x25, 0x3D,
2532                 0x43, 0xA3, 0x8F, 0xB0, 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
2533                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 0x6A, 0x42, 0xB7, 0x3B,
2534                 0xBE, 0xAC, 0x01, 0xFA
2535         };
2536
2537         unsigned i, j;
2538
2539         /* If RSS is enabled, fill table and key with default values */
2540         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) {
2541                 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = internals->rss_key;
2542                 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len = 0;
2543                 memcpy(internals->rss_key, default_rss_key, 40);
2544
2545                 for (i = 0; i < RTE_DIM(internals->reta_conf); i++) {
2546                         internals->reta_conf[i].mask = ~0LL;
2547                         for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
2548                                 internals->reta_conf[i].reta[j] = j % dev->data->nb_rx_queues;
2549                 }
2550         }
2551
2552         /* set the max_rx_pktlen */
2553         internals->max_rx_pktlen = internals->candidate_max_rx_pktlen;
2554
2555         /*
2556          * if no kvlist, it means that this bonded device has been created
2557          * through the bonding api.
2558          */
2559         if (!kvlist)
2560                 return 0;
2561
2562         /* Parse MAC address for bonded device */
2563         arg_count = rte_kvargs_count(kvlist, PMD_BOND_MAC_ADDR_KVARG);
2564         if (arg_count == 1) {
2565                 struct ether_addr bond_mac;
2566
2567                 if (rte_kvargs_process(kvlist, PMD_BOND_MAC_ADDR_KVARG,
2568                                 &bond_ethdev_parse_bond_mac_addr_kvarg, &bond_mac) < 0) {
2569                         RTE_LOG(INFO, EAL, "Invalid mac address for bonded device %s\n",
2570                                         name);
2571                         return -1;
2572                 }
2573
2574                 /* Set MAC address */
2575                 if (rte_eth_bond_mac_address_set(port_id, &bond_mac) != 0) {
2576                         RTE_LOG(ERR, EAL,
2577                                         "Failed to set mac address on bonded device %s\n",
2578                                         name);
2579                         return -1;
2580                 }
2581         } else if (arg_count > 1) {
2582                 RTE_LOG(ERR, EAL,
2583                                 "MAC address can be specified only once for bonded device %s\n",
2584                                 name);
2585                 return -1;
2586         }
2587
2588         /* Parse/set balance mode transmit policy */
2589         arg_count = rte_kvargs_count(kvlist, PMD_BOND_XMIT_POLICY_KVARG);
2590         if (arg_count == 1) {
2591                 uint8_t xmit_policy;
2592
2593                 if (rte_kvargs_process(kvlist, PMD_BOND_XMIT_POLICY_KVARG,
2594                                 &bond_ethdev_parse_balance_xmit_policy_kvarg, &xmit_policy) !=
2595                                                 0) {
2596                         RTE_LOG(INFO, EAL,
2597                                         "Invalid xmit policy specified for bonded device %s\n",
2598                                         name);
2599                         return -1;
2600                 }
2601
2602                 /* Set balance mode transmit policy*/
2603                 if (rte_eth_bond_xmit_policy_set(port_id, xmit_policy) != 0) {
2604                         RTE_LOG(ERR, EAL,
2605                                         "Failed to set balance xmit policy on bonded device %s\n",
2606                                         name);
2607                         return -1;
2608                 }
2609         } else if (arg_count > 1) {
2610                 RTE_LOG(ERR, EAL,
2611                                 "Transmit policy can be specified only once for bonded device"
2612                                 " %s\n", name);
2613                 return -1;
2614         }
2615
2616         /* Parse/add slave ports to bonded device */
2617         if (rte_kvargs_count(kvlist, PMD_BOND_SLAVE_PORT_KVARG) > 0) {
2618                 struct bond_ethdev_slave_ports slave_ports;
2619                 unsigned i;
2620
2621                 memset(&slave_ports, 0, sizeof(slave_ports));
2622
2623                 if (rte_kvargs_process(kvlist, PMD_BOND_SLAVE_PORT_KVARG,
2624                                 &bond_ethdev_parse_slave_port_kvarg, &slave_ports) != 0) {
2625                         RTE_LOG(ERR, EAL,
2626                                         "Failed to parse slave ports for bonded device %s\n",
2627                                         name);
2628                         return -1;
2629                 }
2630
2631                 for (i = 0; i < slave_ports.slave_count; i++) {
2632                         if (rte_eth_bond_slave_add(port_id, slave_ports.slaves[i]) != 0) {
2633                                 RTE_LOG(ERR, EAL,
2634                                                 "Failed to add port %d as slave to bonded device %s\n",
2635                                                 slave_ports.slaves[i], name);
2636                         }
2637                 }
2638
2639         } else {
2640                 RTE_LOG(INFO, EAL, "No slaves specified for bonded device %s\n", name);
2641                 return -1;
2642         }
2643
2644         /* Parse/set primary slave port id*/
2645         arg_count = rte_kvargs_count(kvlist, PMD_BOND_PRIMARY_SLAVE_KVARG);
2646         if (arg_count == 1) {
2647                 uint8_t primary_slave_port_id;
2648
2649                 if (rte_kvargs_process(kvlist,
2650                                 PMD_BOND_PRIMARY_SLAVE_KVARG,
2651                                 &bond_ethdev_parse_primary_slave_port_id_kvarg,
2652                                 &primary_slave_port_id) < 0) {
2653                         RTE_LOG(INFO, EAL,
2654                                         "Invalid primary slave port id specified for bonded device"
2655                                         " %s\n", name);
2656                         return -1;
2657                 }
2658
2659                 /* Set balance mode transmit policy*/
2660                 if (rte_eth_bond_primary_set(port_id, (uint8_t)primary_slave_port_id)
2661                                 != 0) {
2662                         RTE_LOG(ERR, EAL,
2663                                         "Failed to set primary slave port %d on bonded device %s\n",
2664                                         primary_slave_port_id, name);
2665                         return -1;
2666                 }
2667         } else if (arg_count > 1) {
2668                 RTE_LOG(INFO, EAL,
2669                                 "Primary slave can be specified only once for bonded device"
2670                                 " %s\n", name);
2671                 return -1;
2672         }
2673
2674         /* Parse link status monitor polling interval */
2675         arg_count = rte_kvargs_count(kvlist, PMD_BOND_LSC_POLL_PERIOD_KVARG);
2676         if (arg_count == 1) {
2677                 uint32_t lsc_poll_interval_ms;
2678
2679                 if (rte_kvargs_process(kvlist,
2680                                 PMD_BOND_LSC_POLL_PERIOD_KVARG,
2681                                 &bond_ethdev_parse_time_ms_kvarg,
2682                                 &lsc_poll_interval_ms) < 0) {
2683                         RTE_LOG(INFO, EAL,
2684                                         "Invalid lsc polling interval value specified for bonded"
2685                                         " device %s\n", name);
2686                         return -1;
2687                 }
2688
2689                 if (rte_eth_bond_link_monitoring_set(port_id, lsc_poll_interval_ms)
2690                                 != 0) {
2691                         RTE_LOG(ERR, EAL,
2692                                         "Failed to set lsc monitor polling interval (%u ms) on"
2693                                         " bonded device %s\n", lsc_poll_interval_ms, name);
2694                         return -1;
2695                 }
2696         } else if (arg_count > 1) {
2697                 RTE_LOG(INFO, EAL,
2698                                 "LSC polling interval can be specified only once for bonded"
2699                                 " device %s\n", name);
2700                 return -1;
2701         }
2702
2703         /* Parse link up interrupt propagation delay */
2704         arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_UP_PROP_DELAY_KVARG);
2705         if (arg_count == 1) {
2706                 uint32_t link_up_delay_ms;
2707
2708                 if (rte_kvargs_process(kvlist,
2709                                 PMD_BOND_LINK_UP_PROP_DELAY_KVARG,
2710                                 &bond_ethdev_parse_time_ms_kvarg,
2711                                 &link_up_delay_ms) < 0) {
2712                         RTE_LOG(INFO, EAL,
2713                                         "Invalid link up propagation delay value specified for"
2714                                         " bonded device %s\n", name);
2715                         return -1;
2716                 }
2717
2718                 /* Set balance mode transmit policy*/
2719                 if (rte_eth_bond_link_up_prop_delay_set(port_id, link_up_delay_ms)
2720                                 != 0) {
2721                         RTE_LOG(ERR, EAL,
2722                                         "Failed to set link up propagation delay (%u ms) on bonded"
2723                                         " device %s\n", link_up_delay_ms, name);
2724                         return -1;
2725                 }
2726         } else if (arg_count > 1) {
2727                 RTE_LOG(INFO, EAL,
2728                                 "Link up propagation delay can be specified only once for"
2729                                 " bonded device %s\n", name);
2730                 return -1;
2731         }
2732
2733         /* Parse link down interrupt propagation delay */
2734         arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG);
2735         if (arg_count == 1) {
2736                 uint32_t link_down_delay_ms;
2737
2738                 if (rte_kvargs_process(kvlist,
2739                                 PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG,
2740                                 &bond_ethdev_parse_time_ms_kvarg,
2741                                 &link_down_delay_ms) < 0) {
2742                         RTE_LOG(INFO, EAL,
2743                                         "Invalid link down propagation delay value specified for"
2744                                         " bonded device %s\n", name);
2745                         return -1;
2746                 }
2747
2748                 /* Set balance mode transmit policy*/
2749                 if (rte_eth_bond_link_down_prop_delay_set(port_id, link_down_delay_ms)
2750                                 != 0) {
2751                         RTE_LOG(ERR, EAL,
2752                                         "Failed to set link down propagation delay (%u ms) on"
2753                                         " bonded device %s\n", link_down_delay_ms, name);
2754                         return -1;
2755                 }
2756         } else if (arg_count > 1) {
2757                 RTE_LOG(INFO, EAL,
2758                                 "Link down propagation delay can be specified only once for"
2759                                 " bonded device %s\n", name);
2760                 return -1;
2761         }
2762
2763         return 0;
2764 }
2765
2766 struct rte_vdev_driver pmd_bond_drv = {
2767         .probe = bond_probe,
2768         .remove = bond_remove,
2769 };
2770
2771 RTE_PMD_REGISTER_VDEV(net_bonding, pmd_bond_drv);
2772 RTE_PMD_REGISTER_ALIAS(net_bonding, eth_bond);
2773
2774 RTE_PMD_REGISTER_PARAM_STRING(net_bonding,
2775         "slave=<ifc> "
2776         "primary=<ifc> "
2777         "mode=[0-6] "
2778         "xmit_policy=[l2 | l23 | l34] "
2779         "socket_id=<int> "
2780         "mac=<mac addr> "
2781         "lsc_poll_period_ms=<int> "
2782         "up_delay=<int> "
2783         "down_delay=<int>");