drivers/net/bonding/rte_eth_bond_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33 #include <stdlib.h>
  34 #include <netinet/in.h>
  35
  36 #include <rte_mbuf.h>
  37 #include <rte_malloc.h>
  38 #include <rte_ethdev.h>
  39 #include <rte_ethdev_vdev.h>
  40 #include <rte_tcp.h>
  41 #include <rte_udp.h>
  42 #include <rte_ip.h>
  43 #include <rte_ip_frag.h>
  44 #include <rte_devargs.h>
  45 #include <rte_kvargs.h>
  46 #include <rte_vdev.h>
  47 #include <rte_alarm.h>
  48 #include <rte_cycles.h>
  49
  50 #include "rte_eth_bond.h"
  51 #include "rte_eth_bond_private.h"
  52 #include "rte_eth_bond_8023ad_private.h"
  53
  54 #define REORDER_PERIOD_MS 10
  55 #define DEFAULT_POLLING_INTERVAL_10_MS (10)
  56
  57 #define HASH_L4_PORTS(h) ((h)->src_port ^ (h)->dst_port)
  58
  59 /* Table for statistics in mode 5 TLB */
  60 static uint64_t tlb_last_obytets[RTE_MAX_ETHPORTS];
  61
  62 static inline size_t
  63 get_vlan_offset(struct ether_hdr *eth_hdr, uint16_t *proto)
  64 {
  65         size_t vlan_offset = 0;
  66
  67         if (rte_cpu_to_be_16(ETHER_TYPE_VLAN) == *proto) {
  68                 struct vlan_hdr *vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1);
  69
  70                 vlan_offset = sizeof(struct vlan_hdr);
  71                 *proto = vlan_hdr->eth_proto;
  72
  73                 if (rte_cpu_to_be_16(ETHER_TYPE_VLAN) == *proto) {
  74                         vlan_hdr = vlan_hdr + 1;
  75                         *proto = vlan_hdr->eth_proto;
  76                         vlan_offset += sizeof(struct vlan_hdr);
  77                 }
  78         }
  79         return vlan_offset;
  80 }
  81
  82 static uint16_t
  83 bond_ethdev_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
  84 {
  85         struct bond_dev_private *internals;
  86
  87         uint16_t num_rx_slave = 0;
  88         uint16_t num_rx_total = 0;
  89
  90         int i;
  91
  92         /* Cast to structure, containing bonded device's port id and queue id */
  93         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
  94
  95         internals = bd_rx_q->dev_private;
  96
  97
  98         for (i = 0; i < internals->active_slave_count && nb_pkts; i++) {
  99                 /* Offset of pointer to *bufs increases as packets are received
 100                  * from other slaves */
 101                 num_rx_slave = rte_eth_rx_burst(internals->active_slaves[i],
 102                                 bd_rx_q->queue_id, bufs + num_rx_total, nb_pkts);
 103                 if (num_rx_slave) {
 104                         num_rx_total += num_rx_slave;
 105                         nb_pkts -= num_rx_slave;
 106                 }
 107         }
 108
 109         return num_rx_total;
 110 }
 111
 112 static uint16_t
 113 bond_ethdev_rx_burst_active_backup(void *queue, struct rte_mbuf **bufs,
 114                 uint16_t nb_pkts)
 115 {
 116         struct bond_dev_private *internals;
 117
 118         /* Cast to structure, containing bonded device's port id and queue id */
 119         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
 120
 121         internals = bd_rx_q->dev_private;
 122
 123         return rte_eth_rx_burst(internals->current_primary_port,
 124                         bd_rx_q->queue_id, bufs, nb_pkts);
 125 }
 126
 127 static inline uint8_t
 128 is_lacp_packets(uint16_t ethertype, uint8_t subtype, uint16_t vlan_tci)
 129 {
 130         const uint16_t ether_type_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW);
 131
 132         return !vlan_tci && (ethertype == ether_type_slow_be &&
 133                 (subtype == SLOW_SUBTYPE_MARKER || subtype == SLOW_SUBTYPE_LACP));
 134 }
 135
 136 static uint16_t
 137 bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 138                 uint16_t nb_pkts)
 139 {
 140         /* Cast to structure, containing bonded device's port id and queue id */
 141         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
 142         struct bond_dev_private *internals = bd_rx_q->dev_private;
 143         struct ether_addr bond_mac;
 144
 145         struct ether_hdr *hdr;
 146
 147         const uint16_t ether_type_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW);
 148         uint16_t num_rx_total = 0;      /* Total number of received packets */
 149         uint8_t slaves[RTE_MAX_ETHPORTS];
 150         uint8_t slave_count, idx;
 151
 152         uint8_t collecting;  /* current slave collecting status */
 153         const uint8_t promisc = internals->promiscuous_en;
 154         uint8_t i, j, k;
 155         uint8_t subtype;
 156
 157         rte_eth_macaddr_get(internals->port_id, &bond_mac);
 158         /* Copy slave list to protect against slave up/down changes during tx
 159          * bursting */
 160         slave_count = internals->active_slave_count;
 161         memcpy(slaves, internals->active_slaves,
 162                         sizeof(internals->active_slaves[0]) * slave_count);
 163
 164         idx = internals->active_slave;
 165         if (idx >= slave_count) {
 166                 internals->active_slave = 0;
 167                 idx = 0;
 168         }
 169         for (i = 0; i < slave_count && num_rx_total < nb_pkts; i++) {
 170                 j = num_rx_total;
 171                 collecting = ACTOR_STATE(&mode_8023ad_ports[slaves[idx]],
 172                                          COLLECTING);
 173
 174                 /* Read packets from this slave */
 175                 num_rx_total += rte_eth_rx_burst(slaves[idx], bd_rx_q->queue_id,
 176                                 &bufs[num_rx_total], nb_pkts - num_rx_total);
 177
 178                 for (k = j; k < 2 && k < num_rx_total; k++)
 179                         rte_prefetch0(rte_pktmbuf_mtod(bufs[k], void *));
 180
 181                 /* Handle slow protocol packets. */
 182                 while (j < num_rx_total) {
 183                         if (j + 3 < num_rx_total)
 184                                 rte_prefetch0(rte_pktmbuf_mtod(bufs[j + 3], void *));
 185
 186                         hdr = rte_pktmbuf_mtod(bufs[j], struct ether_hdr *);
 187                         subtype = ((struct slow_protocol_frame *)hdr)->slow_protocol.subtype;
 188
 189                         /* Remove packet from array if it is slow packet or slave is not
 190                          * in collecting state or bondign interface is not in promiscus
 191                          * mode and packet address does not match. */
 192                         if (unlikely(is_lacp_packets(hdr->ether_type, subtype, bufs[j]->vlan_tci) ||
 193                                 !collecting || (!promisc &&
 194                                         !is_multicast_ether_addr(&hdr->d_addr) &&
 195                                         !is_same_ether_addr(&bond_mac, &hdr->d_addr)))) {
 196
 197                                 if (hdr->ether_type == ether_type_slow_be) {
 198                                         bond_mode_8023ad_handle_slow_pkt(
 199                                             internals, slaves[idx], bufs[j]);
 200                                 } else
 201                                         rte_pktmbuf_free(bufs[j]);
 202
 203                                 /* Packet is managed by mode 4 or dropped, shift the array */
 204                                 num_rx_total--;
 205                                 if (j < num_rx_total) {
 206                                         memmove(&bufs[j], &bufs[j + 1], sizeof(bufs[0]) *
 207                                                 (num_rx_total - j));
 208                                 }
 209                         } else
 210                                 j++;
 211                 }
 212                 if (unlikely(++idx == slave_count))
 213                         idx = 0;
 214         }
 215
 216         internals->active_slave = idx;
 217         return num_rx_total;
 218 }
 219
 220 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 221 uint32_t burstnumberRX;
 222 uint32_t burstnumberTX;
 223
 224 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 225
 226 static void
 227 arp_op_name(uint16_t arp_op, char *buf)
 228 {
 229         switch (arp_op) {
 230         case ARP_OP_REQUEST:
 231                 snprintf(buf, sizeof("ARP Request"), "%s", "ARP Request");
 232                 return;
 233         case ARP_OP_REPLY:
 234                 snprintf(buf, sizeof("ARP Reply"), "%s", "ARP Reply");
 235                 return;
 236         case ARP_OP_REVREQUEST:
 237                 snprintf(buf, sizeof("Reverse ARP Request"), "%s",
 238                                 "Reverse ARP Request");
 239                 return;
 240         case ARP_OP_REVREPLY:
 241                 snprintf(buf, sizeof("Reverse ARP Reply"), "%s",
 242                                 "Reverse ARP Reply");
 243                 return;
 244         case ARP_OP_INVREQUEST:
 245                 snprintf(buf, sizeof("Peer Identify Request"), "%s",
 246                                 "Peer Identify Request");
 247                 return;
 248         case ARP_OP_INVREPLY:
 249                 snprintf(buf, sizeof("Peer Identify Reply"), "%s",
 250                                 "Peer Identify Reply");
 251                 return;
 252         default:
 253                 break;
 254         }
 255         snprintf(buf, sizeof("Unknown"), "%s", "Unknown");
 256         return;
 257 }
 258 #endif
 259 #define MaxIPv4String   16
 260 static void
 261 ipv4_addr_to_dot(uint32_t be_ipv4_addr, char *buf, uint8_t buf_size)
 262 {
 263         uint32_t ipv4_addr;
 264
 265         ipv4_addr = rte_be_to_cpu_32(be_ipv4_addr);
 266         snprintf(buf, buf_size, "%d.%d.%d.%d", (ipv4_addr >> 24) & 0xFF,
 267                 (ipv4_addr >> 16) & 0xFF, (ipv4_addr >> 8) & 0xFF,
 268                 ipv4_addr & 0xFF);
 269 }
 270
 271 #define MAX_CLIENTS_NUMBER      128
 272 uint8_t active_clients;
 273 struct client_stats_t {
 274         uint8_t port;
 275         uint32_t ipv4_addr;
 276         uint32_t ipv4_rx_packets;
 277         uint32_t ipv4_tx_packets;
 278 };
 279 struct client_stats_t client_stats[MAX_CLIENTS_NUMBER];
 280
 281 static void
 282 update_client_stats(uint32_t addr, uint8_t port, uint32_t *TXorRXindicator)
 283 {
 284         int i = 0;
 285
 286         for (; i < MAX_CLIENTS_NUMBER; i++)     {
 287                 if ((client_stats[i].ipv4_addr == addr) && (client_stats[i].port == port))      {
 288                         /* Just update RX packets number for this client */
 289                         if (TXorRXindicator == &burstnumberRX)
 290                                 client_stats[i].ipv4_rx_packets++;
 291                         else
 292                                 client_stats[i].ipv4_tx_packets++;
 293                         return;
 294                 }
 295         }
 296         /* We have a new client. Insert him to the table, and increment stats */
 297         if (TXorRXindicator == &burstnumberRX)
 298                 client_stats[active_clients].ipv4_rx_packets++;
 299         else
 300                 client_stats[active_clients].ipv4_tx_packets++;
 301         client_stats[active_clients].ipv4_addr = addr;
 302         client_stats[active_clients].port = port;
 303         active_clients++;
 304
 305 }
 306
 307 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 308 #define MODE6_DEBUG(info, src_ip, dst_ip, eth_h, arp_op, port, burstnumber)     \
 309                 RTE_LOG(DEBUG, PMD, \
 310                 "%s " \
 311                 "port:%d " \
 312                 "SrcMAC:%02X:%02X:%02X:%02X:%02X:%02X " \
 313                 "SrcIP:%s " \
 314                 "DstMAC:%02X:%02X:%02X:%02X:%02X:%02X " \
 315                 "DstIP:%s " \
 316                 "%s " \
 317                 "%d\n", \
 318                 info, \
 319                 port, \
 320                 eth_h->s_addr.addr_bytes[0], \
 321                 eth_h->s_addr.addr_bytes[1], \
 322                 eth_h->s_addr.addr_bytes[2], \
 323                 eth_h->s_addr.addr_bytes[3], \
 324                 eth_h->s_addr.addr_bytes[4], \
 325                 eth_h->s_addr.addr_bytes[5], \
 326                 src_ip, \
 327                 eth_h->d_addr.addr_bytes[0], \
 328                 eth_h->d_addr.addr_bytes[1], \
 329                 eth_h->d_addr.addr_bytes[2], \
 330                 eth_h->d_addr.addr_bytes[3], \
 331                 eth_h->d_addr.addr_bytes[4], \
 332                 eth_h->d_addr.addr_bytes[5], \
 333                 dst_ip, \
 334                 arp_op, \
 335                 ++burstnumber)
 336 #endif
 337
 338 static void
 339 mode6_debug(const char __attribute__((unused)) *info, struct ether_hdr *eth_h,
 340                 uint8_t port, uint32_t __attribute__((unused)) *burstnumber)
 341 {
 342         struct ipv4_hdr *ipv4_h;
 343 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 344         struct arp_hdr *arp_h;
 345         char dst_ip[16];
 346         char ArpOp[24];
 347         char buf[16];
 348 #endif
 349         char src_ip[16];
 350
 351         uint16_t ether_type = eth_h->ether_type;
 352         uint16_t offset = get_vlan_offset(eth_h, &ether_type);
 353
 354 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 355         snprintf(buf, 16, "%s", info);
 356 #endif
 357
 358         if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) {
 359                 ipv4_h = (struct ipv4_hdr *)((char *)(eth_h + 1) + offset);
 360                 ipv4_addr_to_dot(ipv4_h->src_addr, src_ip, MaxIPv4String);
 361 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 362                 ipv4_addr_to_dot(ipv4_h->dst_addr, dst_ip, MaxIPv4String);
 363                 MODE6_DEBUG(buf, src_ip, dst_ip, eth_h, "", port, *burstnumber);
 364 #endif
 365                 update_client_stats(ipv4_h->src_addr, port, burstnumber);
 366         }
 367 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 368         else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_ARP)) {
 369                 arp_h = (struct arp_hdr *)((char *)(eth_h + 1) + offset);
 370                 ipv4_addr_to_dot(arp_h->arp_data.arp_sip, src_ip, MaxIPv4String);
 371                 ipv4_addr_to_dot(arp_h->arp_data.arp_tip, dst_ip, MaxIPv4String);
 372                 arp_op_name(rte_be_to_cpu_16(arp_h->arp_op), ArpOp);
 373                 MODE6_DEBUG(buf, src_ip, dst_ip, eth_h, ArpOp, port, *burstnumber);
 374         }
 375 #endif
 376 }
 377 #endif
 378
 379 static uint16_t
 380 bond_ethdev_rx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 381 {
 382         struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
 383         struct bond_dev_private *internals = bd_tx_q->dev_private;
 384         struct ether_hdr *eth_h;
 385         uint16_t ether_type, offset;
 386         uint16_t nb_recv_pkts;
 387         int i;
 388
 389         nb_recv_pkts = bond_ethdev_rx_burst(queue, bufs, nb_pkts);
 390
 391         for (i = 0; i < nb_recv_pkts; i++) {
 392                 eth_h = rte_pktmbuf_mtod(bufs[i], struct ether_hdr *);
 393                 ether_type = eth_h->ether_type;
 394                 offset = get_vlan_offset(eth_h, &ether_type);
 395
 396                 if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_ARP)) {
 397 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 398                         mode6_debug("RX ARP:", eth_h, bufs[i]->port, &burstnumberRX);
 399 #endif
 400                         bond_mode_alb_arp_recv(eth_h, offset, internals);
 401                 }
 402 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 403                 else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4))
 404                         mode6_debug("RX IPv4:", eth_h, bufs[i]->port, &burstnumberRX);
 405 #endif
 406         }
 407
 408         return nb_recv_pkts;
 409 }
 410
 411 static uint16_t
 412 bond_ethdev_tx_burst_round_robin(void *queue, struct rte_mbuf **bufs,
 413                 uint16_t nb_pkts)
 414 {
 415         struct bond_dev_private *internals;
 416         struct bond_tx_queue *bd_tx_q;
 417
 418         struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts];
 419         uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
 420
 421         uint8_t num_of_slaves;
 422         uint8_t slaves[RTE_MAX_ETHPORTS];
 423
 424         uint16_t num_tx_total = 0, num_tx_slave;
 425
 426         static int slave_idx = 0;
 427         int i, cslave_idx = 0, tx_fail_total = 0;
 428
 429         bd_tx_q = (struct bond_tx_queue *)queue;
 430         internals = bd_tx_q->dev_private;
 431
 432         /* Copy slave list to protect against slave up/down changes during tx
 433          * bursting */
 434         num_of_slaves = internals->active_slave_count;
 435         memcpy(slaves, internals->active_slaves,
 436                         sizeof(internals->active_slaves[0]) * num_of_slaves);
 437
 438         if (num_of_slaves < 1)
 439                 return num_tx_total;
 440
 441         /* Populate slaves mbuf with which packets are to be sent on it  */
 442         for (i = 0; i < nb_pkts; i++) {
 443                 cslave_idx = (slave_idx + i) % num_of_slaves;
 444                 slave_bufs[cslave_idx][(slave_nb_pkts[cslave_idx])++] = bufs[i];
 445         }
 446
 447         /* increment current slave index so the next call to tx burst starts on the
 448          * next slave */
 449         slave_idx = ++cslave_idx;
 450
 451         /* Send packet burst on each slave device */
 452         for (i = 0; i < num_of_slaves; i++) {
 453                 if (slave_nb_pkts[i] > 0) {
 454                         num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
 455                                         slave_bufs[i], slave_nb_pkts[i]);
 456
 457                         /* if tx burst fails move packets to end of bufs */
 458                         if (unlikely(num_tx_slave < slave_nb_pkts[i])) {
 459                                 int tx_fail_slave = slave_nb_pkts[i] - num_tx_slave;
 460
 461                                 tx_fail_total += tx_fail_slave;
 462
 463                                 memcpy(&bufs[nb_pkts - tx_fail_total],
 464                                                 &slave_bufs[i][num_tx_slave],
 465                                                 tx_fail_slave * sizeof(bufs[0]));
 466                         }
 467                         num_tx_total += num_tx_slave;
 468                 }
 469         }
 470
 471         return num_tx_total;
 472 }
 473
 474 static uint16_t
 475 bond_ethdev_tx_burst_active_backup(void *queue,
 476                 struct rte_mbuf **bufs, uint16_t nb_pkts)
 477 {
 478         struct bond_dev_private *internals;
 479         struct bond_tx_queue *bd_tx_q;
 480
 481         bd_tx_q = (struct bond_tx_queue *)queue;
 482         internals = bd_tx_q->dev_private;
 483
 484         if (internals->active_slave_count < 1)
 485                 return 0;
 486
 487         return rte_eth_tx_burst(internals->current_primary_port, bd_tx_q->queue_id,
 488                         bufs, nb_pkts);
 489 }
 490
 491 static inline uint16_t
 492 ether_hash(struct ether_hdr *eth_hdr)
 493 {
 494         unaligned_uint16_t *word_src_addr =
 495                 (unaligned_uint16_t *)eth_hdr->s_addr.addr_bytes;
 496         unaligned_uint16_t *word_dst_addr =
 497                 (unaligned_uint16_t *)eth_hdr->d_addr.addr_bytes;
 498
 499         return (word_src_addr[0] ^ word_dst_addr[0]) ^
 500                         (word_src_addr[1] ^ word_dst_addr[1]) ^
 501                         (word_src_addr[2] ^ word_dst_addr[2]);
 502 }
 503
 504 static inline uint32_t
 505 ipv4_hash(struct ipv4_hdr *ipv4_hdr)
 506 {
 507         return ipv4_hdr->src_addr ^ ipv4_hdr->dst_addr;
 508 }
 509
 510 static inline uint32_t
 511 ipv6_hash(struct ipv6_hdr *ipv6_hdr)
 512 {
 513         unaligned_uint32_t *word_src_addr =
 514                 (unaligned_uint32_t *)&(ipv6_hdr->src_addr[0]);
 515         unaligned_uint32_t *word_dst_addr =
 516                 (unaligned_uint32_t *)&(ipv6_hdr->dst_addr[0]);
 517
 518         return (word_src_addr[0] ^ word_dst_addr[0]) ^
 519                         (word_src_addr[1] ^ word_dst_addr[1]) ^
 520                         (word_src_addr[2] ^ word_dst_addr[2]) ^
 521                         (word_src_addr[3] ^ word_dst_addr[3]);
 522 }
 523
 524 uint16_t
 525 xmit_l2_hash(const struct rte_mbuf *buf, uint8_t slave_count)
 526 {
 527         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(buf, struct ether_hdr *);
 528
 529         uint32_t hash = ether_hash(eth_hdr);
 530
 531         return (hash ^= hash >> 8) % slave_count;
 532 }
 533
 534 uint16_t
 535 xmit_l23_hash(const struct rte_mbuf *buf, uint8_t slave_count)
 536 {
 537         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(buf, struct ether_hdr *);
 538         uint16_t proto = eth_hdr->ether_type;
 539         size_t vlan_offset = get_vlan_offset(eth_hdr, &proto);
 540         uint32_t hash, l3hash = 0;
 541
 542         hash = ether_hash(eth_hdr);
 543
 544         if (rte_cpu_to_be_16(ETHER_TYPE_IPv4) == proto) {
 545                 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *)
 546                                 ((char *)(eth_hdr + 1) + vlan_offset);
 547                 l3hash = ipv4_hash(ipv4_hdr);
 548
 549         } else if (rte_cpu_to_be_16(ETHER_TYPE_IPv6) == proto) {
 550                 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)
 551                                 ((char *)(eth_hdr + 1) + vlan_offset);
 552                 l3hash = ipv6_hash(ipv6_hdr);
 553         }
 554
 555         hash = hash ^ l3hash;
 556         hash ^= hash >> 16;
 557         hash ^= hash >> 8;
 558
 559         return hash % slave_count;
 560 }
 561
 562 uint16_t
 563 xmit_l34_hash(const struct rte_mbuf *buf, uint8_t slave_count)
 564 {
 565         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(buf, struct ether_hdr *);
 566         uint16_t proto = eth_hdr->ether_type;
 567         size_t vlan_offset = get_vlan_offset(eth_hdr, &proto);
 568
 569         struct udp_hdr *udp_hdr = NULL;
 570         struct tcp_hdr *tcp_hdr = NULL;
 571         uint32_t hash, l3hash = 0, l4hash = 0;
 572
 573         if (rte_cpu_to_be_16(ETHER_TYPE_IPv4) == proto) {
 574                 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *)
 575                                 ((char *)(eth_hdr + 1) + vlan_offset);
 576                 size_t ip_hdr_offset;
 577
 578                 l3hash = ipv4_hash(ipv4_hdr);
 579
 580                 /* there is no L4 header in fragmented packet */
 581                 if (likely(rte_ipv4_frag_pkt_is_fragmented(ipv4_hdr) == 0)) {
 582                         ip_hdr_offset = (ipv4_hdr->version_ihl & IPV4_HDR_IHL_MASK) *
 583                                         IPV4_IHL_MULTIPLIER;
 584
 585                         if (ipv4_hdr->next_proto_id == IPPROTO_TCP) {
 586                                 tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr +
 587                                                 ip_hdr_offset);
 588                                 l4hash = HASH_L4_PORTS(tcp_hdr);
 589                         } else if (ipv4_hdr->next_proto_id == IPPROTO_UDP) {
 590                                 udp_hdr = (struct udp_hdr *)((char *)ipv4_hdr +
 591                                                 ip_hdr_offset);
 592                                 l4hash = HASH_L4_PORTS(udp_hdr);
 593                         }
 594                 }
 595         } else if  (rte_cpu_to_be_16(ETHER_TYPE_IPv6) == proto) {
 596                 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)
 597                                 ((char *)(eth_hdr + 1) + vlan_offset);
 598                 l3hash = ipv6_hash(ipv6_hdr);
 599
 600                 if (ipv6_hdr->proto == IPPROTO_TCP) {
 601                         tcp_hdr = (struct tcp_hdr *)(ipv6_hdr + 1);
 602                         l4hash = HASH_L4_PORTS(tcp_hdr);
 603                 } else if (ipv6_hdr->proto == IPPROTO_UDP) {
 604                         udp_hdr = (struct udp_hdr *)(ipv6_hdr + 1);
 605                         l4hash = HASH_L4_PORTS(udp_hdr);
 606                 }
 607         }
 608
 609         hash = l3hash ^ l4hash;
 610         hash ^= hash >> 16;
 611         hash ^= hash >> 8;
 612
 613         return hash % slave_count;
 614 }
 615
 616 struct bwg_slave {
 617         uint64_t bwg_left_int;
 618         uint64_t bwg_left_remainder;
 619         uint8_t slave;
 620 };
 621
 622 void
 623 bond_tlb_activate_slave(struct bond_dev_private *internals) {
 624         int i;
 625
 626         for (i = 0; i < internals->active_slave_count; i++) {
 627                 tlb_last_obytets[internals->active_slaves[i]] = 0;
 628         }
 629 }
 630
 631 static int
 632 bandwidth_cmp(const void *a, const void *b)
 633 {
 634         const struct bwg_slave *bwg_a = a;
 635         const struct bwg_slave *bwg_b = b;
 636         int64_t diff = (int64_t)bwg_b->bwg_left_int - (int64_t)bwg_a->bwg_left_int;
 637         int64_t diff2 = (int64_t)bwg_b->bwg_left_remainder -
 638                         (int64_t)bwg_a->bwg_left_remainder;
 639         if (diff > 0)
 640                 return 1;
 641         else if (diff < 0)
 642                 return -1;
 643         else if (diff2 > 0)
 644                 return 1;
 645         else if (diff2 < 0)
 646                 return -1;
 647         else
 648                 return 0;
 649 }
 650
 651 static void
 652 bandwidth_left(uint8_t port_id, uint64_t load, uint8_t update_idx,
 653                 struct bwg_slave *bwg_slave)
 654 {
 655         struct rte_eth_link link_status;
 656
 657         rte_eth_link_get(port_id, &link_status);
 658         uint64_t link_bwg = link_status.link_speed * 1000000ULL / 8;
 659         if (link_bwg == 0)
 660                 return;
 661         link_bwg = link_bwg * (update_idx+1) * REORDER_PERIOD_MS;
 662         bwg_slave->bwg_left_int = (link_bwg - 1000*load) / link_bwg;
 663         bwg_slave->bwg_left_remainder = (link_bwg - 1000*load) % link_bwg;
 664 }
 665
 666 static void
 667 bond_ethdev_update_tlb_slave_cb(void *arg)
 668 {
 669         struct bond_dev_private *internals = arg;
 670         struct rte_eth_stats slave_stats;
 671         struct bwg_slave bwg_array[RTE_MAX_ETHPORTS];
 672         uint8_t slave_count;
 673         uint64_t tx_bytes;
 674
 675         uint8_t update_stats = 0;
 676         uint8_t i, slave_id;
 677
 678         internals->slave_update_idx++;
 679
 680
 681         if (internals->slave_update_idx >= REORDER_PERIOD_MS)
 682                 update_stats = 1;
 683
 684         for (i = 0; i < internals->active_slave_count; i++) {
 685                 slave_id = internals->active_slaves[i];
 686                 rte_eth_stats_get(slave_id, &slave_stats);
 687                 tx_bytes = slave_stats.obytes - tlb_last_obytets[slave_id];
 688                 bandwidth_left(slave_id, tx_bytes,
 689                                 internals->slave_update_idx, &bwg_array[i]);
 690                 bwg_array[i].slave = slave_id;
 691
 692                 if (update_stats) {
 693                         tlb_last_obytets[slave_id] = slave_stats.obytes;
 694                 }
 695         }
 696
 697         if (update_stats == 1)
 698                 internals->slave_update_idx = 0;
 699
 700         slave_count = i;
 701         qsort(bwg_array, slave_count, sizeof(bwg_array[0]), bandwidth_cmp);
 702         for (i = 0; i < slave_count; i++)
 703                 internals->tlb_slaves_order[i] = bwg_array[i].slave;
 704
 705         rte_eal_alarm_set(REORDER_PERIOD_MS * 1000, bond_ethdev_update_tlb_slave_cb,
 706                         (struct bond_dev_private *)internals);
 707 }
 708
 709 static uint16_t
 710 bond_ethdev_tx_burst_tlb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 711 {
 712         struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
 713         struct bond_dev_private *internals = bd_tx_q->dev_private;
 714
 715         struct rte_eth_dev *primary_port =
 716                         &rte_eth_devices[internals->primary_port];
 717         uint16_t num_tx_total = 0;
 718         uint8_t i, j;
 719
 720         uint8_t num_of_slaves = internals->active_slave_count;
 721         uint8_t slaves[RTE_MAX_ETHPORTS];
 722
 723         struct ether_hdr *ether_hdr;
 724         struct ether_addr primary_slave_addr;
 725         struct ether_addr active_slave_addr;
 726
 727         if (num_of_slaves < 1)
 728                 return num_tx_total;
 729
 730         memcpy(slaves, internals->tlb_slaves_order,
 731                                 sizeof(internals->tlb_slaves_order[0]) * num_of_slaves);
 732
 733
 734         ether_addr_copy(primary_port->data->mac_addrs, &primary_slave_addr);
 735
 736         if (nb_pkts > 3) {
 737                 for (i = 0; i < 3; i++)
 738                         rte_prefetch0(rte_pktmbuf_mtod(bufs[i], void*));
 739         }
 740
 741         for (i = 0; i < num_of_slaves; i++) {
 742                 rte_eth_macaddr_get(slaves[i], &active_slave_addr);
 743                 for (j = num_tx_total; j < nb_pkts; j++) {
 744                         if (j + 3 < nb_pkts)
 745                                 rte_prefetch0(rte_pktmbuf_mtod(bufs[j+3], void*));
 746
 747                         ether_hdr = rte_pktmbuf_mtod(bufs[j], struct ether_hdr *);
 748                         if (is_same_ether_addr(&ether_hdr->s_addr, &primary_slave_addr))
 749                                 ether_addr_copy(&active_slave_addr, &ether_hdr->s_addr);
 750 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 751                                         mode6_debug("TX IPv4:", ether_hdr, slaves[i], &burstnumberTX);
 752 #endif
 753                 }
 754
 755                 num_tx_total += rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
 756                                 bufs + num_tx_total, nb_pkts - num_tx_total);
 757
 758                 if (num_tx_total == nb_pkts)
 759                         break;
 760         }
 761
 762         return num_tx_total;
 763 }
 764
 765 void
 766 bond_tlb_disable(struct bond_dev_private *internals)
 767 {
 768         rte_eal_alarm_cancel(bond_ethdev_update_tlb_slave_cb, internals);
 769 }
 770
 771 void
 772 bond_tlb_enable(struct bond_dev_private *internals)
 773 {
 774         bond_ethdev_update_tlb_slave_cb(internals);
 775 }
 776
 777 static uint16_t
 778 bond_ethdev_tx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 779 {
 780         struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
 781         struct bond_dev_private *internals = bd_tx_q->dev_private;
 782
 783         struct ether_hdr *eth_h;
 784         uint16_t ether_type, offset;
 785
 786         struct client_data *client_info;
 787
 788         /*
 789          * We create transmit buffers for every slave and one additional to send
 790          * through tlb. In worst case every packet will be send on one port.
 791          */
 792         struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS + 1][nb_pkts];
 793         uint16_t slave_bufs_pkts[RTE_MAX_ETHPORTS + 1] = { 0 };
 794
 795         /*
 796          * We create separate transmit buffers for update packets as they wont be
 797          * counted in num_tx_total.
 798          */
 799         struct rte_mbuf *update_bufs[RTE_MAX_ETHPORTS][ALB_HASH_TABLE_SIZE];
 800         uint16_t update_bufs_pkts[RTE_MAX_ETHPORTS] = { 0 };
 801
 802         struct rte_mbuf *upd_pkt;
 803         size_t pkt_size;
 804
 805         uint16_t num_send, num_not_send = 0;
 806         uint16_t num_tx_total = 0;
 807         uint8_t slave_idx;
 808
 809         int i, j;
 810
 811         /* Search tx buffer for ARP packets and forward them to alb */
 812         for (i = 0; i < nb_pkts; i++) {
 813                 eth_h = rte_pktmbuf_mtod(bufs[i], struct ether_hdr *);
 814                 ether_type = eth_h->ether_type;
 815                 offset = get_vlan_offset(eth_h, &ether_type);
 816
 817                 if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_ARP)) {
 818                         slave_idx = bond_mode_alb_arp_xmit(eth_h, offset, internals);
 819
 820                         /* Change src mac in eth header */
 821                         rte_eth_macaddr_get(slave_idx, &eth_h->s_addr);
 822
 823                         /* Add packet to slave tx buffer */
 824                         slave_bufs[slave_idx][slave_bufs_pkts[slave_idx]] = bufs[i];
 825                         slave_bufs_pkts[slave_idx]++;
 826                 } else {
 827                         /* If packet is not ARP, send it with TLB policy */
 828                         slave_bufs[RTE_MAX_ETHPORTS][slave_bufs_pkts[RTE_MAX_ETHPORTS]] =
 829                                         bufs[i];
 830                         slave_bufs_pkts[RTE_MAX_ETHPORTS]++;
 831                 }
 832         }
 833
 834         /* Update connected client ARP tables */
 835         if (internals->mode6.ntt) {
 836                 for (i = 0; i < ALB_HASH_TABLE_SIZE; i++) {
 837                         client_info = &internals->mode6.client_table[i];
 838
 839                         if (client_info->in_use) {
 840                                 /* Allocate new packet to send ARP update on current slave */
 841                                 upd_pkt = rte_pktmbuf_alloc(internals->mode6.mempool);
 842                                 if (upd_pkt == NULL) {
 843                                         RTE_LOG(ERR, PMD, "Failed to allocate ARP packet from pool\n");
 844                                         continue;
 845                                 }
 846                                 pkt_size = sizeof(struct ether_hdr) + sizeof(struct arp_hdr)
 847                                                 + client_info->vlan_count * sizeof(struct vlan_hdr);
 848                                 upd_pkt->data_len = pkt_size;
 849                                 upd_pkt->pkt_len = pkt_size;
 850
 851                                 slave_idx = bond_mode_alb_arp_upd(client_info, upd_pkt,
 852                                                 internals);
 853
 854                                 /* Add packet to update tx buffer */
 855                                 update_bufs[slave_idx][update_bufs_pkts[slave_idx]] = upd_pkt;
 856                                 update_bufs_pkts[slave_idx]++;
 857                         }
 858                 }
 859                 internals->mode6.ntt = 0;
 860         }
 861
 862         /* Send ARP packets on proper slaves */
 863         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 864                 if (slave_bufs_pkts[i] > 0) {
 865                         num_send = rte_eth_tx_burst(i, bd_tx_q->queue_id,
 866                                         slave_bufs[i], slave_bufs_pkts[i]);
 867                         for (j = 0; j < slave_bufs_pkts[i] - num_send; j++) {
 868                                 bufs[nb_pkts - 1 - num_not_send - j] =
 869                                                 slave_bufs[i][nb_pkts - 1 - j];
 870                         }
 871
 872                         num_tx_total += num_send;
 873                         num_not_send += slave_bufs_pkts[i] - num_send;
 874
 875 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 876         /* Print TX stats including update packets */
 877                         for (j = 0; j < slave_bufs_pkts[i]; j++) {
 878                                 eth_h = rte_pktmbuf_mtod(slave_bufs[i][j], struct ether_hdr *);
 879                                 mode6_debug("TX ARP:", eth_h, i, &burstnumberTX);
 880                         }
 881 #endif
 882                 }
 883         }
 884
 885         /* Send update packets on proper slaves */
 886         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 887                 if (update_bufs_pkts[i] > 0) {
 888                         num_send = rte_eth_tx_burst(i, bd_tx_q->queue_id, update_bufs[i],
 889                                         update_bufs_pkts[i]);
 890                         for (j = num_send; j < update_bufs_pkts[i]; j++) {
 891                                 rte_pktmbuf_free(update_bufs[i][j]);
 892                         }
 893 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 894                         for (j = 0; j < update_bufs_pkts[i]; j++) {
 895                                 eth_h = rte_pktmbuf_mtod(update_bufs[i][j], struct ether_hdr *);
 896                                 mode6_debug("TX ARPupd:", eth_h, i, &burstnumberTX);
 897                         }
 898 #endif
 899                 }
 900         }
 901
 902         /* Send non-ARP packets using tlb policy */
 903         if (slave_bufs_pkts[RTE_MAX_ETHPORTS] > 0) {
 904                 num_send = bond_ethdev_tx_burst_tlb(queue,
 905                                 slave_bufs[RTE_MAX_ETHPORTS],
 906                                 slave_bufs_pkts[RTE_MAX_ETHPORTS]);
 907
 908                 for (j = 0; j < slave_bufs_pkts[RTE_MAX_ETHPORTS]; j++) {
 909                         bufs[nb_pkts - 1 - num_not_send - j] =
 910                                         slave_bufs[RTE_MAX_ETHPORTS][nb_pkts - 1 - j];
 911                 }
 912
 913                 num_tx_total += num_send;
 914         }
 915
 916         return num_tx_total;
 917 }
 918
 919 static uint16_t
 920 bond_ethdev_tx_burst_balance(void *queue, struct rte_mbuf **bufs,
 921                 uint16_t nb_pkts)
 922 {
 923         struct bond_dev_private *internals;
 924         struct bond_tx_queue *bd_tx_q;
 925
 926         uint8_t num_of_slaves;
 927         uint8_t slaves[RTE_MAX_ETHPORTS];
 928
 929         uint16_t num_tx_total = 0, num_tx_slave = 0, tx_fail_total = 0;
 930
 931         int i, op_slave_id;
 932
 933         struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts];
 934         uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
 935
 936         bd_tx_q = (struct bond_tx_queue *)queue;
 937         internals = bd_tx_q->dev_private;
 938
 939         /* Copy slave list to protect against slave up/down changes during tx
 940          * bursting */
 941         num_of_slaves = internals->active_slave_count;
 942         memcpy(slaves, internals->active_slaves,
 943                         sizeof(internals->active_slaves[0]) * num_of_slaves);
 944
 945         if (num_of_slaves < 1)
 946                 return num_tx_total;
 947
 948         /* Populate slaves mbuf with the packets which are to be sent on it  */
 949         for (i = 0; i < nb_pkts; i++) {
 950                 /* Select output slave using hash based on xmit policy */
 951                 op_slave_id = internals->xmit_hash(bufs[i], num_of_slaves);
 952
 953                 /* Populate slave mbuf arrays with mbufs for that slave */
 954                 slave_bufs[op_slave_id][slave_nb_pkts[op_slave_id]++] = bufs[i];
 955         }
 956
 957         /* Send packet burst on each slave device */
 958         for (i = 0; i < num_of_slaves; i++) {
 959                 if (slave_nb_pkts[i] > 0) {
 960                         num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
 961                                         slave_bufs[i], slave_nb_pkts[i]);
 962
 963                         /* if tx burst fails move packets to end of bufs */
 964                         if (unlikely(num_tx_slave < slave_nb_pkts[i])) {
 965                                 int slave_tx_fail_count = slave_nb_pkts[i] - num_tx_slave;
 966
 967                                 tx_fail_total += slave_tx_fail_count;
 968                                 memcpy(&bufs[nb_pkts - tx_fail_total],
 969                                                 &slave_bufs[i][num_tx_slave],
 970                                                 slave_tx_fail_count * sizeof(bufs[0]));
 971                         }
 972
 973                         num_tx_total += num_tx_slave;
 974                 }
 975         }
 976
 977         return num_tx_total;
 978 }
 979
 980 static uint16_t
 981 bond_ethdev_tx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 982                 uint16_t nb_pkts)
 983 {
 984         struct bond_dev_private *internals;
 985         struct bond_tx_queue *bd_tx_q;
 986
 987         uint8_t num_of_slaves;
 988         uint8_t slaves[RTE_MAX_ETHPORTS];
 989          /* positions in slaves, not ID */
 990         uint8_t distributing_offsets[RTE_MAX_ETHPORTS];
 991         uint8_t distributing_count;
 992
 993         uint16_t num_tx_slave, num_tx_total = 0, num_tx_fail_total = 0;
 994         uint16_t i, j, op_slave_idx;
 995         const uint16_t buffs_size = nb_pkts + BOND_MODE_8023AX_SLAVE_TX_PKTS + 1;
 996
 997         /* Allocate additional packets in case 8023AD mode. */
 998         struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][buffs_size];
 999         void *slow_pkts[BOND_MODE_8023AX_SLAVE_TX_PKTS] = { NULL };
1000
1001         /* Total amount of packets in slave_bufs */
1002         uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
1003         /* Slow packets placed in each slave */
1004         uint8_t slave_slow_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
1005
1006         bd_tx_q = (struct bond_tx_queue *)queue;
1007         internals = bd_tx_q->dev_private;
1008
1009         /* Copy slave list to protect against slave up/down changes during tx
1010          * bursting */
1011         num_of_slaves = internals->active_slave_count;
1012         if (num_of_slaves < 1)
1013                 return num_tx_total;
1014
1015         memcpy(slaves, internals->active_slaves, sizeof(slaves[0]) * num_of_slaves);
1016
1017         distributing_count = 0;
1018         for (i = 0; i < num_of_slaves; i++) {
1019                 struct port *port = &mode_8023ad_ports[slaves[i]];
1020
1021                 slave_slow_nb_pkts[i] = rte_ring_dequeue_burst(port->tx_ring,
1022                                 slow_pkts, BOND_MODE_8023AX_SLAVE_TX_PKTS,
1023                                 NULL);
1024                 slave_nb_pkts[i] = slave_slow_nb_pkts[i];
1025
1026                 for (j = 0; j < slave_slow_nb_pkts[i]; j++)
1027                         slave_bufs[i][j] = slow_pkts[j];
1028
1029                 if (ACTOR_STATE(port, DISTRIBUTING))
1030                         distributing_offsets[distributing_count++] = i;
1031         }
1032
1033         if (likely(distributing_count > 0)) {
1034                 /* Populate slaves mbuf with the packets which are to be sent on it */
1035                 for (i = 0; i < nb_pkts; i++) {
1036                         /* Select output slave using hash based on xmit policy */
1037                         op_slave_idx = internals->xmit_hash(bufs[i], distributing_count);
1038
1039                         /* Populate slave mbuf arrays with mbufs for that slave. Use only
1040                          * slaves that are currently distributing. */
1041                         uint8_t slave_offset = distributing_offsets[op_slave_idx];
1042                         slave_bufs[slave_offset][slave_nb_pkts[slave_offset]] = bufs[i];
1043                         slave_nb_pkts[slave_offset]++;
1044                 }
1045         }
1046
1047         /* Send packet burst on each slave device */
1048         for (i = 0; i < num_of_slaves; i++) {
1049                 if (slave_nb_pkts[i] == 0)
1050                         continue;
1051
1052                 num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
1053                                 slave_bufs[i], slave_nb_pkts[i]);
1054
1055                 /* If tx burst fails drop slow packets */
1056                 for ( ; num_tx_slave < slave_slow_nb_pkts[i]; num_tx_slave++)
1057                         rte_pktmbuf_free(slave_bufs[i][num_tx_slave]);
1058
1059                 num_tx_total += num_tx_slave - slave_slow_nb_pkts[i];
1060                 num_tx_fail_total += slave_nb_pkts[i] - num_tx_slave;
1061
1062                 /* If tx burst fails move packets to end of bufs */
1063                 if (unlikely(num_tx_slave < slave_nb_pkts[i])) {
1064                         uint16_t j = nb_pkts - num_tx_fail_total;
1065                         for ( ; num_tx_slave < slave_nb_pkts[i]; j++, num_tx_slave++)
1066                                 bufs[j] = slave_bufs[i][num_tx_slave];
1067                 }
1068         }
1069
1070         return num_tx_total;
1071 }
1072
1073 static uint16_t
1074 bond_ethdev_tx_burst_broadcast(void *queue, struct rte_mbuf **bufs,
1075                 uint16_t nb_pkts)
1076 {
1077         struct bond_dev_private *internals;
1078         struct bond_tx_queue *bd_tx_q;
1079
1080         uint8_t tx_failed_flag = 0, num_of_slaves;
1081         uint8_t slaves[RTE_MAX_ETHPORTS];
1082
1083         uint16_t max_nb_of_tx_pkts = 0;
1084
1085         int slave_tx_total[RTE_MAX_ETHPORTS];
1086         int i, most_successful_tx_slave = -1;
1087
1088         bd_tx_q = (struct bond_tx_queue *)queue;
1089         internals = bd_tx_q->dev_private;
1090
1091         /* Copy slave list to protect against slave up/down changes during tx
1092          * bursting */
1093         num_of_slaves = internals->active_slave_count;
1094         memcpy(slaves, internals->active_slaves,
1095                         sizeof(internals->active_slaves[0]) * num_of_slaves);
1096
1097         if (num_of_slaves < 1)
1098                 return 0;
1099
1100         /* Increment reference count on mbufs */
1101         for (i = 0; i < nb_pkts; i++)
1102                 rte_mbuf_refcnt_update(bufs[i], num_of_slaves - 1);
1103
1104         /* Transmit burst on each active slave */
1105         for (i = 0; i < num_of_slaves; i++) {
1106                 slave_tx_total[i] = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
1107                                         bufs, nb_pkts);
1108
1109                 if (unlikely(slave_tx_total[i] < nb_pkts))
1110                         tx_failed_flag = 1;
1111
1112                 /* record the value and slave index for the slave which transmits the
1113                  * maximum number of packets */
1114                 if (slave_tx_total[i] > max_nb_of_tx_pkts) {
1115                         max_nb_of_tx_pkts = slave_tx_total[i];
1116                         most_successful_tx_slave = i;
1117                 }
1118         }
1119
1120         /* if slaves fail to transmit packets from burst, the calling application
1121          * is not expected to know about multiple references to packets so we must
1122          * handle failures of all packets except those of the most successful slave
1123          */
1124         if (unlikely(tx_failed_flag))
1125                 for (i = 0; i < num_of_slaves; i++)
1126                         if (i != most_successful_tx_slave)
1127                                 while (slave_tx_total[i] < nb_pkts)
1128                                         rte_pktmbuf_free(bufs[slave_tx_total[i]++]);
1129
1130         return max_nb_of_tx_pkts;
1131 }
1132
1133 void
1134 link_properties_set(struct rte_eth_dev *bonded_eth_dev,
1135                 struct rte_eth_link *slave_dev_link)
1136 {
1137         struct rte_eth_link *bonded_dev_link = &bonded_eth_dev->data->dev_link;
1138         struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1139
1140         if (slave_dev_link->link_status &&
1141                 bonded_eth_dev->data->dev_started) {
1142                 bonded_dev_link->link_duplex = slave_dev_link->link_duplex;
1143                 bonded_dev_link->link_speed = slave_dev_link->link_speed;
1144
1145                 internals->link_props_set = 1;
1146         }
1147 }
1148
1149 void
1150 link_properties_reset(struct rte_eth_dev *bonded_eth_dev)
1151 {
1152         struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1153
1154         memset(&(bonded_eth_dev->data->dev_link), 0,
1155                         sizeof(bonded_eth_dev->data->dev_link));
1156
1157         internals->link_props_set = 0;
1158 }
1159
1160 int
1161 link_properties_valid(struct rte_eth_link *bonded_dev_link,
1162                 struct rte_eth_link *slave_dev_link)
1163 {
1164         if (bonded_dev_link->link_duplex != slave_dev_link->link_duplex ||
1165                 bonded_dev_link->link_speed !=  slave_dev_link->link_speed)
1166                 return -1;
1167
1168         return 0;
1169 }
1170
1171 int
1172 mac_address_get(struct rte_eth_dev *eth_dev, struct ether_addr *dst_mac_addr)
1173 {
1174         struct ether_addr *mac_addr;
1175
1176         if (eth_dev == NULL) {
1177                 RTE_LOG(ERR, PMD, "%s: NULL pointer eth_dev specified\n", __func__);
1178                 return -1;
1179         }
1180
1181         if (dst_mac_addr == NULL) {
1182                 RTE_LOG(ERR, PMD, "%s: NULL pointer MAC specified\n", __func__);
1183                 return -1;
1184         }
1185
1186         mac_addr = eth_dev->data->mac_addrs;
1187
1188         ether_addr_copy(mac_addr, dst_mac_addr);
1189         return 0;
1190 }
1191
1192 int
1193 mac_address_set(struct rte_eth_dev *eth_dev, struct ether_addr *new_mac_addr)
1194 {
1195         struct ether_addr *mac_addr;
1196
1197         if (eth_dev == NULL) {
1198                 RTE_BOND_LOG(ERR, "NULL pointer eth_dev specified");
1199                 return -1;
1200         }
1201
1202         if (new_mac_addr == NULL) {
1203                 RTE_BOND_LOG(ERR, "NULL pointer MAC specified");
1204                 return -1;
1205         }
1206
1207         mac_addr = eth_dev->data->mac_addrs;
1208
1209         /* If new MAC is different to current MAC then update */
1210         if (memcmp(mac_addr, new_mac_addr, sizeof(*mac_addr)) != 0)
1211                 memcpy(mac_addr, new_mac_addr, sizeof(*mac_addr));
1212
1213         return 0;
1214 }
1215
1216 int
1217 mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev)
1218 {
1219         struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1220         int i;
1221
1222         /* Update slave devices MAC addresses */
1223         if (internals->slave_count < 1)
1224                 return -1;
1225
1226         switch (internals->mode) {
1227         case BONDING_MODE_ROUND_ROBIN:
1228         case BONDING_MODE_BALANCE:
1229         case BONDING_MODE_BROADCAST:
1230                 for (i = 0; i < internals->slave_count; i++) {
1231                         if (mac_address_set(&rte_eth_devices[internals->slaves[i].port_id],
1232                                         bonded_eth_dev->data->mac_addrs)) {
1233                                 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1234                                                 internals->slaves[i].port_id);
1235                                 return -1;
1236                         }
1237                 }
1238                 break;
1239         case BONDING_MODE_8023AD:
1240                 bond_mode_8023ad_mac_address_update(bonded_eth_dev);
1241                 break;
1242         case BONDING_MODE_ACTIVE_BACKUP:
1243         case BONDING_MODE_TLB:
1244         case BONDING_MODE_ALB:
1245         default:
1246                 for (i = 0; i < internals->slave_count; i++) {
1247                         if (internals->slaves[i].port_id ==
1248                                         internals->current_primary_port) {
1249                                 if (mac_address_set(&rte_eth_devices[internals->primary_port],
1250                                                 bonded_eth_dev->data->mac_addrs)) {
1251                                         RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1252                                                         internals->current_primary_port);
1253                                         return -1;
1254                                 }
1255                         } else {
1256                                 if (mac_address_set(
1257                                                 &rte_eth_devices[internals->slaves[i].port_id],
1258                                                 &internals->slaves[i].persisted_mac_addr)) {
1259                                         RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1260                                                         internals->slaves[i].port_id);
1261                                         return -1;
1262                                 }
1263                         }
1264                 }
1265         }
1266
1267         return 0;
1268 }
1269
1270 int
1271 bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode)
1272 {
1273         struct bond_dev_private *internals;
1274
1275         internals = eth_dev->data->dev_private;
1276
1277         switch (mode) {
1278         case BONDING_MODE_ROUND_ROBIN:
1279                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_round_robin;
1280                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1281                 break;
1282         case BONDING_MODE_ACTIVE_BACKUP:
1283                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_active_backup;
1284                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup;
1285                 break;
1286         case BONDING_MODE_BALANCE:
1287                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_balance;
1288                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1289                 break;
1290         case BONDING_MODE_BROADCAST:
1291                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_broadcast;
1292                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1293                 break;
1294         case BONDING_MODE_8023AD:
1295                 if (bond_mode_8023ad_enable(eth_dev) != 0)
1296                         return -1;
1297
1298                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_8023ad;
1299                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_8023ad;
1300                 RTE_LOG(WARNING, PMD,
1301                                 "Using mode 4, it is necessary to do TX burst and RX burst "
1302                                 "at least every 100ms.\n");
1303                 break;
1304         case BONDING_MODE_TLB:
1305                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_tlb;
1306                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup;
1307                 break;
1308         case BONDING_MODE_ALB:
1309                 if (bond_mode_alb_enable(eth_dev) != 0)
1310                         return -1;
1311
1312                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_alb;
1313                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_alb;
1314                 break;
1315         default:
1316                 return -1;
1317         }
1318
1319         internals->mode = mode;
1320
1321         return 0;
1322 }
1323
1324 int
1325 slave_configure(struct rte_eth_dev *bonded_eth_dev,
1326                 struct rte_eth_dev *slave_eth_dev)
1327 {
1328         struct bond_rx_queue *bd_rx_q;
1329         struct bond_tx_queue *bd_tx_q;
1330
1331         int errval;
1332         uint16_t q_id;
1333
1334         /* Stop slave */
1335         rte_eth_dev_stop(slave_eth_dev->data->port_id);
1336
1337         /* Enable interrupts on slave device if supported */
1338         if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1339                 slave_eth_dev->data->dev_conf.intr_conf.lsc = 1;
1340
1341         /* If RSS is enabled for bonding, try to enable it for slaves  */
1342         if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
1343                 if (bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len
1344                                 != 0) {
1345                         slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len =
1346                                         bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len;
1347                         slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key =
1348                                         bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key;
1349                 } else {
1350                         slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1351                 }
1352
1353                 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf =
1354                                 bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
1355                 slave_eth_dev->data->dev_conf.rxmode.mq_mode =
1356                                 bonded_eth_dev->data->dev_conf.rxmode.mq_mode;
1357         }
1358
1359         slave_eth_dev->data->dev_conf.rxmode.hw_vlan_filter =
1360                         bonded_eth_dev->data->dev_conf.rxmode.hw_vlan_filter;
1361
1362         /* Configure device */
1363         errval = rte_eth_dev_configure(slave_eth_dev->data->port_id,
1364                         bonded_eth_dev->data->nb_rx_queues,
1365                         bonded_eth_dev->data->nb_tx_queues,
1366                         &(slave_eth_dev->data->dev_conf));
1367         if (errval != 0) {
1368                 RTE_BOND_LOG(ERR, "Cannot configure slave device: port %u , err (%d)",
1369                                 slave_eth_dev->data->port_id, errval);
1370                 return errval;
1371         }
1372
1373         /* Setup Rx Queues */
1374         for (q_id = 0; q_id < bonded_eth_dev->data->nb_rx_queues; q_id++) {
1375                 bd_rx_q = (struct bond_rx_queue *)bonded_eth_dev->data->rx_queues[q_id];
1376
1377                 errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id, q_id,
1378                                 bd_rx_q->nb_rx_desc,
1379                                 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1380                                 &(bd_rx_q->rx_conf), bd_rx_q->mb_pool);
1381                 if (errval != 0) {
1382                         RTE_BOND_LOG(ERR,
1383                                         "rte_eth_rx_queue_setup: port=%d queue_id %d, err (%d)",
1384                                         slave_eth_dev->data->port_id, q_id, errval);
1385                         return errval;
1386                 }
1387         }
1388
1389         /* Setup Tx Queues */
1390         for (q_id = 0; q_id < bonded_eth_dev->data->nb_tx_queues; q_id++) {
1391                 bd_tx_q = (struct bond_tx_queue *)bonded_eth_dev->data->tx_queues[q_id];
1392
1393                 errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id, q_id,
1394                                 bd_tx_q->nb_tx_desc,
1395                                 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1396                                 &bd_tx_q->tx_conf);
1397                 if (errval != 0) {
1398                         RTE_BOND_LOG(ERR,
1399                                         "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)",
1400                                         slave_eth_dev->data->port_id, q_id, errval);
1401                         return errval;
1402                 }
1403         }
1404
1405         /* Start device */
1406         errval = rte_eth_dev_start(slave_eth_dev->data->port_id);
1407         if (errval != 0) {
1408                 RTE_BOND_LOG(ERR, "rte_eth_dev_start: port=%u, err (%d)",
1409                                 slave_eth_dev->data->port_id, errval);
1410                 return -1;
1411         }
1412
1413         /* If RSS is enabled for bonding, synchronize RETA */
1414         if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) {
1415                 int i;
1416                 struct bond_dev_private *internals;
1417
1418                 internals = bonded_eth_dev->data->dev_private;
1419
1420                 for (i = 0; i < internals->slave_count; i++) {
1421                         if (internals->slaves[i].port_id == slave_eth_dev->data->port_id) {
1422                                 errval = rte_eth_dev_rss_reta_update(
1423                                                 slave_eth_dev->data->port_id,
1424                                                 &internals->reta_conf[0],
1425                                                 internals->slaves[i].reta_size);
1426                                 if (errval != 0) {
1427                                         RTE_LOG(WARNING, PMD,
1428                                                         "rte_eth_dev_rss_reta_update on slave port %d fails (err %d)."
1429                                                         " RSS Configuration for bonding may be inconsistent.\n",
1430                                                         slave_eth_dev->data->port_id, errval);
1431                                 }
1432                                 break;
1433                         }
1434                 }
1435         }
1436
1437         /* If lsc interrupt is set, check initial slave's link status */
1438         if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
1439                 slave_eth_dev->dev_ops->link_update(slave_eth_dev, 0);
1440                 bond_ethdev_lsc_event_callback(slave_eth_dev->data->port_id,
1441                         RTE_ETH_EVENT_INTR_LSC, &bonded_eth_dev->data->port_id);
1442         }
1443
1444         return 0;
1445 }
1446
1447 void
1448 slave_remove(struct bond_dev_private *internals,
1449                 struct rte_eth_dev *slave_eth_dev)
1450 {
1451         uint8_t i;
1452
1453         for (i = 0; i < internals->slave_count; i++)
1454                 if (internals->slaves[i].port_id ==
1455                                 slave_eth_dev->data->port_id)
1456                         break;
1457
1458         if (i < (internals->slave_count - 1))
1459                 memmove(&internals->slaves[i], &internals->slaves[i + 1],
1460                                 sizeof(internals->slaves[0]) *
1461                                 (internals->slave_count - i - 1));
1462
1463         internals->slave_count--;
1464
1465         /* force reconfiguration of slave interfaces */
1466         _rte_eth_dev_reset(slave_eth_dev);
1467 }
1468
1469 static void
1470 bond_ethdev_slave_link_status_change_monitor(void *cb_arg);
1471
1472 void
1473 slave_add(struct bond_dev_private *internals,
1474                 struct rte_eth_dev *slave_eth_dev)
1475 {
1476         struct bond_slave_details *slave_details =
1477                         &internals->slaves[internals->slave_count];
1478
1479         slave_details->port_id = slave_eth_dev->data->port_id;
1480         slave_details->last_link_status = 0;
1481
1482         /* Mark slave devices that don't support interrupts so we can
1483          * compensate when we start the bond
1484          */
1485         if (!(slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
1486                 slave_details->link_status_poll_enabled = 1;
1487         }
1488
1489         slave_details->link_status_wait_to_complete = 0;
1490         /* clean tlb_last_obytes when adding port for bonding device */
1491         memcpy(&(slave_details->persisted_mac_addr), slave_eth_dev->data->mac_addrs,
1492                         sizeof(struct ether_addr));
1493 }
1494
1495 void
1496 bond_ethdev_primary_set(struct bond_dev_private *internals,
1497                 uint8_t slave_port_id)
1498 {
1499         int i;
1500
1501         if (internals->active_slave_count < 1)
1502                 internals->current_primary_port = slave_port_id;
1503         else
1504                 /* Search bonded device slave ports for new proposed primary port */
1505                 for (i = 0; i < internals->active_slave_count; i++) {
1506                         if (internals->active_slaves[i] == slave_port_id)
1507                                 internals->current_primary_port = slave_port_id;
1508                 }
1509 }
1510
1511 static void
1512 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev);
1513
1514 static int
1515 bond_ethdev_start(struct rte_eth_dev *eth_dev)
1516 {
1517         struct bond_dev_private *internals;
1518         int i;
1519
1520         /* slave eth dev will be started by bonded device */
1521         if (check_for_bonded_ethdev(eth_dev)) {
1522                 RTE_BOND_LOG(ERR, "User tried to explicitly start a slave eth_dev (%d)",
1523                                 eth_dev->data->port_id);
1524                 return -1;
1525         }
1526
1527         eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
1528         eth_dev->data->dev_started = 1;
1529
1530         internals = eth_dev->data->dev_private;
1531
1532         if (internals->slave_count == 0) {
1533                 RTE_BOND_LOG(ERR, "Cannot start port since there are no slave devices");
1534                 return -1;
1535         }
1536
1537         if (internals->user_defined_mac == 0) {
1538                 struct ether_addr *new_mac_addr = NULL;
1539
1540                 for (i = 0; i < internals->slave_count; i++)
1541                         if (internals->slaves[i].port_id == internals->primary_port)
1542                                 new_mac_addr = &internals->slaves[i].persisted_mac_addr;
1543
1544                 if (new_mac_addr == NULL)
1545                         return -1;
1546
1547                 if (mac_address_set(eth_dev, new_mac_addr) != 0) {
1548                         RTE_BOND_LOG(ERR, "bonded port (%d) failed to update MAC address",
1549                                         eth_dev->data->port_id);
1550                         return -1;
1551                 }
1552         }
1553
1554         /* Update all slave devices MACs*/
1555         if (mac_address_slaves_update(eth_dev) != 0)
1556                 return -1;
1557
1558         /* If bonded device is configure in promiscuous mode then re-apply config */
1559         if (internals->promiscuous_en)
1560                 bond_ethdev_promiscuous_enable(eth_dev);
1561
1562         /* Reconfigure each slave device if starting bonded device */
1563         for (i = 0; i < internals->slave_count; i++) {
1564                 if (slave_configure(eth_dev,
1565                                 &(rte_eth_devices[internals->slaves[i].port_id])) != 0) {
1566                         RTE_BOND_LOG(ERR,
1567                                         "bonded port (%d) failed to reconfigure slave device (%d)",
1568                                         eth_dev->data->port_id, internals->slaves[i].port_id);
1569                         return -1;
1570                 }
1571                 /* We will need to poll for link status if any slave doesn't
1572                  * support interrupts
1573                  */
1574                 if (internals->slaves[i].link_status_poll_enabled)
1575                         internals->link_status_polling_enabled = 1;
1576         }
1577         /* start polling if needed */
1578         if (internals->link_status_polling_enabled) {
1579                 rte_eal_alarm_set(
1580                         internals->link_status_polling_interval_ms * 1000,
1581                         bond_ethdev_slave_link_status_change_monitor,
1582                         (void *)&rte_eth_devices[internals->port_id]);
1583         }
1584
1585         if (internals->user_defined_primary_port)
1586                 bond_ethdev_primary_set(internals, internals->primary_port);
1587
1588         if (internals->mode == BONDING_MODE_8023AD)
1589                 bond_mode_8023ad_start(eth_dev);
1590
1591         if (internals->mode == BONDING_MODE_TLB ||
1592                         internals->mode == BONDING_MODE_ALB)
1593                 bond_tlb_enable(internals);
1594
1595         return 0;
1596 }
1597
1598 static void
1599 bond_ethdev_free_queues(struct rte_eth_dev *dev)
1600 {
1601         uint8_t i;
1602
1603         if (dev->data->rx_queues != NULL) {
1604                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1605                         rte_free(dev->data->rx_queues[i]);
1606                         dev->data->rx_queues[i] = NULL;
1607                 }
1608                 dev->data->nb_rx_queues = 0;
1609         }
1610
1611         if (dev->data->tx_queues != NULL) {
1612                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1613                         rte_free(dev->data->tx_queues[i]);
1614                         dev->data->tx_queues[i] = NULL;
1615                 }
1616                 dev->data->nb_tx_queues = 0;
1617         }
1618 }
1619
1620 void
1621 bond_ethdev_stop(struct rte_eth_dev *eth_dev)
1622 {
1623         struct bond_dev_private *internals = eth_dev->data->dev_private;
1624         uint8_t i;
1625
1626         if (internals->mode == BONDING_MODE_8023AD) {
1627                 struct port *port;
1628                 void *pkt = NULL;
1629
1630                 bond_mode_8023ad_stop(eth_dev);
1631
1632                 /* Discard all messages to/from mode 4 state machines */
1633                 for (i = 0; i < internals->active_slave_count; i++) {
1634                         port = &mode_8023ad_ports[internals->active_slaves[i]];
1635
1636                         RTE_ASSERT(port->rx_ring != NULL);
1637                         while (rte_ring_dequeue(port->rx_ring, &pkt) != -ENOENT)
1638                                 rte_pktmbuf_free(pkt);
1639
1640                         RTE_ASSERT(port->tx_ring != NULL);
1641                         while (rte_ring_dequeue(port->tx_ring, &pkt) != -ENOENT)
1642                                 rte_pktmbuf_free(pkt);
1643                 }
1644         }
1645
1646         if (internals->mode == BONDING_MODE_TLB ||
1647                         internals->mode == BONDING_MODE_ALB) {
1648                 bond_tlb_disable(internals);
1649                 for (i = 0; i < internals->active_slave_count; i++)
1650                         tlb_last_obytets[internals->active_slaves[i]] = 0;
1651         }
1652
1653         internals->active_slave_count = 0;
1654         internals->link_status_polling_enabled = 0;
1655         for (i = 0; i < internals->slave_count; i++)
1656                 internals->slaves[i].last_link_status = 0;
1657
1658         eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
1659         eth_dev->data->dev_started = 0;
1660 }
1661
1662 void
1663 bond_ethdev_close(struct rte_eth_dev *dev)
1664 {
1665         struct bond_dev_private *internals = dev->data->dev_private;
1666         uint8_t bond_port_id = internals->port_id;
1667         int skipped = 0;
1668
1669         RTE_LOG(INFO, EAL, "Closing bonded device %s\n", dev->data->name);
1670         while (internals->slave_count != skipped) {
1671                 uint8_t port_id = internals->slaves[skipped].port_id;
1672
1673                 rte_eth_dev_stop(port_id);
1674
1675                 if (rte_eth_bond_slave_remove(bond_port_id, port_id) != 0) {
1676                         RTE_LOG(ERR, EAL,
1677                                 "Failed to remove port %d from bonded device "
1678                                 "%s\n", port_id, dev->data->name);
1679                         skipped++;
1680                 }
1681         }
1682         bond_ethdev_free_queues(dev);
1683         rte_bitmap_reset(internals->vlan_filter_bmp);
1684 }
1685
1686 /* forward declaration */
1687 static int bond_ethdev_configure(struct rte_eth_dev *dev);
1688
1689 static void
1690 bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1691 {
1692         struct bond_dev_private *internals = dev->data->dev_private;
1693
1694         dev_info->max_mac_addrs = 1;
1695
1696         dev_info->max_rx_pktlen = internals->candidate_max_rx_pktlen
1697                                   ? internals->candidate_max_rx_pktlen
1698                                   : ETHER_MAX_JUMBO_FRAME_LEN;
1699
1700         dev_info->max_rx_queues = (uint16_t)128;
1701         dev_info->max_tx_queues = (uint16_t)512;
1702
1703         dev_info->min_rx_bufsize = 0;
1704
1705         dev_info->rx_offload_capa = internals->rx_offload_capa;
1706         dev_info->tx_offload_capa = internals->tx_offload_capa;
1707         dev_info->flow_type_rss_offloads = internals->flow_type_rss_offloads;
1708
1709         dev_info->reta_size = internals->reta_size;
1710 }
1711
1712 static int
1713 bond_ethdev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1714 {
1715         int res;
1716         uint8_t i;
1717         struct bond_dev_private *internals = dev->data->dev_private;
1718
1719         /* don't do this while a slave is being added */
1720         rte_spinlock_lock(&internals->lock);
1721
1722         if (on)
1723                 rte_bitmap_set(internals->vlan_filter_bmp, vlan_id);
1724         else
1725                 rte_bitmap_clear(internals->vlan_filter_bmp, vlan_id);
1726
1727         for (i = 0; i < internals->slave_count; i++) {
1728                 uint8_t port_id = internals->slaves[i].port_id;
1729
1730                 res = rte_eth_dev_vlan_filter(port_id, vlan_id, on);
1731                 if (res == ENOTSUP)
1732                         RTE_LOG(WARNING, PMD,
1733                                 "Setting VLAN filter on slave port %u not supported.\n",
1734                                 port_id);
1735         }
1736
1737         rte_spinlock_unlock(&internals->lock);
1738         return 0;
1739 }
1740
1741 static int
1742 bond_ethdev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1743                 uint16_t nb_rx_desc, unsigned int socket_id __rte_unused,
1744                 const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mb_pool)
1745 {
1746         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)
1747                         rte_zmalloc_socket(NULL, sizeof(struct bond_rx_queue),
1748                                         0, dev->data->numa_node);
1749         if (bd_rx_q == NULL)
1750                 return -1;
1751
1752         bd_rx_q->queue_id = rx_queue_id;
1753         bd_rx_q->dev_private = dev->data->dev_private;
1754
1755         bd_rx_q->nb_rx_desc = nb_rx_desc;
1756
1757         memcpy(&(bd_rx_q->rx_conf), rx_conf, sizeof(struct rte_eth_rxconf));
1758         bd_rx_q->mb_pool = mb_pool;
1759
1760         dev->data->rx_queues[rx_queue_id] = bd_rx_q;
1761
1762         return 0;
1763 }
1764
1765 static int
1766 bond_ethdev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1767                 uint16_t nb_tx_desc, unsigned int socket_id __rte_unused,
1768                 const struct rte_eth_txconf *tx_conf)
1769 {
1770         struct bond_tx_queue *bd_tx_q  = (struct bond_tx_queue *)
1771                         rte_zmalloc_socket(NULL, sizeof(struct bond_tx_queue),
1772                                         0, dev->data->numa_node);
1773
1774         if (bd_tx_q == NULL)
1775                 return -1;
1776
1777         bd_tx_q->queue_id = tx_queue_id;
1778         bd_tx_q->dev_private = dev->data->dev_private;
1779
1780         bd_tx_q->nb_tx_desc = nb_tx_desc;
1781         memcpy(&(bd_tx_q->tx_conf), tx_conf, sizeof(bd_tx_q->tx_conf));
1782
1783         dev->data->tx_queues[tx_queue_id] = bd_tx_q;
1784
1785         return 0;
1786 }
1787
1788 static void
1789 bond_ethdev_rx_queue_release(void *queue)
1790 {
1791         if (queue == NULL)
1792                 return;
1793
1794         rte_free(queue);
1795 }
1796
1797 static void
1798 bond_ethdev_tx_queue_release(void *queue)
1799 {
1800         if (queue == NULL)
1801                 return;
1802
1803         rte_free(queue);
1804 }
1805
1806 static void
1807 bond_ethdev_slave_link_status_change_monitor(void *cb_arg)
1808 {
1809         struct rte_eth_dev *bonded_ethdev, *slave_ethdev;
1810         struct bond_dev_private *internals;
1811
1812         /* Default value for polling slave found is true as we don't want to
1813          * disable the polling thread if we cannot get the lock */
1814         int i, polling_slave_found = 1;
1815
1816         if (cb_arg == NULL)
1817                 return;
1818
1819         bonded_ethdev = (struct rte_eth_dev *)cb_arg;
1820         internals = (struct bond_dev_private *)bonded_ethdev->data->dev_private;
1821
1822         if (!bonded_ethdev->data->dev_started ||
1823                 !internals->link_status_polling_enabled)
1824                 return;
1825
1826         /* If device is currently being configured then don't check slaves link
1827          * status, wait until next period */
1828         if (rte_spinlock_trylock(&internals->lock)) {
1829                 if (internals->slave_count > 0)
1830                         polling_slave_found = 0;
1831
1832                 for (i = 0; i < internals->slave_count; i++) {
1833                         if (!internals->slaves[i].link_status_poll_enabled)
1834                                 continue;
1835
1836                         slave_ethdev = &rte_eth_devices[internals->slaves[i].port_id];
1837                         polling_slave_found = 1;
1838
1839                         /* Update slave link status */
1840                         (*slave_ethdev->dev_ops->link_update)(slave_ethdev,
1841                                         internals->slaves[i].link_status_wait_to_complete);
1842
1843                         /* if link status has changed since last checked then call lsc
1844                          * event callback */
1845                         if (slave_ethdev->data->dev_link.link_status !=
1846                                         internals->slaves[i].last_link_status) {
1847                                 internals->slaves[i].last_link_status =
1848                                                 slave_ethdev->data->dev_link.link_status;
1849
1850                                 bond_ethdev_lsc_event_callback(internals->slaves[i].port_id,
1851                                                 RTE_ETH_EVENT_INTR_LSC,
1852                                                 &bonded_ethdev->data->port_id);
1853                         }
1854                 }
1855                 rte_spinlock_unlock(&internals->lock);
1856         }
1857
1858         if (polling_slave_found)
1859                 /* Set alarm to continue monitoring link status of slave ethdev's */
1860                 rte_eal_alarm_set(internals->link_status_polling_interval_ms * 1000,
1861                                 bond_ethdev_slave_link_status_change_monitor, cb_arg);
1862 }
1863
1864 static int
1865 bond_ethdev_link_update(struct rte_eth_dev *bonded_eth_dev,
1866                 int wait_to_complete)
1867 {
1868         struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1869
1870         if (!bonded_eth_dev->data->dev_started ||
1871                 internals->active_slave_count == 0) {
1872                 bonded_eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
1873                 return 0;
1874         } else {
1875                 struct rte_eth_dev *slave_eth_dev;
1876                 int i, link_up = 0;
1877
1878                 for (i = 0; i < internals->active_slave_count; i++) {
1879                         slave_eth_dev = &rte_eth_devices[internals->active_slaves[i]];
1880
1881                         (*slave_eth_dev->dev_ops->link_update)(slave_eth_dev,
1882                                         wait_to_complete);
1883                         if (slave_eth_dev->data->dev_link.link_status == ETH_LINK_UP) {
1884                                 link_up = 1;
1885                                 break;
1886                         }
1887                 }
1888
1889                 bonded_eth_dev->data->dev_link.link_status = link_up;
1890         }
1891
1892         return 0;
1893 }
1894
1895 static void
1896 bond_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1897 {
1898         struct bond_dev_private *internals = dev->data->dev_private;
1899         struct rte_eth_stats slave_stats;
1900         int i, j;
1901
1902         for (i = 0; i < internals->slave_count; i++) {
1903                 rte_eth_stats_get(internals->slaves[i].port_id, &slave_stats);
1904
1905                 stats->ipackets += slave_stats.ipackets;
1906                 stats->opackets += slave_stats.opackets;
1907                 stats->ibytes += slave_stats.ibytes;
1908                 stats->obytes += slave_stats.obytes;
1909                 stats->imissed += slave_stats.imissed;
1910                 stats->ierrors += slave_stats.ierrors;
1911                 stats->oerrors += slave_stats.oerrors;
1912                 stats->rx_nombuf += slave_stats.rx_nombuf;
1913
1914                 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1915                         stats->q_ipackets[j] += slave_stats.q_ipackets[j];
1916                         stats->q_opackets[j] += slave_stats.q_opackets[j];
1917                         stats->q_ibytes[j] += slave_stats.q_ibytes[j];
1918                         stats->q_obytes[j] += slave_stats.q_obytes[j];
1919                         stats->q_errors[j] += slave_stats.q_errors[j];
1920                 }
1921
1922         }
1923 }
1924
1925 static void
1926 bond_ethdev_stats_reset(struct rte_eth_dev *dev)
1927 {
1928         struct bond_dev_private *internals = dev->data->dev_private;
1929         int i;
1930
1931         for (i = 0; i < internals->slave_count; i++)
1932                 rte_eth_stats_reset(internals->slaves[i].port_id);
1933 }
1934
1935 static void
1936 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev)
1937 {
1938         struct bond_dev_private *internals = eth_dev->data->dev_private;
1939         int i;
1940
1941         internals->promiscuous_en = 1;
1942
1943         switch (internals->mode) {
1944         /* Promiscuous mode is propagated to all slaves */
1945         case BONDING_MODE_ROUND_ROBIN:
1946         case BONDING_MODE_BALANCE:
1947         case BONDING_MODE_BROADCAST:
1948                 for (i = 0; i < internals->slave_count; i++)
1949                         rte_eth_promiscuous_enable(internals->slaves[i].port_id);
1950                 break;
1951         /* In mode4 promiscus mode is managed when slave is added/removed */
1952         case BONDING_MODE_8023AD:
1953                 break;
1954         /* Promiscuous mode is propagated only to primary slave */
1955         case BONDING_MODE_ACTIVE_BACKUP:
1956         case BONDING_MODE_TLB:
1957         case BONDING_MODE_ALB:
1958         default:
1959                 rte_eth_promiscuous_enable(internals->current_primary_port);
1960         }
1961 }
1962
1963 static void
1964 bond_ethdev_promiscuous_disable(struct rte_eth_dev *dev)
1965 {
1966         struct bond_dev_private *internals = dev->data->dev_private;
1967         int i;
1968
1969         internals->promiscuous_en = 0;
1970
1971         switch (internals->mode) {
1972         /* Promiscuous mode is propagated to all slaves */
1973         case BONDING_MODE_ROUND_ROBIN:
1974         case BONDING_MODE_BALANCE:
1975         case BONDING_MODE_BROADCAST:
1976                 for (i = 0; i < internals->slave_count; i++)
1977                         rte_eth_promiscuous_disable(internals->slaves[i].port_id);
1978                 break;
1979         /* In mode4 promiscus mode is set managed when slave is added/removed */
1980         case BONDING_MODE_8023AD:
1981                 break;
1982         /* Promiscuous mode is propagated only to primary slave */
1983         case BONDING_MODE_ACTIVE_BACKUP:
1984         case BONDING_MODE_TLB:
1985         case BONDING_MODE_ALB:
1986         default:
1987                 rte_eth_promiscuous_disable(internals->current_primary_port);
1988         }
1989 }
1990
1991 static void
1992 bond_ethdev_delayed_lsc_propagation(void *arg)
1993 {
1994         if (arg == NULL)
1995                 return;
1996
1997         _rte_eth_dev_callback_process((struct rte_eth_dev *)arg,
1998                         RTE_ETH_EVENT_INTR_LSC, NULL);
1999 }
2000
2001 void
2002 bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type,
2003                 void *param)
2004 {
2005         struct rte_eth_dev *bonded_eth_dev, *slave_eth_dev;
2006         struct bond_dev_private *internals;
2007         struct rte_eth_link link;
2008
2009         int i, valid_slave = 0;
2010         uint8_t active_pos;
2011         uint8_t lsc_flag = 0;
2012
2013         if (type != RTE_ETH_EVENT_INTR_LSC || param == NULL)
2014                 return;
2015
2016         bonded_eth_dev = &rte_eth_devices[*(uint8_t *)param];
2017         slave_eth_dev = &rte_eth_devices[port_id];
2018
2019         if (check_for_bonded_ethdev(bonded_eth_dev))
2020                 return;
2021
2022         internals = bonded_eth_dev->data->dev_private;
2023
2024         /* If the device isn't started don't handle interrupts */
2025         if (!bonded_eth_dev->data->dev_started)
2026                 return;
2027
2028         /* verify that port_id is a valid slave of bonded port */
2029         for (i = 0; i < internals->slave_count; i++) {
2030                 if (internals->slaves[i].port_id == port_id) {
2031                         valid_slave = 1;
2032                         break;
2033                 }
2034         }
2035
2036         if (!valid_slave)
2037                 return;
2038
2039         /* Search for port in active port list */
2040         active_pos = find_slave_by_id(internals->active_slaves,
2041                         internals->active_slave_count, port_id);
2042
2043         rte_eth_link_get_nowait(port_id, &link);
2044         if (link.link_status) {
2045                 if (active_pos < internals->active_slave_count)
2046                         return;
2047
2048                 /* if no active slave ports then set this port to be primary port */
2049                 if (internals->active_slave_count < 1) {
2050                         /* If first active slave, then change link status */
2051                         bonded_eth_dev->data->dev_link.link_status = ETH_LINK_UP;
2052                         internals->current_primary_port = port_id;
2053                         lsc_flag = 1;
2054
2055                         mac_address_slaves_update(bonded_eth_dev);
2056
2057                         /* Inherit eth dev link properties from first active slave */
2058                         link_properties_set(bonded_eth_dev,
2059                                         &(slave_eth_dev->data->dev_link));
2060                 } else {
2061                         if (link_properties_valid(
2062                                 &bonded_eth_dev->data->dev_link, &link) != 0) {
2063                                 slave_eth_dev->data->dev_flags &=
2064                                         (~RTE_ETH_DEV_BONDED_SLAVE);
2065                                 RTE_LOG(ERR, PMD,
2066                                         "port %u invalid speed/duplex\n",
2067                                         port_id);
2068                                 return;
2069                         }
2070                 }
2071
2072                 activate_slave(bonded_eth_dev, port_id);
2073
2074                 /* If user has defined the primary port then default to using it */
2075                 if (internals->user_defined_primary_port &&
2076                                 internals->primary_port == port_id)
2077                         bond_ethdev_primary_set(internals, port_id);
2078         } else {
2079                 if (active_pos == internals->active_slave_count)
2080                         return;
2081
2082                 /* Remove from active slave list */
2083                 deactivate_slave(bonded_eth_dev, port_id);
2084
2085                 /* No active slaves, change link status to down and reset other
2086                  * link properties */
2087                 if (internals->active_slave_count < 1) {
2088                         lsc_flag = 1;
2089                         bonded_eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
2090
2091                         link_properties_reset(bonded_eth_dev);
2092                 }
2093
2094                 /* Update primary id, take first active slave from list or if none
2095                  * available set to -1 */
2096                 if (port_id == internals->current_primary_port) {
2097                         if (internals->active_slave_count > 0)
2098                                 bond_ethdev_primary_set(internals,
2099                                                 internals->active_slaves[0]);
2100                         else
2101                                 internals->current_primary_port = internals->primary_port;
2102                 }
2103         }
2104
2105         if (lsc_flag) {
2106                 /* Cancel any possible outstanding interrupts if delays are enabled */
2107                 if (internals->link_up_delay_ms > 0 ||
2108                         internals->link_down_delay_ms > 0)
2109                         rte_eal_alarm_cancel(bond_ethdev_delayed_lsc_propagation,
2110                                         bonded_eth_dev);
2111
2112                 if (bonded_eth_dev->data->dev_link.link_status) {
2113                         if (internals->link_up_delay_ms > 0)
2114                                 rte_eal_alarm_set(internals->link_up_delay_ms * 1000,
2115                                                 bond_ethdev_delayed_lsc_propagation,
2116                                                 (void *)bonded_eth_dev);
2117                         else
2118                                 _rte_eth_dev_callback_process(bonded_eth_dev,
2119                                                 RTE_ETH_EVENT_INTR_LSC, NULL);
2120
2121                 } else {
2122                         if (internals->link_down_delay_ms > 0)
2123                                 rte_eal_alarm_set(internals->link_down_delay_ms * 1000,
2124                                                 bond_ethdev_delayed_lsc_propagation,
2125                                                 (void *)bonded_eth_dev);
2126                         else
2127                                 _rte_eth_dev_callback_process(bonded_eth_dev,
2128                                                 RTE_ETH_EVENT_INTR_LSC, NULL);
2129                 }
2130         }
2131 }
2132
2133 static int
2134 bond_ethdev_rss_reta_update(struct rte_eth_dev *dev,
2135                 struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size)
2136 {
2137         unsigned i, j;
2138         int result = 0;
2139         int slave_reta_size;
2140         unsigned reta_count;
2141         struct bond_dev_private *internals = dev->data->dev_private;
2142
2143         if (reta_size != internals->reta_size)
2144                 return -EINVAL;
2145
2146          /* Copy RETA table */
2147         reta_count = reta_size / RTE_RETA_GROUP_SIZE;
2148
2149         for (i = 0; i < reta_count; i++) {
2150                 internals->reta_conf[i].mask = reta_conf[i].mask;
2151                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
2152                         if ((reta_conf[i].mask >> j) & 0x01)
2153                                 internals->reta_conf[i].reta[j] = reta_conf[i].reta[j];
2154         }
2155
2156         /* Fill rest of array */
2157         for (; i < RTE_DIM(internals->reta_conf); i += reta_count)
2158                 memcpy(&internals->reta_conf[i], &internals->reta_conf[0],
2159                                 sizeof(internals->reta_conf[0]) * reta_count);
2160
2161         /* Propagate RETA over slaves */
2162         for (i = 0; i < internals->slave_count; i++) {
2163                 slave_reta_size = internals->slaves[i].reta_size;
2164                 result = rte_eth_dev_rss_reta_update(internals->slaves[i].port_id,
2165                                 &internals->reta_conf[0], slave_reta_size);
2166                 if (result < 0)
2167                         return result;
2168         }
2169
2170         return 0;
2171 }
2172
2173 static int
2174 bond_ethdev_rss_reta_query(struct rte_eth_dev *dev,
2175                 struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size)
2176 {
2177         int i, j;
2178         struct bond_dev_private *internals = dev->data->dev_private;
2179
2180         if (reta_size != internals->reta_size)
2181                 return -EINVAL;
2182
2183          /* Copy RETA table */
2184         for (i = 0; i < reta_size / RTE_RETA_GROUP_SIZE; i++)
2185                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
2186                         if ((reta_conf[i].mask >> j) & 0x01)
2187                                 reta_conf[i].reta[j] = internals->reta_conf[i].reta[j];
2188
2189         return 0;
2190 }
2191
2192 static int
2193 bond_ethdev_rss_hash_update(struct rte_eth_dev *dev,
2194                 struct rte_eth_rss_conf *rss_conf)
2195 {
2196         int i, result = 0;
2197         struct bond_dev_private *internals = dev->data->dev_private;
2198         struct rte_eth_rss_conf bond_rss_conf;
2199
2200         memcpy(&bond_rss_conf, rss_conf, sizeof(struct rte_eth_rss_conf));
2201
2202         bond_rss_conf.rss_hf &= internals->flow_type_rss_offloads;
2203
2204         if (bond_rss_conf.rss_hf != 0)
2205                 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf = bond_rss_conf.rss_hf;
2206
2207         if (bond_rss_conf.rss_key && bond_rss_conf.rss_key_len <
2208                         sizeof(internals->rss_key)) {
2209                 if (bond_rss_conf.rss_key_len == 0)
2210                         bond_rss_conf.rss_key_len = 40;
2211                 internals->rss_key_len = bond_rss_conf.rss_key_len;
2212                 memcpy(internals->rss_key, bond_rss_conf.rss_key,
2213                                 internals->rss_key_len);
2214         }
2215
2216         for (i = 0; i < internals->slave_count; i++) {
2217                 result = rte_eth_dev_rss_hash_update(internals->slaves[i].port_id,
2218                                 &bond_rss_conf);
2219                 if (result < 0)
2220                         return result;
2221         }
2222
2223         return 0;
2224 }
2225
2226 static int
2227 bond_ethdev_rss_hash_conf_get(struct rte_eth_dev *dev,
2228                 struct rte_eth_rss_conf *rss_conf)
2229 {
2230         struct bond_dev_private *internals = dev->data->dev_private;
2231
2232         rss_conf->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
2233         rss_conf->rss_key_len = internals->rss_key_len;
2234         if (rss_conf->rss_key)
2235                 memcpy(rss_conf->rss_key, internals->rss_key, internals->rss_key_len);
2236
2237         return 0;
2238 }
2239
2240 const struct eth_dev_ops default_dev_ops = {
2241         .dev_start            = bond_ethdev_start,
2242         .dev_stop             = bond_ethdev_stop,
2243         .dev_close            = bond_ethdev_close,
2244         .dev_configure        = bond_ethdev_configure,
2245         .dev_infos_get        = bond_ethdev_info,
2246         .vlan_filter_set      = bond_ethdev_vlan_filter_set,
2247         .rx_queue_setup       = bond_ethdev_rx_queue_setup,
2248         .tx_queue_setup       = bond_ethdev_tx_queue_setup,
2249         .rx_queue_release     = bond_ethdev_rx_queue_release,
2250         .tx_queue_release     = bond_ethdev_tx_queue_release,
2251         .link_update          = bond_ethdev_link_update,
2252         .stats_get            = bond_ethdev_stats_get,
2253         .stats_reset          = bond_ethdev_stats_reset,
2254         .promiscuous_enable   = bond_ethdev_promiscuous_enable,
2255         .promiscuous_disable  = bond_ethdev_promiscuous_disable,
2256         .reta_update          = bond_ethdev_rss_reta_update,
2257         .reta_query           = bond_ethdev_rss_reta_query,
2258         .rss_hash_update      = bond_ethdev_rss_hash_update,
2259         .rss_hash_conf_get    = bond_ethdev_rss_hash_conf_get
2260 };
2261
2262 static int
2263 bond_alloc(struct rte_vdev_device *dev, uint8_t mode)
2264 {
2265         const char *name = rte_vdev_device_name(dev);
2266         uint8_t socket_id = dev->device.numa_node;
2267         struct bond_dev_private *internals = NULL;
2268         struct rte_eth_dev *eth_dev = NULL;
2269         uint32_t vlan_filter_bmp_size;
2270
2271         /* now do all data allocation - for eth_dev structure, dummy pci driver
2272          * and internal (private) data
2273          */
2274
2275         if (socket_id >= number_of_sockets()) {
2276                 RTE_BOND_LOG(ERR,
2277                                 "Invalid socket id specified to create bonded device on.");
2278                 goto err;
2279         }
2280
2281         /* reserve an ethdev entry */
2282         eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internals));
2283         if (eth_dev == NULL) {
2284                 RTE_BOND_LOG(ERR, "Unable to allocate rte_eth_dev");
2285                 goto err;
2286         }
2287
2288         internals = eth_dev->data->dev_private;
2289         eth_dev->data->nb_rx_queues = (uint16_t)1;
2290         eth_dev->data->nb_tx_queues = (uint16_t)1;
2291
2292         eth_dev->data->mac_addrs = rte_zmalloc_socket(name, ETHER_ADDR_LEN, 0,
2293                         socket_id);
2294         if (eth_dev->data->mac_addrs == NULL) {
2295                 RTE_BOND_LOG(ERR, "Unable to malloc mac_addrs");
2296                 goto err;
2297         }
2298
2299         eth_dev->dev_ops = &default_dev_ops;
2300         eth_dev->data->dev_flags = RTE_ETH_DEV_INTR_LSC |
2301                 RTE_ETH_DEV_DETACHABLE;
2302
2303         rte_spinlock_init(&internals->lock);
2304
2305         internals->port_id = eth_dev->data->port_id;
2306         internals->mode = BONDING_MODE_INVALID;
2307         internals->current_primary_port = RTE_MAX_ETHPORTS + 1;
2308         internals->balance_xmit_policy = BALANCE_XMIT_POLICY_LAYER2;
2309         internals->xmit_hash = xmit_l2_hash;
2310         internals->user_defined_mac = 0;
2311         internals->link_props_set = 0;
2312
2313         internals->link_status_polling_enabled = 0;
2314
2315         internals->link_status_polling_interval_ms =
2316                 DEFAULT_POLLING_INTERVAL_10_MS;
2317         internals->link_down_delay_ms = 0;
2318         internals->link_up_delay_ms = 0;
2319
2320         internals->slave_count = 0;
2321         internals->active_slave_count = 0;
2322         internals->rx_offload_capa = 0;
2323         internals->tx_offload_capa = 0;
2324         internals->candidate_max_rx_pktlen = 0;
2325         internals->max_rx_pktlen = 0;
2326
2327         /* Initially allow to choose any offload type */
2328         internals->flow_type_rss_offloads = ETH_RSS_PROTO_MASK;
2329
2330         memset(internals->active_slaves, 0, sizeof(internals->active_slaves));
2331         memset(internals->slaves, 0, sizeof(internals->slaves));
2332
2333         /* Set mode 4 default configuration */
2334         bond_mode_8023ad_setup(eth_dev, NULL);
2335         if (bond_ethdev_mode_set(eth_dev, mode)) {
2336                 RTE_BOND_LOG(ERR, "Failed to set bonded device %d mode too %d",
2337                                  eth_dev->data->port_id, mode);
2338                 goto err;
2339         }
2340
2341         vlan_filter_bmp_size =
2342                 rte_bitmap_get_memory_footprint(ETHER_MAX_VLAN_ID + 1);
2343         internals->vlan_filter_bmpmem = rte_malloc(name, vlan_filter_bmp_size,
2344                                                    RTE_CACHE_LINE_SIZE);
2345         if (internals->vlan_filter_bmpmem == NULL) {
2346                 RTE_BOND_LOG(ERR,
2347                              "Failed to allocate vlan bitmap for bonded device %u\n",
2348                              eth_dev->data->port_id);
2349                 goto err;
2350         }
2351
2352         internals->vlan_filter_bmp = rte_bitmap_init(ETHER_MAX_VLAN_ID + 1,
2353                         internals->vlan_filter_bmpmem, vlan_filter_bmp_size);
2354         if (internals->vlan_filter_bmp == NULL) {
2355                 RTE_BOND_LOG(ERR,
2356                              "Failed to init vlan bitmap for bonded device %u\n",
2357                              eth_dev->data->port_id);
2358                 rte_free(internals->vlan_filter_bmpmem);
2359                 goto err;
2360         }
2361
2362         return eth_dev->data->port_id;
2363
2364 err:
2365         rte_free(internals);
2366         if (eth_dev != NULL) {
2367                 rte_free(eth_dev->data->mac_addrs);
2368                 rte_eth_dev_release_port(eth_dev);
2369         }
2370         return -1;
2371 }
2372
2373 static int
2374 bond_probe(struct rte_vdev_device *dev)
2375 {
2376         const char *name;
2377         struct bond_dev_private *internals;
2378         struct rte_kvargs *kvlist;
2379         uint8_t bonding_mode, socket_id;
2380         int  arg_count, port_id;
2381
2382         if (!dev)
2383                 return -EINVAL;
2384
2385         name = rte_vdev_device_name(dev);
2386         RTE_LOG(INFO, EAL, "Initializing pmd_bond for %s\n", name);
2387
2388         kvlist = rte_kvargs_parse(rte_vdev_device_args(dev),
2389                 pmd_bond_init_valid_arguments);
2390         if (kvlist == NULL)
2391                 return -1;
2392
2393         /* Parse link bonding mode */
2394         if (rte_kvargs_count(kvlist, PMD_BOND_MODE_KVARG) == 1) {
2395                 if (rte_kvargs_process(kvlist, PMD_BOND_MODE_KVARG,
2396                                 &bond_ethdev_parse_slave_mode_kvarg,
2397                                 &bonding_mode) != 0) {
2398                         RTE_LOG(ERR, EAL, "Invalid mode for bonded device %s\n",
2399                                         name);
2400                         goto parse_error;
2401                 }
2402         } else {
2403                 RTE_LOG(ERR, EAL, "Mode must be specified only once for bonded "
2404                                 "device %s\n", name);
2405                 goto parse_error;
2406         }
2407
2408         /* Parse socket id to create bonding device on */
2409         arg_count = rte_kvargs_count(kvlist, PMD_BOND_SOCKET_ID_KVARG);
2410         if (arg_count == 1) {
2411                 if (rte_kvargs_process(kvlist, PMD_BOND_SOCKET_ID_KVARG,
2412                                 &bond_ethdev_parse_socket_id_kvarg, &socket_id)
2413                                 != 0) {
2414                         RTE_LOG(ERR, EAL, "Invalid socket Id specified for "
2415                                         "bonded device %s\n", name);
2416                         goto parse_error;
2417                 }
2418         } else if (arg_count > 1) {
2419                 RTE_LOG(ERR, EAL, "Socket Id can be specified only once for "
2420                                 "bonded device %s\n", name);
2421                 goto parse_error;
2422         } else {
2423                 socket_id = rte_socket_id();
2424         }
2425
2426         dev->device.numa_node = socket_id;
2427
2428         /* Create link bonding eth device */
2429         port_id = bond_alloc(dev, bonding_mode);
2430         if (port_id < 0) {
2431                 RTE_LOG(ERR, EAL, "Failed to create socket %s in mode %u on "
2432                                 "socket %u.\n", name, bonding_mode, socket_id);
2433                 goto parse_error;
2434         }
2435         internals = rte_eth_devices[port_id].data->dev_private;
2436         internals->kvlist = kvlist;
2437
2438         RTE_LOG(INFO, EAL, "Create bonded device %s on port %d in mode %u on "
2439                         "socket %u.\n", name, port_id, bonding_mode, socket_id);
2440         return 0;
2441
2442 parse_error:
2443         rte_kvargs_free(kvlist);
2444
2445         return -1;
2446 }
2447
2448 static int
2449 bond_remove(struct rte_vdev_device *dev)
2450 {
2451         struct rte_eth_dev *eth_dev;
2452         struct bond_dev_private *internals;
2453         const char *name;
2454
2455         if (!dev)
2456                 return -EINVAL;
2457
2458         name = rte_vdev_device_name(dev);
2459         RTE_LOG(INFO, EAL, "Uninitializing pmd_bond for %s\n", name);
2460
2461         /* now free all data allocation - for eth_dev structure,
2462          * dummy pci driver and internal (private) data
2463          */
2464
2465         /* find an ethdev entry */
2466         eth_dev = rte_eth_dev_allocated(name);
2467         if (eth_dev == NULL)
2468                 return -ENODEV;
2469
2470         RTE_ASSERT(eth_dev->device == &dev->device);
2471
2472         internals = eth_dev->data->dev_private;
2473         if (internals->slave_count != 0)
2474                 return -EBUSY;
2475
2476         if (eth_dev->data->dev_started == 1) {
2477                 bond_ethdev_stop(eth_dev);
2478                 bond_ethdev_close(eth_dev);
2479         }
2480
2481         eth_dev->dev_ops = NULL;
2482         eth_dev->rx_pkt_burst = NULL;
2483         eth_dev->tx_pkt_burst = NULL;
2484
2485         internals = eth_dev->data->dev_private;
2486         rte_bitmap_free(internals->vlan_filter_bmp);
2487         rte_free(internals->vlan_filter_bmpmem);
2488         rte_free(eth_dev->data->dev_private);
2489         rte_free(eth_dev->data->mac_addrs);
2490
2491         rte_eth_dev_release_port(eth_dev);
2492
2493         return 0;
2494 }
2495
2496 /* this part will resolve the slave portids after all the other pdev and vdev
2497  * have been allocated */
2498 static int
2499 bond_ethdev_configure(struct rte_eth_dev *dev)
2500 {
2501         char *name = dev->data->name;
2502         struct bond_dev_private *internals = dev->data->dev_private;
2503         struct rte_kvargs *kvlist = internals->kvlist;
2504         int arg_count;
2505         uint8_t port_id = dev - rte_eth_devices;
2506
2507         static const uint8_t default_rss_key[40] = {
2508                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 0x41, 0x67, 0x25, 0x3D,
2509                 0x43, 0xA3, 0x8F, 0xB0, 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
2510                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 0x6A, 0x42, 0xB7, 0x3B,
2511                 0xBE, 0xAC, 0x01, 0xFA
2512         };
2513
2514         unsigned i, j;
2515
2516         /* If RSS is enabled, fill table and key with default values */
2517         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) {
2518                 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = internals->rss_key;
2519                 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len = 0;
2520                 memcpy(internals->rss_key, default_rss_key, 40);
2521
2522                 for (i = 0; i < RTE_DIM(internals->reta_conf); i++) {
2523                         internals->reta_conf[i].mask = ~0LL;
2524                         for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
2525                                 internals->reta_conf[i].reta[j] = j % dev->data->nb_rx_queues;
2526                 }
2527         }
2528
2529         /* set the max_rx_pktlen */
2530         internals->max_rx_pktlen = internals->candidate_max_rx_pktlen;
2531
2532         /*
2533          * if no kvlist, it means that this bonded device has been created
2534          * through the bonding api.
2535          */
2536         if (!kvlist)
2537                 return 0;
2538
2539         /* Parse MAC address for bonded device */
2540         arg_count = rte_kvargs_count(kvlist, PMD_BOND_MAC_ADDR_KVARG);
2541         if (arg_count == 1) {
2542                 struct ether_addr bond_mac;
2543
2544                 if (rte_kvargs_process(kvlist, PMD_BOND_MAC_ADDR_KVARG,
2545                                 &bond_ethdev_parse_bond_mac_addr_kvarg, &bond_mac) < 0) {
2546                         RTE_LOG(INFO, EAL, "Invalid mac address for bonded device %s\n",
2547                                         name);
2548                         return -1;
2549                 }
2550
2551                 /* Set MAC address */
2552                 if (rte_eth_bond_mac_address_set(port_id, &bond_mac) != 0) {
2553                         RTE_LOG(ERR, EAL,
2554                                         "Failed to set mac address on bonded device %s\n",
2555                                         name);
2556                         return -1;
2557                 }
2558         } else if (arg_count > 1) {
2559                 RTE_LOG(ERR, EAL,
2560                                 "MAC address can be specified only once for bonded device %s\n",
2561                                 name);
2562                 return -1;
2563         }
2564
2565         /* Parse/set balance mode transmit policy */
2566         arg_count = rte_kvargs_count(kvlist, PMD_BOND_XMIT_POLICY_KVARG);
2567         if (arg_count == 1) {
2568                 uint8_t xmit_policy;
2569
2570                 if (rte_kvargs_process(kvlist, PMD_BOND_XMIT_POLICY_KVARG,
2571                                 &bond_ethdev_parse_balance_xmit_policy_kvarg, &xmit_policy) !=
2572                                                 0) {
2573                         RTE_LOG(INFO, EAL,
2574                                         "Invalid xmit policy specified for bonded device %s\n",
2575                                         name);
2576                         return -1;
2577                 }
2578
2579                 /* Set balance mode transmit policy*/
2580                 if (rte_eth_bond_xmit_policy_set(port_id, xmit_policy) != 0) {
2581                         RTE_LOG(ERR, EAL,
2582                                         "Failed to set balance xmit policy on bonded device %s\n",
2583                                         name);
2584                         return -1;
2585                 }
2586         } else if (arg_count > 1) {
2587                 RTE_LOG(ERR, EAL,
2588                                 "Transmit policy can be specified only once for bonded device"
2589                                 " %s\n", name);
2590                 return -1;
2591         }
2592
2593         /* Parse/add slave ports to bonded device */
2594         if (rte_kvargs_count(kvlist, PMD_BOND_SLAVE_PORT_KVARG) > 0) {
2595                 struct bond_ethdev_slave_ports slave_ports;
2596                 unsigned i;
2597
2598                 memset(&slave_ports, 0, sizeof(slave_ports));
2599
2600                 if (rte_kvargs_process(kvlist, PMD_BOND_SLAVE_PORT_KVARG,
2601                                 &bond_ethdev_parse_slave_port_kvarg, &slave_ports) != 0) {
2602                         RTE_LOG(ERR, EAL,
2603                                         "Failed to parse slave ports for bonded device %s\n",
2604                                         name);
2605                         return -1;
2606                 }
2607
2608                 for (i = 0; i < slave_ports.slave_count; i++) {
2609                         if (rte_eth_bond_slave_add(port_id, slave_ports.slaves[i]) != 0) {
2610                                 RTE_LOG(ERR, EAL,
2611                                                 "Failed to add port %d as slave to bonded device %s\n",
2612                                                 slave_ports.slaves[i], name);
2613                         }
2614                 }
2615
2616         } else {
2617                 RTE_LOG(INFO, EAL, "No slaves specified for bonded device %s\n", name);
2618                 return -1;
2619         }
2620
2621         /* Parse/set primary slave port id*/
2622         arg_count = rte_kvargs_count(kvlist, PMD_BOND_PRIMARY_SLAVE_KVARG);
2623         if (arg_count == 1) {
2624                 uint8_t primary_slave_port_id;
2625
2626                 if (rte_kvargs_process(kvlist,
2627                                 PMD_BOND_PRIMARY_SLAVE_KVARG,
2628                                 &bond_ethdev_parse_primary_slave_port_id_kvarg,
2629                                 &primary_slave_port_id) < 0) {
2630                         RTE_LOG(INFO, EAL,
2631                                         "Invalid primary slave port id specified for bonded device"
2632                                         " %s\n", name);
2633                         return -1;
2634                 }
2635
2636                 /* Set balance mode transmit policy*/
2637                 if (rte_eth_bond_primary_set(port_id, (uint8_t)primary_slave_port_id)
2638                                 != 0) {
2639                         RTE_LOG(ERR, EAL,
2640                                         "Failed to set primary slave port %d on bonded device %s\n",
2641                                         primary_slave_port_id, name);
2642                         return -1;
2643                 }
2644         } else if (arg_count > 1) {
2645                 RTE_LOG(INFO, EAL,
2646                                 "Primary slave can be specified only once for bonded device"
2647                                 " %s\n", name);
2648                 return -1;
2649         }
2650
2651         /* Parse link status monitor polling interval */
2652         arg_count = rte_kvargs_count(kvlist, PMD_BOND_LSC_POLL_PERIOD_KVARG);
2653         if (arg_count == 1) {
2654                 uint32_t lsc_poll_interval_ms;
2655
2656                 if (rte_kvargs_process(kvlist,
2657                                 PMD_BOND_LSC_POLL_PERIOD_KVARG,
2658                                 &bond_ethdev_parse_time_ms_kvarg,
2659                                 &lsc_poll_interval_ms) < 0) {
2660                         RTE_LOG(INFO, EAL,
2661                                         "Invalid lsc polling interval value specified for bonded"
2662                                         " device %s\n", name);
2663                         return -1;
2664                 }
2665
2666                 if (rte_eth_bond_link_monitoring_set(port_id, lsc_poll_interval_ms)
2667                                 != 0) {
2668                         RTE_LOG(ERR, EAL,
2669                                         "Failed to set lsc monitor polling interval (%u ms) on"
2670                                         " bonded device %s\n", lsc_poll_interval_ms, name);
2671                         return -1;
2672                 }
2673         } else if (arg_count > 1) {
2674                 RTE_LOG(INFO, EAL,
2675                                 "LSC polling interval can be specified only once for bonded"
2676                                 " device %s\n", name);
2677                 return -1;
2678         }
2679
2680         /* Parse link up interrupt propagation delay */
2681         arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_UP_PROP_DELAY_KVARG);
2682         if (arg_count == 1) {
2683                 uint32_t link_up_delay_ms;
2684
2685                 if (rte_kvargs_process(kvlist,
2686                                 PMD_BOND_LINK_UP_PROP_DELAY_KVARG,
2687                                 &bond_ethdev_parse_time_ms_kvarg,
2688                                 &link_up_delay_ms) < 0) {
2689                         RTE_LOG(INFO, EAL,
2690                                         "Invalid link up propagation delay value specified for"
2691                                         " bonded device %s\n", name);
2692                         return -1;
2693                 }
2694
2695                 /* Set balance mode transmit policy*/
2696                 if (rte_eth_bond_link_up_prop_delay_set(port_id, link_up_delay_ms)
2697                                 != 0) {
2698                         RTE_LOG(ERR, EAL,
2699                                         "Failed to set link up propagation delay (%u ms) on bonded"
2700                                         " device %s\n", link_up_delay_ms, name);
2701                         return -1;
2702                 }
2703         } else if (arg_count > 1) {
2704                 RTE_LOG(INFO, EAL,
2705                                 "Link up propagation delay can be specified only once for"
2706                                 " bonded device %s\n", name);
2707                 return -1;
2708         }
2709
2710         /* Parse link down interrupt propagation delay */
2711         arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG);
2712         if (arg_count == 1) {
2713                 uint32_t link_down_delay_ms;
2714
2715                 if (rte_kvargs_process(kvlist,
2716                                 PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG,
2717                                 &bond_ethdev_parse_time_ms_kvarg,
2718                                 &link_down_delay_ms) < 0) {
2719                         RTE_LOG(INFO, EAL,
2720                                         "Invalid link down propagation delay value specified for"
2721                                         " bonded device %s\n", name);
2722                         return -1;
2723                 }
2724
2725                 /* Set balance mode transmit policy*/
2726                 if (rte_eth_bond_link_down_prop_delay_set(port_id, link_down_delay_ms)
2727                                 != 0) {
2728                         RTE_LOG(ERR, EAL,
2729                                         "Failed to set link down propagation delay (%u ms) on"
2730                                         " bonded device %s\n", link_down_delay_ms, name);
2731                         return -1;
2732                 }
2733         } else if (arg_count > 1) {
2734                 RTE_LOG(INFO, EAL,
2735                                 "Link down propagation delay can be specified only once for"
2736                                 " bonded device %s\n", name);
2737                 return -1;
2738         }
2739
2740         return 0;
2741 }
2742
2743 struct rte_vdev_driver pmd_bond_drv = {
2744         .probe = bond_probe,
2745         .remove = bond_remove,
2746 };
2747
2748 RTE_PMD_REGISTER_VDEV(net_bonding, pmd_bond_drv);
2749 RTE_PMD_REGISTER_ALIAS(net_bonding, eth_bond);
2750
2751 RTE_PMD_REGISTER_PARAM_STRING(net_bonding,
2752         "slave=<ifc> "
2753         "primary=<ifc> "
2754         "mode=[0-6] "
2755         "xmit_policy=[l2 | l23 | l34] "
2756         "socket_id=<int> "
2757         "mac=<mac addr> "
2758         "lsc_poll_period_ms=<int> "
2759         "up_delay=<int> "
2760         "down_delay=<int>");