lib/librte_gro/rte_gro.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_malloc.h>
  34 #include <rte_mbuf.h>
  35 #include <rte_cycles.h>
  36 #include <rte_ethdev.h>
  37
  38 #include "rte_gro.h"
  39 #include "gro_tcp4.h"
  40
  41 typedef void *(*gro_tbl_create_fn)(uint16_t socket_id,
  42                 uint16_t max_flow_num,
  43                 uint16_t max_item_per_flow);
  44 typedef void (*gro_tbl_destroy_fn)(void *tbl);
  45 typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
  46
  47 static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
  48                 gro_tcp4_tbl_create, NULL};
  49 static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
  50                         gro_tcp4_tbl_destroy, NULL};
  51 static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
  52                         gro_tcp4_tbl_pkt_count, NULL};
  53
  54 /*
  55  * GRO context structure, which is used to merge packets. It keeps
  56  * many reassembly tables of desired GRO types. Applications need to
  57  * create GRO context objects before using rte_gro_reassemble to
  58  * perform GRO.
  59  */
  60 struct gro_ctx {
  61         /* GRO types to perform */
  62         uint64_t gro_types;
  63         /* reassembly tables */
  64         void *tbls[RTE_GRO_TYPE_MAX_NUM];
  65 };
  66
  67 void *
  68 rte_gro_ctx_create(const struct rte_gro_param *param)
  69 {
  70         struct gro_ctx *gro_ctx;
  71         gro_tbl_create_fn create_tbl_fn;
  72         uint64_t gro_type_flag = 0;
  73         uint64_t gro_types = 0;
  74         uint8_t i;
  75
  76         gro_ctx = rte_zmalloc_socket(__func__,
  77                         sizeof(struct gro_ctx),
  78                         RTE_CACHE_LINE_SIZE,
  79                         param->socket_id);
  80         if (gro_ctx == NULL)
  81                 return NULL;
  82
  83         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
  84                 gro_type_flag = 1ULL << i;
  85                 if ((param->gro_types & gro_type_flag) == 0)
  86                         continue;
  87
  88                 create_tbl_fn = tbl_create_fn[i];
  89                 if (create_tbl_fn == NULL)
  90                         continue;
  91
  92                 gro_ctx->tbls[i] = create_tbl_fn(param->socket_id,
  93                                 param->max_flow_num,
  94                                 param->max_item_per_flow);
  95                 if (gro_ctx->tbls[i] == NULL) {
  96                         /* destroy all created tables */
  97                         gro_ctx->gro_types = gro_types;
  98                         rte_gro_ctx_destroy(gro_ctx);
  99                         return NULL;
 100                 }
 101                 gro_types |= gro_type_flag;
 102         }
 103         gro_ctx->gro_types = param->gro_types;
 104
 105         return gro_ctx;
 106 }
 107
 108 void
 109 rte_gro_ctx_destroy(void *ctx)
 110 {
 111         gro_tbl_destroy_fn destroy_tbl_fn;
 112         struct gro_ctx *gro_ctx = ctx;
 113         uint64_t gro_type_flag;
 114         uint8_t i;
 115
 116         if (gro_ctx == NULL)
 117                 return;
 118         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
 119                 gro_type_flag = 1ULL << i;
 120                 if ((gro_ctx->gro_types & gro_type_flag) == 0)
 121                         continue;
 122                 destroy_tbl_fn = tbl_destroy_fn[i];
 123                 if (destroy_tbl_fn)
 124                         destroy_tbl_fn(gro_ctx->tbls[i]);
 125         }
 126         rte_free(gro_ctx);
 127 }
 128
 129 uint16_t
 130 rte_gro_reassemble_burst(struct rte_mbuf **pkts,
 131                 uint16_t nb_pkts,
 132                 const struct rte_gro_param *param)
 133 {
 134         uint16_t i;
 135         uint16_t nb_after_gro = nb_pkts;
 136         uint32_t item_num;
 137
 138         /* allocate a reassembly table for TCP/IPv4 GRO */
 139         struct gro_tcp4_tbl tcp_tbl;
 140         struct gro_tcp4_key tcp_keys[RTE_GRO_MAX_BURST_ITEM_NUM];
 141         struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
 142
 143         struct rte_mbuf *unprocess_pkts[nb_pkts];
 144         uint16_t unprocess_num = 0;
 145         int32_t ret;
 146         uint64_t current_time;
 147
 148         if ((param->gro_types & RTE_GRO_TCP_IPV4) == 0)
 149                 return nb_pkts;
 150
 151         /* get the actual number of packets */
 152         item_num = RTE_MIN(nb_pkts, (param->max_flow_num *
 153                         param->max_item_per_flow));
 154         item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM);
 155
 156         for (i = 0; i < item_num; i++)
 157                 tcp_keys[i].start_index = INVALID_ARRAY_INDEX;
 158
 159         tcp_tbl.keys = tcp_keys;
 160         tcp_tbl.items = tcp_items;
 161         tcp_tbl.key_num = 0;
 162         tcp_tbl.item_num = 0;
 163         tcp_tbl.max_key_num = item_num;
 164         tcp_tbl.max_item_num = item_num;
 165
 166         current_time = rte_rdtsc();
 167
 168         for (i = 0; i < nb_pkts; i++) {
 169                 if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 |
 170                                         RTE_PTYPE_L4_TCP)) ==
 171                                 (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) {
 172                         ret = gro_tcp4_reassemble(pkts[i],
 173                                         &tcp_tbl,
 174                                         current_time);
 175                         if (ret > 0)
 176                                 /* merge successfully */
 177                                 nb_after_gro--;
 178                         else if (ret < 0) {
 179                                 unprocess_pkts[unprocess_num++] =
 180                                         pkts[i];
 181                         }
 182                 } else
 183                         unprocess_pkts[unprocess_num++] = pkts[i];
 184         }
 185
 186         /* re-arrange GROed packets */
 187         if (nb_after_gro < nb_pkts) {
 188                 i = gro_tcp4_tbl_timeout_flush(&tcp_tbl, current_time,
 189                                 pkts, nb_pkts);
 190                 if (unprocess_num > 0) {
 191                         memcpy(&pkts[i], unprocess_pkts,
 192                                         sizeof(struct rte_mbuf *) *
 193                                         unprocess_num);
 194                 }
 195         }
 196
 197         return nb_after_gro;
 198 }
 199
 200 uint16_t
 201 rte_gro_reassemble(struct rte_mbuf **pkts,
 202                 uint16_t nb_pkts,
 203                 void *ctx)
 204 {
 205         uint16_t i, unprocess_num = 0;
 206         struct rte_mbuf *unprocess_pkts[nb_pkts];
 207         struct gro_ctx *gro_ctx = ctx;
 208         uint64_t current_time;
 209
 210         if ((gro_ctx->gro_types & RTE_GRO_TCP_IPV4) == 0)
 211                 return nb_pkts;
 212
 213         current_time = rte_rdtsc();
 214
 215         for (i = 0; i < nb_pkts; i++) {
 216                 if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 |
 217                                         RTE_PTYPE_L4_TCP)) ==
 218                                 (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) {
 219                         if (gro_tcp4_reassemble(pkts[i],
 220                                                 gro_ctx->tbls
 221                                                 [RTE_GRO_TCP_IPV4_INDEX],
 222                                                 current_time) < 0)
 223                                 unprocess_pkts[unprocess_num++] = pkts[i];
 224                 } else
 225                         unprocess_pkts[unprocess_num++] = pkts[i];
 226         }
 227         if (unprocess_num > 0) {
 228                 memcpy(pkts, unprocess_pkts,
 229                                 sizeof(struct rte_mbuf *) *
 230                                 unprocess_num);
 231         }
 232
 233         return unprocess_num;
 234 }
 235
 236 uint16_t
 237 rte_gro_timeout_flush(void *ctx,
 238                 uint64_t timeout_cycles,
 239                 uint64_t gro_types,
 240                 struct rte_mbuf **out,
 241                 uint16_t max_nb_out)
 242 {
 243         struct gro_ctx *gro_ctx = ctx;
 244         uint64_t flush_timestamp;
 245
 246         gro_types = gro_types & gro_ctx->gro_types;
 247         flush_timestamp = rte_rdtsc() - timeout_cycles;
 248
 249         if (gro_types & RTE_GRO_TCP_IPV4) {
 250                 return gro_tcp4_tbl_timeout_flush(
 251                                 gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX],
 252                                 flush_timestamp,
 253                                 out, max_nb_out);
 254         }
 255         return 0;
 256 }
 257
 258 uint64_t
 259 rte_gro_get_pkt_count(void *ctx)
 260 {
 261         struct gro_ctx *gro_ctx = ctx;
 262         gro_tbl_pkt_count_fn pkt_count_fn;
 263         uint64_t item_num = 0;
 264         uint64_t gro_type_flag;
 265         uint8_t i;
 266
 267         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
 268                 gro_type_flag = 1ULL << i;
 269                 if ((gro_ctx->gro_types & gro_type_flag) == 0)
 270                         continue;
 271
 272                 pkt_count_fn = tbl_pkt_count_fn[i];
 273                 if (pkt_count_fn == NULL)
 274                         continue;
 275                 item_num += pkt_count_fn(gro_ctx->tbls[i]);
 276         }
 277         return item_num;
 278 }