lib/librte_latencystats/rte_latencystats.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <unistd.h>
   6 #include <sys/types.h>
   7 #include <stdbool.h>
   8 #include <math.h>
   9
  10 #include <rte_mbuf.h>
  11 #include <rte_log.h>
  12 #include <rte_cycles.h>
  13 #include <rte_ethdev.h>
  14 #include <rte_metrics.h>
  15 #include <rte_memzone.h>
  16 #include <rte_lcore.h>
  17
  18 #include "rte_latencystats.h"
  19
  20 /** Nano seconds per second */
  21 #define NS_PER_SEC 1E9
  22
  23 /** Clock cycles per nano second */
  24 static uint64_t
  25 latencystat_cycles_per_ns(void)
  26 {
  27         return rte_get_timer_hz() / NS_PER_SEC;
  28 }
  29
  30 /* Macros for printing using RTE_LOG */
  31 #define RTE_LOGTYPE_LATENCY_STATS RTE_LOGTYPE_USER1
  32
  33 static const char *MZ_RTE_LATENCY_STATS = "rte_latencystats";
  34 static int latency_stats_index;
  35 static uint64_t samp_intvl;
  36 static uint64_t timer_tsc;
  37 static uint64_t prev_tsc;
  38
  39 struct rte_latency_stats {
  40         float min_latency; /**< Minimum latency in nano seconds */
  41         float avg_latency; /**< Average latency in nano seconds */
  42         float max_latency; /**< Maximum latency in nano seconds */
  43         float jitter; /** Latency variation */
  44 };
  45
  46 static struct rte_latency_stats *glob_stats;
  47
  48 struct rxtx_cbs {
  49         const struct rte_eth_rxtx_callback *cb;
  50 };
  51
  52 static struct rxtx_cbs rx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
  53 static struct rxtx_cbs tx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
  54
  55 struct latency_stats_nameoff {
  56         char name[RTE_ETH_XSTATS_NAME_SIZE];
  57         unsigned int offset;
  58 };
  59
  60 static const struct latency_stats_nameoff lat_stats_strings[] = {
  61         {"min_latency_ns", offsetof(struct rte_latency_stats, min_latency)},
  62         {"avg_latency_ns", offsetof(struct rte_latency_stats, avg_latency)},
  63         {"max_latency_ns", offsetof(struct rte_latency_stats, max_latency)},
  64         {"jitter_ns", offsetof(struct rte_latency_stats, jitter)},
  65 };
  66
  67 #define NUM_LATENCY_STATS (sizeof(lat_stats_strings) / \
  68                                 sizeof(lat_stats_strings[0]))
  69
  70 int32_t
  71 rte_latencystats_update(void)
  72 {
  73         unsigned int i;
  74         float *stats_ptr = NULL;
  75         uint64_t values[NUM_LATENCY_STATS] = {0};
  76         int ret;
  77
  78         for (i = 0; i < NUM_LATENCY_STATS; i++) {
  79                 stats_ptr = RTE_PTR_ADD(glob_stats,
  80                                 lat_stats_strings[i].offset);
  81                 values[i] = (uint64_t)floor((*stats_ptr)/
  82                                 latencystat_cycles_per_ns());
  83         }
  84
  85         ret = rte_metrics_update_values(RTE_METRICS_GLOBAL,
  86                                         latency_stats_index,
  87                                         values, NUM_LATENCY_STATS);
  88         if (ret < 0)
  89                 RTE_LOG(INFO, LATENCY_STATS, "Failed to push the stats\n");
  90
  91         return ret;
  92 }
  93
  94 static void
  95 rte_latencystats_fill_values(struct rte_metric_value *values)
  96 {
  97         unsigned int i;
  98         float *stats_ptr = NULL;
  99
 100         for (i = 0; i < NUM_LATENCY_STATS; i++) {
 101                 stats_ptr = RTE_PTR_ADD(glob_stats,
 102                                 lat_stats_strings[i].offset);
 103                 values[i].key = i;
 104                 values[i].value = (uint64_t)floor((*stats_ptr)/
 105                                                 latencystat_cycles_per_ns());
 106         }
 107 }
 108
 109 static uint16_t
 110 add_time_stamps(uint16_t pid __rte_unused,
 111                 uint16_t qid __rte_unused,
 112                 struct rte_mbuf **pkts,
 113                 uint16_t nb_pkts,
 114                 uint16_t max_pkts __rte_unused,
 115                 void *user_cb __rte_unused)
 116 {
 117         unsigned int i;
 118         uint64_t diff_tsc, now;
 119
 120         /*
 121          * For every sample interval,
 122          * time stamp is marked on one received packet.
 123          */
 124         now = rte_rdtsc();
 125         for (i = 0; i < nb_pkts; i++) {
 126                 diff_tsc = now - prev_tsc;
 127                 timer_tsc += diff_tsc;
 128                 if (timer_tsc >= samp_intvl) {
 129                         pkts[i]->timestamp = now;
 130                         timer_tsc = 0;
 131                 }
 132                 prev_tsc = now;
 133                 now = rte_rdtsc();
 134         }
 135
 136         return nb_pkts;
 137 }
 138
 139 static uint16_t
 140 calc_latency(uint16_t pid __rte_unused,
 141                 uint16_t qid __rte_unused,
 142                 struct rte_mbuf **pkts,
 143                 uint16_t nb_pkts,
 144                 void *_ __rte_unused)
 145 {
 146         unsigned int i, cnt = 0;
 147         uint64_t now;
 148         float latency[nb_pkts];
 149         static float prev_latency;
 150         /*
 151          * Alpha represents degree of weighting decrease in EWMA,
 152          * a constant smoothing factor between 0 and 1. The value
 153          * is used below for measuring average latency.
 154          */
 155         const float alpha = 0.2;
 156
 157         now = rte_rdtsc();
 158         for (i = 0; i < nb_pkts; i++) {
 159                 if (pkts[i]->timestamp)
 160                         latency[cnt++] = now - pkts[i]->timestamp;
 161         }
 162
 163         for (i = 0; i < cnt; i++) {
 164                 /*
 165                  * The jitter is calculated as statistical mean of interpacket
 166                  * delay variation. The "jitter estimate" is computed by taking
 167                  * the absolute values of the ipdv sequence and applying an
 168                  * exponential filter with parameter 1/16 to generate the
 169                  * estimate. i.e J=J+(|D(i-1,i)|-J)/16. Where J is jitter,
 170                  * D(i-1,i) is difference in latency of two consecutive packets
 171                  * i-1 and i.
 172                  * Reference: Calculated as per RFC 5481, sec 4.1,
 173                  * RFC 3393 sec 4.5, RFC 1889 sec.
 174                  */
 175                 glob_stats->jitter +=  (fabsf(prev_latency - latency[i])
 176                                         - glob_stats->jitter)/16;
 177                 if (glob_stats->min_latency == 0)
 178                         glob_stats->min_latency = latency[i];
 179                 else if (latency[i] < glob_stats->min_latency)
 180                         glob_stats->min_latency = latency[i];
 181                 else if (latency[i] > glob_stats->max_latency)
 182                         glob_stats->max_latency = latency[i];
 183                 /*
 184                  * The average latency is measured using exponential moving
 185                  * average, i.e. using EWMA
 186                  * https://en.wikipedia.org/wiki/Moving_average
 187                  */
 188                 glob_stats->avg_latency +=
 189                         alpha * (latency[i] - glob_stats->avg_latency);
 190                 prev_latency = latency[i];
 191         }
 192
 193         return nb_pkts;
 194 }
 195
 196 int
 197 rte_latencystats_init(uint64_t app_samp_intvl,
 198                 rte_latency_stats_flow_type_fn user_cb)
 199 {
 200         unsigned int i;
 201         uint16_t pid;
 202         uint16_t qid;
 203         struct rxtx_cbs *cbs = NULL;
 204         const char *ptr_strings[NUM_LATENCY_STATS] = {0};
 205         const struct rte_memzone *mz = NULL;
 206         const unsigned int flags = 0;
 207
 208         if (rte_memzone_lookup(MZ_RTE_LATENCY_STATS))
 209                 return -EEXIST;
 210
 211         /** Allocate stats in shared memory fo multi process support */
 212         mz = rte_memzone_reserve(MZ_RTE_LATENCY_STATS, sizeof(*glob_stats),
 213                                         rte_socket_id(), flags);
 214         if (mz == NULL) {
 215                 RTE_LOG(ERR, LATENCY_STATS, "Cannot reserve memory: %s:%d\n",
 216                         __func__, __LINE__);
 217                 return -ENOMEM;
 218         }
 219
 220         glob_stats = mz->addr;
 221         samp_intvl = app_samp_intvl * latencystat_cycles_per_ns();
 222
 223         /** Register latency stats with stats library */
 224         for (i = 0; i < NUM_LATENCY_STATS; i++)
 225                 ptr_strings[i] = lat_stats_strings[i].name;
 226
 227         latency_stats_index = rte_metrics_reg_names(ptr_strings,
 228                                                         NUM_LATENCY_STATS);
 229         if (latency_stats_index < 0) {
 230                 RTE_LOG(DEBUG, LATENCY_STATS,
 231                         "Failed to register latency stats names\n");
 232                 return -1;
 233         }
 234
 235         /** Register Rx/Tx callbacks */
 236         RTE_ETH_FOREACH_DEV(pid) {
 237                 struct rte_eth_dev_info dev_info;
 238                 rte_eth_dev_info_get(pid, &dev_info);
 239                 for (qid = 0; qid < dev_info.nb_rx_queues; qid++) {
 240                         cbs = &rx_cbs[pid][qid];
 241                         cbs->cb = rte_eth_add_first_rx_callback(pid, qid,
 242                                         add_time_stamps, user_cb);
 243                         if (!cbs->cb)
 244                                 RTE_LOG(INFO, LATENCY_STATS, "Failed to "
 245                                         "register Rx callback for pid=%d, "
 246                                         "qid=%d\n", pid, qid);
 247                 }
 248                 for (qid = 0; qid < dev_info.nb_tx_queues; qid++) {
 249                         cbs = &tx_cbs[pid][qid];
 250                         cbs->cb =  rte_eth_add_tx_callback(pid, qid,
 251                                         calc_latency, user_cb);
 252                         if (!cbs->cb)
 253                                 RTE_LOG(INFO, LATENCY_STATS, "Failed to "
 254                                         "register Tx callback for pid=%d, "
 255                                         "qid=%d\n", pid, qid);
 256                 }
 257         }
 258         return 0;
 259 }
 260
 261 int
 262 rte_latencystats_uninit(void)
 263 {
 264         uint16_t pid;
 265         uint16_t qid;
 266         int ret = 0;
 267         struct rxtx_cbs *cbs = NULL;
 268         const struct rte_memzone *mz = NULL;
 269
 270         /** De register Rx/Tx callbacks */
 271         RTE_ETH_FOREACH_DEV(pid) {
 272                 struct rte_eth_dev_info dev_info;
 273                 rte_eth_dev_info_get(pid, &dev_info);
 274                 for (qid = 0; qid < dev_info.nb_rx_queues; qid++) {
 275                         cbs = &rx_cbs[pid][qid];
 276                         ret = rte_eth_remove_rx_callback(pid, qid, cbs->cb);
 277                         if (ret)
 278                                 RTE_LOG(INFO, LATENCY_STATS, "failed to "
 279                                         "remove Rx callback for pid=%d, "
 280                                         "qid=%d\n", pid, qid);
 281                 }
 282                 for (qid = 0; qid < dev_info.nb_tx_queues; qid++) {
 283                         cbs = &tx_cbs[pid][qid];
 284                         ret = rte_eth_remove_tx_callback(pid, qid, cbs->cb);
 285                         if (ret)
 286                                 RTE_LOG(INFO, LATENCY_STATS, "failed to "
 287                                         "remove Tx callback for pid=%d, "
 288                                         "qid=%d\n", pid, qid);
 289                 }
 290         }
 291
 292         /* free up the memzone */
 293         mz = rte_memzone_lookup(MZ_RTE_LATENCY_STATS);
 294         if (mz)
 295                 rte_memzone_free(mz);
 296
 297         return 0;
 298 }
 299
 300 int
 301 rte_latencystats_get_names(struct rte_metric_name *names, uint16_t size)
 302 {
 303         unsigned int i;
 304
 305         if (names == NULL || size < NUM_LATENCY_STATS)
 306                 return NUM_LATENCY_STATS;
 307
 308         for (i = 0; i < NUM_LATENCY_STATS; i++)
 309                 snprintf(names[i].name, sizeof(names[i].name),
 310                                 "%s", lat_stats_strings[i].name);
 311
 312         return NUM_LATENCY_STATS;
 313 }
 314
 315 int
 316 rte_latencystats_get(struct rte_metric_value *values, uint16_t size)
 317 {
 318         if (size < NUM_LATENCY_STATS || values == NULL)
 319                 return NUM_LATENCY_STATS;
 320
 321         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 322                 const struct rte_memzone *mz;
 323                 mz = rte_memzone_lookup(MZ_RTE_LATENCY_STATS);
 324                 if (mz == NULL) {
 325                         RTE_LOG(ERR, LATENCY_STATS,
 326                                 "Latency stats memzone not found\n");
 327                         return -ENOMEM;
 328                 }
 329                 glob_stats =  mz->addr;
 330         }
 331
 332         /* Retrieve latency stats */
 333         rte_latencystats_fill_values(values);
 334
 335         return NUM_LATENCY_STATS;
 336 }