From a8e71c8981f039588a7ca94e6ab66b4ebac784a5 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Tue, 22 Oct 2019 19:01:39 -0700 Subject: [PATCH] session tcp: refactor pacer time Type: refactor Change-Id: Ic1c3e1f7987702cd88972acc34849dc1f585d5fe Signed-off-by: Florin Coras --- src/vnet/session/session.c | 2 +- src/vnet/session/session.h | 15 ++++-- src/vnet/session/session_node.c | 10 ++-- src/vnet/session/transport.c | 100 ++++++++++++++++--------------------- src/vnet/session/transport.h | 25 +++------- src/vnet/session/transport_types.h | 2 +- src/vnet/tcp/tcp.c | 6 +-- src/vnet/tcp/tcp_input.c | 4 +- src/vnet/tcp/tcp_output.c | 14 ++---- src/vppinfra/time.h | 7 +++ 10 files changed, 81 insertions(+), 104 deletions(-) diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c index d9a733d356e..5cd840bd834 100644 --- a/src/vnet/session/session.c +++ b/src/vnet/session/session.c @@ -1494,6 +1494,7 @@ session_manager_main_enable (vlib_main_t * vm) wrk->old_head = clib_llist_make_head (wrk->event_elts, evt_list); wrk->vm = vlib_mains[i]; wrk->last_vlib_time = vlib_time_now (vlib_mains[i]); + wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ; if (num_threads > 1) clib_rwlock_init (&smm->wrk[i].peekers_rw_locks); @@ -1537,7 +1538,6 @@ session_manager_main_enable (vlib_main_t * vm) /* Enable transports */ transport_enable_disable (vm, 1); - transport_init_tx_pacers_period (); return 0; } diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index 9fe35b19b25..830a34b72e8 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -84,7 +84,10 @@ typedef struct session_worker_ svm_msg_q_t *vpp_event_queue; /** vlib_time_now last time around the track */ - f64 last_vlib_time; + clib_time_type_t last_vlib_time; + + /** vlib_time_now rounded to us precision and as u64 */ + clib_us_time_t last_vlib_us_time; /** Convenience pointer to this worker's vlib_main */ vlib_main_t *vm; @@ -124,7 +127,7 @@ typedef struct session_worker_ #if SESSION_DEBUG /** last event poll time by thread */ - f64 last_event_poll; + clib_time_type_t last_event_poll; #endif } session_worker_t; @@ -516,12 +519,18 @@ transport_rx_fifo_has_ooo_data (transport_connection_t * tc) return svm_fifo_has_ooo_data (s->rx_fifo); } -always_inline f64 +always_inline clib_time_type_t transport_time_now (u32 thread_index) { return session_main.wrk[thread_index].last_vlib_time; } +always_inline clib_us_time_t +transport_us_time_now (u32 thread_index) +{ + return session_main.wrk[thread_index].last_vlib_us_time; +} + always_inline void transport_add_tx_event (transport_connection_t * tc) { diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c index 3bfc80eca66..e5faf3c1663 100644 --- a/src/vnet/session/session_node.c +++ b/src/vnet/session/session_node.c @@ -871,10 +871,7 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk, return SESSION_TX_NO_DATA; } - ctx->snd_space = transport_connection_snd_space (ctx->tc, - vm->clib_time. - last_cpu_time, - ctx->snd_mss); + ctx->snd_space = transport_connection_snd_space (ctx->tc, ctx->snd_mss); /* This flow queue is "empty" so it should be re-evaluated before * the ones that have data to send. */ @@ -892,9 +889,7 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk, if (PREDICT_FALSE (!ctx->max_len_to_snd)) { - transport_connection_tx_pacer_reset_bucket (ctx->tc, - vm->clib_time. - last_cpu_time); + transport_connection_tx_pacer_reset_bucket (ctx->tc); return SESSION_TX_NO_DATA; } @@ -1257,6 +1252,7 @@ session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, SESSION_EVT (SESSION_EVT_DISPATCH_START, wrk); wrk->last_vlib_time = vlib_time_now (vm); + wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ; /* * Update transport time diff --git a/src/vnet/session/transport.c b/src/vnet/session/transport.c index 8f7e30c8602..3160a48d37d 100644 --- a/src/vnet/session/transport.c +++ b/src/vnet/session/transport.c @@ -42,11 +42,6 @@ static transport_endpoint_t *local_endpoints; */ static clib_spinlock_t local_endpoints_lock; -/* - * Period used by transport pacers. Initialized by session layer - */ -static double transport_pacer_period; - u8 * format_transport_proto (u8 * s, va_list * args) { @@ -107,7 +102,7 @@ format_transport_connection (u8 * s, va_list * args) { indent = format_get_indent (s) + 1; s = format (s, "%Upacer: %U\n", format_white_space, indent, - format_transport_pacer, &tc->pacer); + format_transport_pacer, &tc->pacer, tc->thread_index); } return s; } @@ -555,33 +550,41 @@ transport_alloc_local_endpoint (u8 proto, transport_endpoint_cfg_t * rmt_cfg, return 0; } -#define SPACER_CPU_TICKS_PER_PERIOD_SHIFT 10 -#define SPACER_CPU_TICKS_PER_PERIOD (1 << SPACER_CPU_TICKS_PER_PERIOD_SHIFT) +u8 * +format_clib_us_time (u8 * s, va_list * args) +{ + clib_us_time_t t = va_arg (*args, clib_us_time_t); + if (t < 1e3) + s = format (s, "%u us", t); + else + s = format (s, "%.3f s", (f64) t * CLIB_US_TIME_PERIOD); + return s; +} u8 * format_transport_pacer (u8 * s, va_list * args) { spacer_t *pacer = va_arg (*args, spacer_t *); - vlib_main_t *vm = vlib_get_main (); - u64 now, diff; + u32 thread_index = va_arg (*args, int); + clib_us_time_t now, diff; - now = vm->clib_time.last_cpu_time; - diff = now - (pacer->last_update << SPACER_CPU_TICKS_PER_PERIOD_SHIFT); - s = format (s, "rate %u bucket %u t/p %.3f last_update %.3f", + now = transport_us_time_now (thread_index); + diff = now - pacer->last_update; + s = format (s, "rate %lu bucket %lu t/p %.3f last_update %U", pacer->bytes_per_sec, pacer->bucket, pacer->tokens_per_period, - diff * vm->clib_time.seconds_per_clock); + format_clib_us_time, diff); return s; } static inline u32 -spacer_max_burst (spacer_t * pacer, u64 norm_time_now) +spacer_max_burst (spacer_t * pacer, clib_us_time_t time_now) { - u64 n_periods = norm_time_now - pacer->last_update; + u64 n_periods = (time_now - pacer->last_update); u64 inc; - if (PREDICT_FALSE (n_periods > 5e5)) + if (PREDICT_FALSE (n_periods > 5e4)) { - pacer->last_update = norm_time_now; + pacer->last_update = time_now; pacer->bucket = TRANSPORT_PACER_MIN_BURST; return TRANSPORT_PACER_MIN_BURST; } @@ -589,7 +592,7 @@ spacer_max_burst (spacer_t * pacer, u64 norm_time_now) if (n_periods > 0 && (inc = (f32) n_periods * pacer->tokens_per_period) > 10) { - pacer->last_update = norm_time_now; + pacer->last_update = time_now; pacer->bucket = clib_min (pacer->bucket + inc, pacer->bytes_per_sec); } @@ -608,7 +611,7 @@ spacer_set_pace_rate (spacer_t * pacer, u64 rate_bytes_per_sec) { ASSERT (rate_bytes_per_sec != 0); pacer->bytes_per_sec = rate_bytes_per_sec; - pacer->tokens_per_period = rate_bytes_per_sec / transport_pacer_period; + pacer->tokens_per_period = rate_bytes_per_sec * CLIB_US_TIME_PERIOD; } static inline u64 @@ -618,33 +621,35 @@ spacer_pace_rate (spacer_t * pacer) } static inline void -spacer_reset_bucket (spacer_t * pacer, u64 norm_time_now) +spacer_reset (spacer_t * pacer, clib_us_time_t time_now, u64 bucket) { - pacer->last_update = norm_time_now; - pacer->bucket = 0; + pacer->last_update = time_now; + pacer->bucket = bucket; } void transport_connection_tx_pacer_reset (transport_connection_t * tc, - u32 rate_bytes_per_sec, - u32 start_bucket, u64 time_now) + u64 rate_bytes_per_sec, u32 start_bucket) { - spacer_t *pacer = &tc->pacer; spacer_set_pace_rate (&tc->pacer, rate_bytes_per_sec); - pacer->last_update = time_now >> SPACER_CPU_TICKS_PER_PERIOD_SHIFT; - pacer->bucket = start_bucket; + spacer_reset (&tc->pacer, transport_us_time_now (tc->thread_index), + start_bucket); +} + +void +transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc) +{ + spacer_reset (&tc->pacer, transport_us_time_now (tc->thread_index), 0); } void transport_connection_tx_pacer_init (transport_connection_t * tc, - u32 rate_bytes_per_sec, + u64 rate_bytes_per_sec, u32 initial_bucket) { - vlib_main_t *vm = vlib_get_main (); tc->flags |= TRANSPORT_CONNECTION_F_IS_TX_PACED; transport_connection_tx_pacer_reset (tc, rate_bytes_per_sec, - initial_bucket, - vm->clib_time.last_cpu_time); + initial_bucket); } void @@ -655,32 +660,22 @@ transport_connection_tx_pacer_update (transport_connection_t * tc, } u32 -transport_connection_tx_pacer_burst (transport_connection_t * tc, - u64 time_now) -{ - time_now >>= SPACER_CPU_TICKS_PER_PERIOD_SHIFT; - return spacer_max_burst (&tc->pacer, time_now); -} - -void -transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc, - u64 time_now) +transport_connection_tx_pacer_burst (transport_connection_t * tc) { - time_now >>= SPACER_CPU_TICKS_PER_PERIOD_SHIFT; - spacer_reset_bucket (&tc->pacer, time_now); + return spacer_max_burst (&tc->pacer, + transport_us_time_now (tc->thread_index)); } u32 -transport_connection_snd_space (transport_connection_t * tc, u64 time_now, - u16 mss) +transport_connection_snd_space (transport_connection_t * tc, u16 mss) { u32 snd_space, max_paced_burst; snd_space = tp_vfts[tc->proto].send_space (tc); if (snd_space && transport_connection_is_tx_paced (tc)) { - time_now >>= SPACER_CPU_TICKS_PER_PERIOD_SHIFT; - max_paced_burst = spacer_max_burst (&tc->pacer, time_now); + clib_us_time_t now = transport_us_time_now (tc->thread_index); + max_paced_burst = spacer_max_burst (&tc->pacer, now); max_paced_burst = (max_paced_burst < TRANSPORT_PACER_MIN_BURST) ? 0 : max_paced_burst; snd_space = clib_min (snd_space, max_paced_burst); @@ -710,14 +705,7 @@ transport_connection_tx_pacer_update_bytes (transport_connection_t * tc, } void -transport_init_tx_pacers_period (void) -{ - f64 cpu_freq = os_cpu_clock_frequency (); - transport_pacer_period = cpu_freq / SPACER_CPU_TICKS_PER_PERIOD; -} - -void -transport_update_time (f64 time_now, u8 thread_index) +transport_update_time (clib_time_type_t time_now, u8 thread_index) { transport_proto_vft_t *vft; vec_foreach (vft, tp_vfts) diff --git a/src/vnet/session/transport.h b/src/vnet/session/transport.h index 914991d5b31..df246a9b305 100644 --- a/src/vnet/session/transport.h +++ b/src/vnet/session/transport.h @@ -152,7 +152,7 @@ void transport_register_protocol (transport_proto_t transport_proto, const transport_proto_vft_t * vft, fib_protocol_t fib_proto, u32 output_node); transport_proto_vft_t *transport_protocol_get_vft (transport_proto_t tp); -void transport_update_time (f64 time_now, u8 thread_index); +void transport_update_time (clib_time_type_t time_now, u8 thread_index); int transport_alloc_local_port (u8 proto, ip46_address_t * ip); int transport_alloc_local_endpoint (u8 proto, transport_endpoint_cfg_t * rmt, @@ -173,8 +173,8 @@ transport_elog_track_index (transport_connection_t * tc) } void transport_connection_tx_pacer_reset (transport_connection_t * tc, - u32 rate_bytes_per_sec, - u32 initial_bucket, u64 time_now); + u64 rate_bytes_per_sec, + u32 initial_bucket); /** * Initialize tx pacer for connection * @@ -183,7 +183,7 @@ void transport_connection_tx_pacer_reset (transport_connection_t * tc, * @param burst_bytes initial burst size in bytes */ void transport_connection_tx_pacer_init (transport_connection_t * tc, - u32 rate_bytes_per_sec, + u64 rate_bytes_per_sec, u32 initial_bucket); /** @@ -202,8 +202,7 @@ void transport_connection_tx_pacer_update (transport_connection_t * tc, * @param time_now current cpu time as returned by @ref clib_cpu_time_now * @param mss transport's mss */ -u32 transport_connection_snd_space (transport_connection_t * tc, - u64 time_now, u16 mss); +u32 transport_connection_snd_space (transport_connection_t * tc, u16 mss); /** * Get tx pacer max burst @@ -212,8 +211,7 @@ u32 transport_connection_snd_space (transport_connection_t * tc, * @param time_now current cpu time * @return max burst for connection */ -u32 transport_connection_tx_pacer_burst (transport_connection_t * tc, - u64 time_now); +u32 transport_connection_tx_pacer_burst (transport_connection_t * tc); /** * Get tx pacer current rate @@ -229,16 +227,7 @@ u64 transport_connection_tx_pacer_rate (transport_connection_t * tc); * @param tc transport connection * @param time_now current cpu time */ -void transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc, - u64 time_now); - -/** - * Initialize period for tx pacers - * - * Defines a unit of time with respect to number of cpu cycles that is to - * be used by all tx pacers. - */ -void transport_init_tx_pacers_period (void); +void transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc); /** * Check if transport connection is paced diff --git a/src/vnet/session/transport_types.h b/src/vnet/session/transport_types.h index b6e01b54345..1e19798348f 100644 --- a/src/vnet/session/transport_types.h +++ b/src/vnet/session/transport_types.h @@ -51,7 +51,7 @@ typedef struct _spacer { u64 bytes_per_sec; u64 bucket; - u64 last_update; + clib_us_time_t last_update; f32 tokens_per_period; } spacer_t; diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c index 1d030c9ce71..d8ba65871d9 100644 --- a/src/vnet/tcp/tcp.c +++ b/src/vnet/tcp/tcp.c @@ -1399,11 +1399,9 @@ void tcp_connection_tx_pacer_reset (tcp_connection_t * tc, u32 window, u32 start_bucket) { - tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index); f64 srtt = clib_min ((f64) tc->srtt * TCP_TICK, tc->mrtt_us); - u64 last_time = wrk->vm->clib_time.last_cpu_time; - transport_connection_tx_pacer_reset (&tc->connection, window / srtt, - start_bucket, last_time); + u64 rate = (u64) window / srtt; + transport_connection_tx_pacer_reset (&tc->connection, rate, start_bucket); } static void diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c index b2dc2f7b1a8..69db57ced95 100755 --- a/src/vnet/tcp/tcp_input.c +++ b/src/vnet/tcp/tcp_input.c @@ -629,9 +629,7 @@ tcp_handle_postponed_dequeues (tcp_worker_ctx_t * wrk) * we're in recovery and snd space constrained */ if (tc->data_segs_out == tc->prev_dsegs_out || tcp_recovery_no_snd_space (tc)) - transport_connection_tx_pacer_reset_bucket (&tc->connection, - wrk->vm->clib_time. - last_cpu_time); + transport_connection_tx_pacer_reset_bucket (&tc->connection); tc->prev_dsegs_out = tc->data_segs_out; } _vec_len (wrk->pending_deq_acked) = 0; diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index 45282168bd7..71e9d7bbdb0 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -1876,13 +1876,10 @@ tcp_retransmit_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, vlib_buffer_t *b = 0; sack_scoreboard_t *sb; int snd_space; - u64 time_now; ASSERT (tcp_in_cong_recovery (tc)); - time_now = wrk->vm->clib_time.last_cpu_time; - burst_bytes = transport_connection_tx_pacer_burst (&tc->connection, - time_now); + burst_bytes = transport_connection_tx_pacer_burst (&tc->connection); burst_size = clib_min (burst_size, burst_bytes / tc->snd_mss); if (!burst_size) { @@ -2017,9 +2014,7 @@ done: if (reset_pacer) { - transport_connection_tx_pacer_reset_bucket (&tc->connection, - vm->clib_time. - last_cpu_time); + transport_connection_tx_pacer_reset_bucket (&tc->connection); } else { @@ -2044,14 +2039,11 @@ tcp_retransmit_no_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, int snd_space, n_segs = 0; u8 cc_limited = 0; vlib_buffer_t *b; - u64 time_now; ASSERT (tcp_in_fastrecovery (tc)); TCP_EVT (TCP_EVT_CC_EVT, tc, 0); - time_now = wrk->vm->clib_time.last_cpu_time; - burst_bytes = transport_connection_tx_pacer_burst (&tc->connection, - time_now); + burst_bytes = transport_connection_tx_pacer_burst (&tc->connection); burst_size = clib_min (burst_size, burst_bytes / tc->snd_mss); if (!burst_size) { diff --git a/src/vppinfra/time.h b/src/vppinfra/time.h index 0327ce0fd05..ae738db1307 100644 --- a/src/vppinfra/time.h +++ b/src/vppinfra/time.h @@ -193,6 +193,13 @@ clib_cpu_time_now (void) void clib_time_verify_frequency (clib_time_t * c); +/* Define it as the type returned by clib_time_now */ +typedef f64 clib_time_type_t; +typedef u64 clib_us_time_t; + +#define CLIB_US_TIME_PERIOD (1e-6) +#define CLIB_US_TIME_FREQ (1.0/CLIB_US_TIME_PERIOD) + always_inline f64 clib_time_now_internal (clib_time_t * c, u64 n) { -- 2.16.6