#define TCP_RTT_MAX 30 * THZ /* 30s (probably too much) */
#define TCP_RTO_SYN_RETRIES 3 /* SYN retries without doubling RTO */
#define TCP_RTO_INIT 1 * THZ /* Initial retransmit timer */
+#define TCP_RTO_BOFF_MAX 8 /* Max number of retries before reset */
/** TCP connection flags */
#define foreach_tcp_connection_flag \
_(SNDACK, "Send ACK") \
_(FINSNT, "FIN sent") \
- _(SENT_RCV_WND0, "Sent 0 rcv_wnd") \
_(RECOVERY, "Recovery") \
_(FAST_RECOVERY, "Fast Recovery") \
_(DCNT_PENDING, "Disconnect pending") \
_(FRXT_FIRST, "Fast-retransmit first again") \
_(DEQ_PENDING, "Pending dequeue acked") \
_(PSH_PENDING, "PSH pending") \
+ _(FINRCVD, "FIN received") \
+ _(RATE_SAMPLE, "Conn does rate sampling") \
+ _(TRACK_BURST, "Track burst") \
+ _(ZERO_RWND_SENT, "Zero RWND sent") \
typedef enum _tcp_connection_flag_bits
{
TCP_CONN_N_FLAGS
} tcp_connection_flags_e;
-/** TCP buffer flags */
-#define foreach_tcp_buf_flag \
- _ (ACK) /**< Sending ACK. */ \
- _ (DUPACK) /**< Sending DUPACK. */ \
-
-enum
-{
-#define _(f) TCP_BUF_BIT_##f,
- foreach_tcp_buf_flag
-#undef _
- TCP_N_BUF_BITS,
-};
-
-enum
-{
-#define _(f) TCP_BUF_FLAG_##f = 1 << TCP_BUF_BIT_##f,
- foreach_tcp_buf_flag
-#undef _
-};
-
#define TCP_SCOREBOARD_TRACE (0)
#define TCP_MAX_SACK_BLOCKS 256 /**< Max number of SACK blocks stored */
#define TCP_INVALID_SACK_HOLE_INDEX ((u32)~0)
u32 tail; /**< Index of last entry */
u32 sacked_bytes; /**< Number of bytes sacked in sb */
u32 last_sacked_bytes; /**< Number of bytes last sacked */
- u32 last_bytes_delivered; /**< Number of sack bytes delivered */
+ u32 last_bytes_delivered; /**< Sack bytes delivered to app */
u32 snd_una_adv; /**< Bytes to add to snd_una */
u32 high_sacked; /**< Highest byte sacked (fack) */
u32 high_rxt; /**< Highest retransmitted sequence */
void scoreboard_init (sack_scoreboard_t * sb);
u8 *format_tcp_scoreboard (u8 * s, va_list * args);
+#define TCP_BTS_INVALID_INDEX ((u32)~0)
+
+typedef enum tcp_bts_flags_
+{
+ TCP_BTS_IS_RXT = 1,
+ TCP_BTS_IS_APP_LIMITED = 1 << 1,
+} __clib_packed tcp_bts_flags_t;
+
+typedef struct tcp_bt_sample_
+{
+ u32 next; /**< Next sample index in list */
+ u32 prev; /**< Previous sample index in list */
+ u32 min_seq; /**< Min seq number in sample */
+ u32 max_seq; /**< Max seq number. Set for rxt samples */
+ u64 delivered; /**< Total delivered when sample taken */
+ f64 delivered_time; /**< Delivered time when sample taken */
+ u64 tx_rate; /**< Tx pacing rate */
+ tcp_bts_flags_t flags; /**< Sample flag */
+} tcp_bt_sample_t;
+
+typedef struct tcp_rate_sample_
+{
+ u64 sample_delivered; /**< Delivered of sample used for rate */
+ u32 delivered; /**< Bytes delivered in ack time */
+ f64 ack_time; /**< Time to ack the bytes delivered */
+ u64 tx_rate; /**< Tx pacing rate */
+ tcp_bts_flags_t flags; /**< Rate sample flags from bt sample */
+} tcp_rate_sample_t;
+
+typedef struct tcp_byte_tracker_
+{
+ tcp_bt_sample_t *samples; /**< Pool of samples */
+ rb_tree_t sample_lookup; /**< Rbtree for sample lookup by min_seq */
+ u32 head; /**< Head of samples linked list */
+ u32 tail; /**< Tail of samples linked list */
+ u32 last_ooo; /**< Cached last ooo sample */
+} tcp_byte_tracker_t;
+
typedef enum _tcp_cc_algorithm_type
{
TCP_CC_NEWRENO,
TCP_CC_CUBIC,
+ TCP_CC_LAST = TCP_CC_CUBIC
} tcp_cc_algorithm_type_e;
typedef struct _tcp_cc_algorithm tcp_cc_algorithm_t;
TCP_CC_PARTIALACK
} tcp_cc_ack_t;
+typedef enum tcp_cc_event_
+{
+ TCP_CC_EVT_START_TX,
+} tcp_cc_event_t;
+
typedef struct _tcp_connection
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
u32 snd_rxt_ts; /**< Timestamp when first packet is retransmitted */
u32 tsecr_last_ack; /**< Timestamp echoed to us in last healthy ACK */
u32 snd_congestion; /**< snd_una_max when congestion is detected */
+ u32 tx_fifo_size; /**< Tx fifo size. Used to constrain cwnd */
tcp_cc_algorithm_t *cc_algo; /**< Congestion control algorithm */
u8 cc_data[TCP_CC_DATA_SZ]; /**< Congestion control algo private data */
f64 rtt_ts; /**< Timestamp for tracked ACK */
f64 mrtt_us; /**< High precision mrtt from tracked acks */
- u16 mss; /**< Our max seg size that includes options */
+ u32 psh_seq; /**< Add psh header for seg that includes this */
+ u32 next_node_index; /**< Can be used to control next node in output */
+ u32 next_node_opaque; /**< Opaque to pass to next node */
u32 limited_transmit; /**< snd_nxt when limited transmit starts */
- u32 last_fib_check; /**< Last time we checked fib route for peer */
u32 sw_if_index; /**< Interface for the connection */
- u32 tx_fifo_size; /**< Tx fifo size. Used to constrain cwnd */
- u32 psh_seq; /**< Add psh header for seg that includes this */
+ /* Delivery rate estimation */
+ u64 delivered; /**< Total bytes delivered to peer */
+ u64 app_limited; /**< Delivered when app-limited detected */
+ f64 delivered_time; /**< Time last bytes were acked */
+ tcp_byte_tracker_t *bt; /**< Tx byte tracker */
+
+ u32 last_fib_check; /**< Last time we checked fib route for peer */
+ u16 mss; /**< Our max seg size that includes options */
+ u32 timestamp_delta;
} tcp_connection_t;
/* *INDENT-OFF* */
{
const char *name;
uword (*unformat_cfg) (unformat_input_t * input);
- void (*rcv_ack) (tcp_connection_t * tc);
- void (*rcv_cong_ack) (tcp_connection_t * tc, tcp_cc_ack_t ack);
+ void (*init) (tcp_connection_t * tc);
+ void (*cleanup) (tcp_connection_t * tc);
+ void (*rcv_ack) (tcp_connection_t * tc, tcp_rate_sample_t *rs);
+ void (*rcv_cong_ack) (tcp_connection_t * tc, tcp_cc_ack_t ack,
+ tcp_rate_sample_t *rs);
void (*congestion) (tcp_connection_t * tc);
+ void (*loss) (tcp_connection_t * tc);
void (*recovered) (tcp_connection_t * tc);
- void (*init) (tcp_connection_t * tc);
+ void (*undo_recovery) (tcp_connection_t * tc);
+ void (*event) (tcp_connection_t *tc, tcp_cc_event_t evt);
};
/* *INDENT-ON* */
tcp_fastrecovery_first_off (tc);
}
+#define tcp_zero_rwnd_sent(tc) (tc)->flags &= TCP_CONN_ZERO_RWND_SENT
+#define tcp_zero_rwnd_sent_on(tc) (tc)->flags |= TCP_CONN_ZERO_RWND_SENT
+#define tcp_zero_rwnd_sent_off(tc) (tc)->flags &= ~TCP_CONN_ZERO_RWND_SENT
+
typedef enum _tcp_error
{
#define tcp_error(n,s) TCP_ERROR_##n,
tcp_connection_t *half_open_connections;
clib_spinlock_t half_open_lock;
- /* Congestion control algorithms registered */
- tcp_cc_algorithm_t *cc_algos;
-
/** vlib buffer size */
u32 bytes_per_buffer;
/* Seed used to generate random iss */
tcp_iss_seed_t iss_seed;
+ /* Congestion control algorithms registered */
+ tcp_cc_algorithm_t *cc_algos;
+
+ /** Hash table of cc algorithms by name */
+ uword *cc_algo_by_name;
+
+ /** Last cc algo registered */
+ tcp_cc_algorithm_type_e cc_last_type;
+
/*
* Configuration
*/
* rfc 7323 window scaling factor */
u32 max_rx_fifo;
+ /** Default MTU to be used when establishing connections */
+ u16 default_mtu;
+
+ /** Initial CWND multiplier, which multiplies MSS to determine initial CWND.
+ * Set 0 to determine the initial CWND by another way */
+ u16 initial_cwnd_multiplier;
+
/** Number of preallocated connections */
u32 preallocated_connections;
u32 preallocated_half_open_connections;
/** Default congestion control algorithm type */
tcp_cc_algorithm_type_e cc_algo;
-
} tcp_main_t;
extern tcp_main_t tcp_main;
extern vlib_node_registration_t tcp6_input_node;
extern vlib_node_registration_t tcp4_output_node;
extern vlib_node_registration_t tcp6_output_node;
+extern vlib_node_registration_t tcp4_established_node;
+extern vlib_node_registration_t tcp6_established_node;
+extern vlib_node_registration_t tcp4_syn_sent_node;
+extern vlib_node_registration_t tcp6_syn_sent_node;
+extern vlib_node_registration_t tcp4_rcv_process_node;
+extern vlib_node_registration_t tcp6_rcv_process_node;
+extern vlib_node_registration_t tcp4_listen_node;
+extern vlib_node_registration_t tcp6_listen_node;
always_inline tcp_main_t *
vnet_get_tcp_main ()
void tcp_program_ack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc);
void tcp_program_dupack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc);
void tcp_send_acks (tcp_worker_ctx_t * wrk);
+void tcp_send_window_update_ack (tcp_connection_t * tc);
+
+/*
+ * Rate estimation
+ */
+
+/**
+ * Byte tracker initialize
+ *
+ * @param tc connection for which the byte tracker should be allocated and
+ * initialized
+ */
+void tcp_bt_init (tcp_connection_t * tc);
+/**
+ * Byte tracker cleanup
+ *
+ * @param tc connection for which the byte tracker should be cleaned up
+ */
+void tcp_bt_cleanup (tcp_connection_t * tc);
+/**
+ * Flush byte tracker samples
+ *
+ * @param tc tcp connection for which samples should be flushed
+ */
+void tcp_bt_flush_samples (tcp_connection_t * tc);
+/**
+ * Track a tcp tx burst
+ *
+ * @param tc tcp connection
+ */
+void tcp_bt_track_tx (tcp_connection_t * tc);
+/**
+ * Track a tcp retransmission
+ *
+ * @param tc tcp connection
+ * @param start start sequence number
+ * @param end end sequence number
+ */
+void tcp_bt_track_rxt (tcp_connection_t * tc, u32 start, u32 end);
+/**
+ * Generate a delivery rate sample from recently acked bytes
+ *
+ * @param tc tcp connection
+ * @param rs resulting rate sample
+ */
+void tcp_bt_sample_delivery_rate (tcp_connection_t * tc,
+ tcp_rate_sample_t * rs);
+/**
+ * Check if sample to be generated is app limited
+ *
+ * @param tc tcp connection
+ */
+void tcp_bt_check_app_limited (tcp_connection_t * tc);
+/**
+ * Check if the byte tracker is in sane state
+ *
+ * Should be used only for testing
+ *
+ * @param bt byte tracker
+ */
+int tcp_bt_is_sane (tcp_byte_tracker_t * bt);
always_inline u32
tcp_end_seq (tcp_header_t * th, u32 len)
{
int flight_size;
- flight_size = (int) (tc->snd_una_max - tc->snd_una) - tcp_bytes_out (tc)
+ flight_size = (int) (tc->snd_nxt - tc->snd_una) - tcp_bytes_out (tc)
+ tc->snd_rxt_bytes;
if (flight_size < 0)
always_inline u32
tcp_initial_cwnd (const tcp_connection_t * tc)
{
+ if (tcp_main.initial_cwnd_multiplier > 0)
+ return tcp_main.initial_cwnd_multiplier * tc->snd_mss;
+
if (tc->snd_mss > 2190)
return 2 * tc->snd_mss;
else if (tc->snd_mss > 1095)
int tcp_fast_retransmit (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
u32 burst_size);
void tcp_cc_init_congestion (tcp_connection_t * tc);
-void tcp_cc_fastrecovery_exit (tcp_connection_t * tc);
+void tcp_cc_fastrecovery_clear (tcp_connection_t * tc);
fib_node_index_t tcp_lookup_rmt_in_fib (tcp_connection_t * tc);
return tcp_main.wrk_ctx[thread_index].time_now;
}
+/**
+ * Generate timestamp for tcp connection
+ */
+always_inline u32
+tcp_tstamp (tcp_connection_t * tc)
+{
+ return (tcp_main.wrk_ctx[tc->c_thread_index].time_now -
+ tc->timestamp_delta);
+}
+
always_inline f64
tcp_time_now_us (u32 thread_index)
{
return wrk->time_now;
}
-u32 tcp_push_header (tcp_connection_t * tconn, vlib_buffer_t * b);
+u32 tcp_session_push_header (transport_connection_t * tconn,
+ vlib_buffer_t * b);
void tcp_connection_timers_init (tcp_connection_t * tc);
void tcp_connection_timers_reset (tcp_connection_t * tc);
u32 start_bucket);
always_inline void
-tcp_cc_rcv_ack (tcp_connection_t * tc)
+tcp_cc_rcv_ack (tcp_connection_t * tc, tcp_rate_sample_t * rs)
{
- tc->cc_algo->rcv_ack (tc);
+ tc->cc_algo->rcv_ack (tc, rs);
tc->tsecr_last_ack = tc->rcv_opts.tsecr;
}
+static inline void
+tcp_cc_rcv_cong_ack (tcp_connection_t * tc, tcp_cc_ack_t ack_type,
+ tcp_rate_sample_t * rs)
+{
+ tc->cc_algo->rcv_cong_ack (tc, ack_type, rs);
+}
+
+static inline void
+tcp_cc_loss (tcp_connection_t * tc)
+{
+ tc->cc_algo->loss (tc);
+}
+
+static inline void
+tcp_cc_recovered (tcp_connection_t * tc)
+{
+ tc->cc_algo->recovered (tc);
+}
+
+static inline void
+tcp_cc_undo_recovery (tcp_connection_t * tc)
+{
+ if (tc->cc_algo->undo_recovery)
+ tc->cc_algo->undo_recovery (tc);
+}
+
+static inline void
+tcp_cc_event (tcp_connection_t * tc, tcp_cc_event_t evt)
+{
+ if (tc->cc_algo->event)
+ tc->cc_algo->event (tc, evt);
+}
+
always_inline void
tcp_timer_set (tcp_connection_t * tc, u8 timer_id, u32 interval)
{
always_inline void
tcp_retransmit_timer_update (tcp_connection_t * tc)
{
- if (tc->snd_una == tc->snd_una_max)
+ if (tc->snd_una == tc->snd_nxt)
{
tcp_retransmit_timer_reset (tc);
if (tc->snd_wnd < tc->snd_mss)
#define tcp_validate_txf_size(_tc, _a) \
ASSERT(_tc->state != TCP_STATE_ESTABLISHED \
- || session_tx_fifo_max_dequeue (&_tc->connection) >= _a)
+ || transport_max_tx_dequeue (&_tc->connection) >= _a)
void tcp_rcv_sacks (tcp_connection_t * tc, u32 ack);
u8 *tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose);
+/**
+ * Register exiting cc algo type
+ */
void tcp_cc_algo_register (tcp_cc_algorithm_type_e type,
const tcp_cc_algorithm_t * vft);
+/**
+ * Register new cc algo type
+ */
+tcp_cc_algorithm_type_e tcp_cc_algo_new_type (const tcp_cc_algorithm_t * vft);
tcp_cc_algorithm_t *tcp_cc_algo_get (tcp_cc_algorithm_type_e type);
static inline void *
return (void *) tc->cc_data;
}
-void newreno_rcv_cong_ack (tcp_connection_t * tc, tcp_cc_ack_t ack_type);
+void newreno_rcv_cong_ack (tcp_connection_t * tc, tcp_cc_ack_t ack_type,
+ tcp_rate_sample_t * rs);
/**
* Push TCP header to buffer
Code Review / vpp.git / blobdiff