Remove c-11 memcpy checks from perf-critical code

[vpp.git] / src / vnet / tcp / tcp_input.c

diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c
index 0f1ab1a..6809a91 100644 (file)
--- a/src/vnet/tcp/tcp_input.c
+++ b/src/vnet/tcp/tcp_input.c
@@ -1131,7 +1131,6 @@ tcp_cc_fastrecovery_exit (tcp_connection_t * tc)
   tc->rtt_ts = 0;
 
   tcp_fastrecovery_off (tc);
-  tcp_fastrecovery_1_smss_off (tc);
   tcp_fastrecovery_first_off (tc);
 
   TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 3);
@@ -1441,7 +1440,7 @@ partial_ack:
    */
 
   /* XXX limit this only to first partial ack? */
-  tcp_retransmit_timer_force_update (tc);
+  tcp_retransmit_timer_update (tc);
 
   /* RFC6675: If the incoming ACK is a cumulative acknowledgment,
    * reset dupacks to 0. Also needed if in congestion recovery */
@@ -1600,6 +1599,54 @@ process_ack:
   return 0;
 }
 
+static void
+tcp_program_disconnect (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
+{
+  if (!tcp_disconnect_pending (tc))
+    {
+      vec_add1 (wrk->pending_disconnects, tc->c_c_index);
+      tcp_disconnect_pending_on (tc);
+    }
+}
+
+static void
+tcp_handle_disconnects (tcp_worker_ctx_t * wrk)
+{
+  u32 thread_index, *pending_disconnects;
+  tcp_connection_t *tc;
+  int i;
+
+  if (!vec_len (wrk->pending_disconnects))
+    return;
+
+  thread_index = wrk->vm->thread_index;
+  pending_disconnects = wrk->pending_disconnects;
+  for (i = 0; i < vec_len (pending_disconnects); i++)
+    {
+      tc = tcp_connection_get (pending_disconnects[i], thread_index);
+      tcp_disconnect_pending_off (tc);
+      stream_session_disconnect_notify (&tc->connection);
+    }
+  _vec_len (wrk->pending_disconnects) = 0;
+}
+
+static void
+tcp_rcv_fin (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, vlib_buffer_t * b,
+            u32 * error)
+{
+  /* Enter CLOSE-WAIT and notify session. To avoid lingering
+   * in CLOSE-WAIT, set timer (reuse WAITCLOSE). */
+  /* Account for the FIN if nothing else was received */
+  if (vnet_buffer (b)->tcp.data_len == 0)
+    tc->rcv_nxt += 1;
+  tcp_program_ack (wrk, tc);
+  tc->state = TCP_STATE_CLOSE_WAIT;
+  tcp_program_disconnect (wrk, tc);
+  tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_CLOSEWAIT_TIME);
+  TCP_EVT_DBG (TCP_EVT_FIN_RCVD, tc);
+  *error = TCP_ERROR_FIN_RCVD;
+}
+
 static u8
 tcp_sack_vector_is_sane (sack_block_t * sacks)
 {
@@ -1943,13 +1990,14 @@ tcp_set_rx_trace_data (tcp_rx_trace_t * t0, tcp_connection_t * tc0,
 {
   if (tc0)
     {
-      clib_memcpy (&t0->tcp_connection, tc0, sizeof (t0->tcp_connection));
+      clib_memcpy_fast (&t0->tcp_connection, tc0,
+                       sizeof (t0->tcp_connection));
     }
   else
     {
       th0 = tcp_buffer_hdr (b0);
     }
-  clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header));
+  clib_memcpy_fast (&t0->tcp_header, th0, sizeof (t0->tcp_header));
 }
 
 static void
@@ -2099,19 +2147,7 @@ tcp46_established_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
       /* 8: check the FIN bit */
       if (PREDICT_FALSE (is_fin))
-       {
-         /* Enter CLOSE-WAIT and notify session. To avoid lingering
-          * in CLOSE-WAIT, set timer (reuse WAITCLOSE). */
-         /* Account for the FIN if nothing else was received */
-         if (vnet_buffer (b0)->tcp.data_len == 0)
-           tc0->rcv_nxt += 1;
-         tcp_program_ack (wrk, tc0);
-         tc0->state = TCP_STATE_CLOSE_WAIT;
-         stream_session_disconnect_notify (&tc0->connection);
-         tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_CLOSEWAIT_TIME);
-         TCP_EVT_DBG (TCP_EVT_FIN_RCVD, tc0);
-         error0 = TCP_ERROR_FIN_RCVD;
-       }
+       tcp_rcv_fin (wrk, tc0, b0, &error0);
 
     done:
       tcp_inc_err_counter (err_counters, error0, 1);
@@ -2122,6 +2158,7 @@ tcp46_established_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   err_counters[TCP_ERROR_EVENT_FIFO_FULL] = errors;
   tcp_store_err_counters (established, err_counters);
   tcp_handle_postponed_dequeues (wrk);
+  tcp_handle_disconnects (wrk);
   vlib_buffer_free (vm, first_buffer, frame->n_vectors);
 
   return frame->n_vectors;
@@ -2407,7 +2444,7 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       /* Valid SYN or SYN-ACK. Move connection from half-open pool to
        * current thread pool. */
       pool_get (tm->connections[my_thread_index], new_tc0);
-      clib_memcpy (new_tc0, tc0, sizeof (*new_tc0));
+      clib_memcpy_fast (new_tc0, tc0, sizeof (*new_tc0));
       new_tc0->c_c_index = new_tc0 - tm->connections[my_thread_index];
       new_tc0->c_thread_index = my_thread_index;
       new_tc0->rcv_nxt = vnet_buffer (b0)->tcp.seq_end;
@@ -2459,6 +2496,8 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
              goto drop;
            }
 
+         new_tc0->tx_fifo_size =
+           transport_tx_fifo_size (&new_tc0->connection);
          /* Update rtt with the syn-ack sample */
          tcp_estimate_initial_rtt (new_tc0);
          TCP_EVT_DBG (TCP_EVT_SYNACK_RCVD, new_tc0);
@@ -2478,8 +2517,10 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
              goto drop;
            }
 
-         tc0->rtt_ts = 0;
-         tcp_init_snd_vars (tc0);
+         new_tc0->tx_fifo_size =
+           transport_tx_fifo_size (&new_tc0->connection);
+         new_tc0->rtt_ts = 0;
+         tcp_init_snd_vars (new_tc0);
          tcp_send_synack (new_tc0);
          error0 = TCP_ERROR_SYNS_RCVD;
          goto drop;
@@ -2504,8 +2545,9 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_IS_TRACED) && tcp0 != 0))
        {
          t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
-         clib_memcpy (&t0->tcp_header, tcp0, sizeof (t0->tcp_header));
-         clib_memcpy (&t0->tcp_connection, tc0, sizeof (t0->tcp_connection));
+         clib_memcpy_fast (&t0->tcp_header, tcp0, sizeof (t0->tcp_header));
+         clib_memcpy_fast (&t0->tcp_connection, tc0,
+                           sizeof (t0->tcp_connection));
        }
     }
 
@@ -3045,10 +3087,10 @@ tcp46_listen_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
        }
       else
        {
-         clib_memcpy (&child0->c_lcl_ip6, &ip60->dst_address,
-                      sizeof (ip6_address_t));
-         clib_memcpy (&child0->c_rmt_ip6, &ip60->src_address,
-                      sizeof (ip6_address_t));
+         clib_memcpy_fast (&child0->c_lcl_ip6, &ip60->dst_address,
+                           sizeof (ip6_address_t));
+         clib_memcpy_fast (&child0->c_rmt_ip6, &ip60->src_address,
+                           sizeof (ip6_address_t));
        }
 
       if (tcp_options_parse (th0, &child0->rcv_opts))
@@ -3090,6 +3132,7 @@ tcp46_listen_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
          goto drop;
        }
 
+      child0->tx_fifo_size = transport_tx_fifo_size (&child0->connection);
       tcp_send_synack (child0);
       tcp_timer_set (child0, TCP_TIMER_ESTABLISH, TCP_SYN_RCVD_TIME);
 
@@ -3098,8 +3141,9 @@ tcp46_listen_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
        {
          t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
-         clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header));
-         clib_memcpy (&t0->tcp_connection, lc0, sizeof (t0->tcp_connection));
+         clib_memcpy_fast (&t0->tcp_header, th0, sizeof (t0->tcp_header));
+         clib_memcpy_fast (&t0->tcp_connection, lc0,
+                           sizeof (t0->tcp_connection));
        }
 
       n_syns += (error0 == TCP_ERROR_NONE);