+#define udp_store_err_counters(vm, is_ip4, cnts) \
+{ \
+ int i; \
+ for (i = 0; i < UDP_N_ERROR; i++) \
+ if (cnts[i]) \
+ udp_input_inc_counter(vm, is_ip4, i, cnts[i]); \
+}
+
+#define udp_inc_err_counter(cnts, err, val) \
+{ \
+ cnts[err] += val; \
+}
+
+static void
+udp_trace_buffer (vlib_main_t * vm, vlib_node_runtime_t * node,
+ vlib_buffer_t * b, session_t * s, u16 error0)
+{
+ udp_input_trace_t *t;
+
+ if (PREDICT_TRUE (!(b->flags & VLIB_BUFFER_IS_TRACED)))
+ return;
+
+ t = vlib_add_trace (vm, node, b, sizeof (*t));
+ t->connection = s ? s->connection_index : ~0;
+ t->disposition = error0;
+ t->thread_index = s->thread_index;
+}
+
+static udp_connection_t *
+udp_connection_accept (udp_connection_t * listener, session_dgram_hdr_t * hdr,
+ u32 thread_index)
+{
+ udp_connection_t *uc;
+
+ uc = udp_connection_alloc (thread_index);
+ ip_copy (&uc->c_lcl_ip, &hdr->lcl_ip, hdr->is_ip4);
+ ip_copy (&uc->c_rmt_ip, &hdr->rmt_ip, hdr->is_ip4);
+ uc->c_lcl_port = hdr->lcl_port;
+ uc->c_rmt_port = hdr->rmt_port;
+ uc->c_is_ip4 = hdr->is_ip4;
+ uc->c_fib_index = listener->c_fib_index;
+ uc->mss = listener->mss;
+ uc->flags |= UDP_CONN_F_CONNECTED;
+
+ if (session_dgram_accept (&uc->connection, listener->c_s_index,
+ listener->c_thread_index))
+ {
+ udp_connection_free (uc);
+ return 0;
+ }
+ udp_connection_share_port (clib_net_to_host_u16
+ (uc->c_lcl_port), uc->c_is_ip4);
+ return uc;
+}
+
+static void
+udp_connection_enqueue (udp_connection_t * uc0, session_t * s0,
+ session_dgram_hdr_t * hdr0, u32 thread_index,
+ vlib_buffer_t * b, u8 queue_event, u32 * error0)
+{
+ int wrote0;
+
+ clib_spinlock_lock (&uc0->rx_lock);
+
+ if (svm_fifo_max_enqueue_prod (s0->rx_fifo)
+ < hdr0->data_length + sizeof (session_dgram_hdr_t))
+ {
+ *error0 = UDP_ERROR_FIFO_FULL;
+ goto unlock_rx_lock;
+ }
+
+ /* If session is owned by another thread and rx event needed,
+ * enqueue event now while we still have the peeker lock */
+ if (s0->thread_index != thread_index)
+ {
+ wrote0 = session_enqueue_dgram_connection (s0, hdr0, b,
+ TRANSPORT_PROTO_UDP,
+ /* queue event */ 0);
+ if (queue_event && !svm_fifo_has_event (s0->rx_fifo))
+ session_enqueue_notify (s0);
+ }
+ else
+ {
+ wrote0 = session_enqueue_dgram_connection (s0, hdr0, b,
+ TRANSPORT_PROTO_UDP,
+ queue_event);
+ }
+ ASSERT (wrote0 > 0);
+
+unlock_rx_lock:
+
+ clib_spinlock_unlock (&uc0->rx_lock);
+}
+
+always_inline session_t *
+udp_parse_and_lookup_buffer (vlib_buffer_t * b, session_dgram_hdr_t * hdr,
+ u8 is_ip4)
+{
+ udp_header_t *udp;
+ u32 fib_index;
+ session_t *s;
+
+ /* udp_local hands us a pointer to the udp data */
+ udp = (udp_header_t *) (vlib_buffer_get_current (b) - sizeof (*udp));
+ fib_index = vnet_buffer (b)->ip.fib_index;
+
+ hdr->data_offset = 0;
+ hdr->lcl_port = udp->dst_port;
+ hdr->rmt_port = udp->src_port;
+ hdr->is_ip4 = is_ip4;
+
+ if (is_ip4)
+ {
+ ip4_header_t *ip4;
+
+ /* TODO: must fix once udp_local does ip options correctly */
+ ip4 = (ip4_header_t *) (((u8 *) udp) - sizeof (*ip4));
+ ip_set (&hdr->lcl_ip, &ip4->dst_address, 1);
+ ip_set (&hdr->rmt_ip, &ip4->src_address, 1);
+ hdr->data_length = clib_net_to_host_u16 (ip4->length);
+ hdr->data_length -= sizeof (ip4_header_t) + sizeof (udp_header_t);
+ s = session_lookup_safe4 (fib_index, &ip4->dst_address,
+ &ip4->src_address, udp->dst_port,
+ udp->src_port, TRANSPORT_PROTO_UDP);
+ }
+ else
+ {
+ ip6_header_t *ip60;
+
+ ip60 = (ip6_header_t *) (((u8 *) udp) - sizeof (*ip60));
+ ip_set (&hdr->lcl_ip, &ip60->dst_address, 0);
+ ip_set (&hdr->rmt_ip, &ip60->src_address, 0);
+ hdr->data_length = clib_net_to_host_u16 (ip60->payload_length);
+ hdr->data_length -= sizeof (udp_header_t);
+ s = session_lookup_safe6 (fib_index, &ip60->dst_address,
+ &ip60->src_address, udp->dst_port,
+ udp->src_port, TRANSPORT_PROTO_UDP);
+ }
+
+ if (PREDICT_TRUE (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)))
+ b->current_length = hdr->data_length;
+ else
+ b->total_length_not_including_first_buffer = hdr->data_length
+ - b->current_length;
+
+ return s;
+}
+