+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ sr_localsid_trace_t *tr =
+ vlib_add_trace (vm, node, b0, sizeof (*tr));
+ tr->num_segments = 0;
+ clib_memcpy (tr->localsid.as_u8, ls0->localsid.as_u8,
+ sizeof (tr->localsid.as_u8));
+ tr->behavior = ls0->behavior;
+ if (ip0 == vlib_buffer_get_current (b0))
+ {
+ if (ip0->protocol == IP_PROTOCOL_IPV6_ROUTE
+ && sr0->type == ROUTING_HEADER_TYPE_SR)
+ {
+ clib_memcpy (tr->sr, sr0->segments, sr0->length * 8);
+ tr->num_segments =
+ sr0->length * 8 / sizeof (ip6_address_t);
+ tr->segments_left = sr0->segments_left;
+ }
+ }
+ else
+ tr->num_segments = 0xFF;
+ }
+
+ if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ sr_localsid_trace_t *tr =
+ vlib_add_trace (vm, node, b1, sizeof (*tr));
+ tr->num_segments = 0;
+ clib_memcpy (tr->localsid.as_u8, ls1->localsid.as_u8,
+ sizeof (tr->localsid.as_u8));
+ tr->behavior = ls1->behavior;
+ if (ip1 == vlib_buffer_get_current (b1))
+ {
+ if (ip1->protocol == IP_PROTOCOL_IPV6_ROUTE
+ && sr1->type == ROUTING_HEADER_TYPE_SR)
+ {
+ clib_memcpy (tr->sr, sr1->segments, sr1->length * 8);
+ tr->num_segments =
+ sr1->length * 8 / sizeof (ip6_address_t);
+ tr->segments_left = sr1->segments_left;
+ }
+ }
+ else
+ tr->num_segments = 0xFF;
+ }
+
+ if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ sr_localsid_trace_t *tr =
+ vlib_add_trace (vm, node, b2, sizeof (*tr));
+ tr->num_segments = 0;
+ clib_memcpy (tr->localsid.as_u8, ls2->localsid.as_u8,
+ sizeof (tr->localsid.as_u8));
+ tr->behavior = ls2->behavior;
+ if (ip2 == vlib_buffer_get_current (b2))
+ {
+ if (ip2->protocol == IP_PROTOCOL_IPV6_ROUTE
+ && sr2->type == ROUTING_HEADER_TYPE_SR)
+ {
+ clib_memcpy (tr->sr, sr2->segments, sr2->length * 8);
+ tr->num_segments =
+ sr2->length * 8 / sizeof (ip6_address_t);
+ tr->segments_left = sr2->segments_left;
+ }
+ }
+ else
+ tr->num_segments = 0xFF;
+ }
+
+ if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ sr_localsid_trace_t *tr =
+ vlib_add_trace (vm, node, b3, sizeof (*tr));
+ tr->num_segments = 0;
+ clib_memcpy (tr->localsid.as_u8, ls3->localsid.as_u8,
+ sizeof (tr->localsid.as_u8));
+ tr->behavior = ls3->behavior;
+ if (ip3 == vlib_buffer_get_current (b3))
+ {
+ if (ip3->protocol == IP_PROTOCOL_IPV6_ROUTE
+ && sr3->type == ROUTING_HEADER_TYPE_SR)
+ {
+ clib_memcpy (tr->sr, sr3->segments, sr3->length * 8);
+ tr->num_segments =
+ sr3->length * 8 / sizeof (ip6_address_t);
+ tr->segments_left = sr3->segments_left;
+ }
+ }
+ else
+ tr->num_segments = 0xFF;
+ }
+
+ vlib_increment_combined_counter
+ (((next0 ==
+ SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+ &(sm->sr_ls_valid_counters)), thread_index, ls0 - sm->localsids,
+ 1, vlib_buffer_length_in_chain (vm, b0));
+
+ vlib_increment_combined_counter
+ (((next1 ==
+ SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+ &(sm->sr_ls_valid_counters)), thread_index, ls1 - sm->localsids,
+ 1, vlib_buffer_length_in_chain (vm, b1));
+
+ vlib_increment_combined_counter
+ (((next2 ==
+ SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+ &(sm->sr_ls_valid_counters)), thread_index, ls2 - sm->localsids,
+ 1, vlib_buffer_length_in_chain (vm, b2));
+
+ vlib_increment_combined_counter
+ (((next3 ==
+ SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+ &(sm->sr_ls_valid_counters)), thread_index, ls3 - sm->localsids,
+ 1, vlib_buffer_length_in_chain (vm, b3));
+
+ vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
+ n_left_to_next, bi0, bi1, bi2, bi3,
+ next0, next1, next2, next3);
+ }
+
+ /* Single loop for potentially the last three packets */
+ while (n_left_from > 0 && n_left_to_next > 0)
+ {
+ u32 bi0;
+ vlib_buffer_t *b0;
+ ip6_header_t *ip0 = 0;
+ ip6_ext_header_t *prev0;
+ ip6_sr_header_t *sr0;
+ u32 next0 = SR_LOCALSID_NEXT_IP6_LOOKUP;
+ ip6_sr_localsid_t *ls0;
+
+ bi0 = from[0];
+ to_next[0] = bi0;
+ from += 1;
+ to_next += 1;
+ n_left_from -= 1;
+ n_left_to_next -= 1;
+
+ b0 = vlib_get_buffer (vm, bi0);
+ ip0 = vlib_buffer_get_current (b0);
+ sr0 =
+ ip6_ext_header_find (vm, b0, ip0, IP_PROTOCOL_IPV6_ROUTE, &prev0);
+
+ /* Lookup the SR End behavior based on IP DA (adj) */
+ ls0 =
+ pool_elt_at_index (sm->localsids,
+ vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+
+ /* SRH processing */
+ end_un_srh_processing (node, b0, ip0, sr0, ls0, &next0,
+ ls0->end_psp, prev0);
+
+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ sr_localsid_trace_t *tr =
+ vlib_add_trace (vm, node, b0, sizeof (*tr));
+ tr->num_segments = 0;
+ clib_memcpy (tr->localsid.as_u8, ls0->localsid.as_u8,
+ sizeof (tr->localsid.as_u8));
+ tr->behavior = ls0->behavior;
+ if (ip0 == vlib_buffer_get_current (b0))
+ {
+ if (ip0->protocol == IP_PROTOCOL_IPV6_ROUTE
+ && sr0->type == ROUTING_HEADER_TYPE_SR)
+ {
+ clib_memcpy (tr->sr, sr0->segments, sr0->length * 8);
+ tr->num_segments =
+ sr0->length * 8 / sizeof (ip6_address_t);
+ tr->segments_left = sr0->segments_left;
+ }
+ }
+ else
+ tr->num_segments = 0xFF;
+ }
+
+ vlib_increment_combined_counter
+ (((next0 ==
+ SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+ &(sm->sr_ls_valid_counters)), thread_index, ls0 - sm->localsids,
+ 1, vlib_buffer_length_in_chain (vm, b0));
+
+ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+ n_left_to_next, bi0, next0);
+ }
+ vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+ }
+ return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (sr_localsid_un_node) = {
+ .function = sr_localsid_un_fn,
+ .name = "sr-localsid-un",
+ .vector_size = sizeof (u32),
+ .format_trace = format_sr_localsid_trace,
+ .type = VLIB_NODE_TYPE_INTERNAL,
+ .n_errors = SR_LOCALSID_N_ERROR,
+ .error_strings = sr_localsid_error_strings,
+ .n_next_nodes = SR_LOCALSID_N_NEXT,
+ .next_nodes = {
+#define _(s,n) [SR_LOCALSID_NEXT_##s] = n,
+ foreach_sr_localsid_next
+#undef _
+ },
+};
+/* *INDENT-ON* */
+
+static uword
+sr_localsid_un_perf_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
+ vlib_frame_t * from_frame)
+{
+ u32 n_left_from, next_index, *from, *to_next;
+ ip6_sr_main_t *sm = &sr_main;
+ from = vlib_frame_vector_args (from_frame);
+ n_left_from = from_frame->n_vectors;
+ next_index = node->cached_next_index;
+ u32 thread_index = vm->thread_index;
+
+ while (n_left_from > 0)
+ {
+ u32 n_left_to_next;
+ vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+ /* Quad - Loop */
+ while (n_left_from >= 8 && n_left_to_next >= 4)
+ {
+ u32 bi0, bi1, bi2, bi3;
+ vlib_buffer_t *b0, *b1, *b2, *b3;
+ ip6_header_t *ip0, *ip1, *ip2, *ip3;
+ u32 next0, next1, next2, next3;
+ next0 = next1 = next2 = next3 = SR_LOCALSID_NEXT_IP6_LOOKUP;
+ ip6_sr_localsid_t *ls0, *ls1, *ls2, *ls3;
+
+ /* Prefetch next iteration. */
+ {
+ vlib_buffer_t *p4, *p5, *p6, *p7;
+
+ p4 = vlib_get_buffer (vm, from[4]);
+ p5 = vlib_get_buffer (vm, from[5]);
+ p6 = vlib_get_buffer (vm, from[6]);
+ p7 = vlib_get_buffer (vm, from[7]);
+
+ /* Prefetch the buffer header and packet for the N+2 loop iteration */
+ vlib_prefetch_buffer_header (p4, LOAD);
+ vlib_prefetch_buffer_header (p5, LOAD);
+ vlib_prefetch_buffer_header (p6, LOAD);
+ vlib_prefetch_buffer_header (p7, LOAD);
+
+ CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
+ CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
+ CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
+ CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+ }
+
+ to_next[0] = bi0 = from[0];
+ to_next[1] = bi1 = from[1];
+ to_next[2] = bi2 = from[2];
+ to_next[3] = bi3 = from[3];
+ from += 4;
+ to_next += 4;
+ n_left_from -= 4;
+ n_left_to_next -= 4;
+
+ b0 = vlib_get_buffer (vm, bi0);
+ b1 = vlib_get_buffer (vm, bi1);
+ b2 = vlib_get_buffer (vm, bi2);
+ b3 = vlib_get_buffer (vm, bi3);
+
+ ip0 = vlib_buffer_get_current (b0);
+ ip1 = vlib_buffer_get_current (b1);
+ ip2 = vlib_buffer_get_current (b2);
+ ip3 = vlib_buffer_get_current (b3);
+
+ ls0 =
+ pool_elt_at_index (sm->localsids,
+ vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+ ls1 =
+ pool_elt_at_index (sm->localsids,
+ vnet_buffer (b1)->ip.adj_index[VLIB_TX]);
+ ls2 =
+ pool_elt_at_index (sm->localsids,
+ vnet_buffer (b2)->ip.adj_index[VLIB_TX]);
+ ls3 =
+ pool_elt_at_index (sm->localsids,
+ vnet_buffer (b3)->ip.adj_index[VLIB_TX]);
+
+ end_un_processing (ip0, ls0);
+ end_un_processing (ip1, ls1);
+ end_un_processing (ip2, ls2);
+ end_un_processing (ip3, ls3);
+
+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ sr_localsid_trace_t *tr =
+ vlib_add_trace (vm, node, b0, sizeof (*tr));
+ tr->num_segments = 0;
+ clib_memcpy (tr->localsid.as_u8, ls0->localsid.as_u8,
+ sizeof (tr->localsid.as_u8));
+ tr->behavior = ls0->behavior;
+ }
+
+ if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ sr_localsid_trace_t *tr =
+ vlib_add_trace (vm, node, b1, sizeof (*tr));
+ tr->num_segments = 0;
+ clib_memcpy (tr->localsid.as_u8, ls1->localsid.as_u8,
+ sizeof (tr->localsid.as_u8));
+ tr->behavior = ls1->behavior;
+ }
+
+ if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ sr_localsid_trace_t *tr =
+ vlib_add_trace (vm, node, b2, sizeof (*tr));
+ tr->num_segments = 0;
+ clib_memcpy (tr->localsid.as_u8, ls2->localsid.as_u8,
+ sizeof (tr->localsid.as_u8));
+ tr->behavior = ls2->behavior;
+ }
+
+ if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ sr_localsid_trace_t *tr =
+ vlib_add_trace (vm, node, b3, sizeof (*tr));
+ tr->num_segments = 0;
+ clib_memcpy (tr->localsid.as_u8, ls3->localsid.as_u8,
+ sizeof (tr->localsid.as_u8));
+ tr->behavior = ls3->behavior;
+ }
+
+ vlib_increment_combined_counter
+ (&(sm->sr_ls_valid_counters), thread_index, ls0 - sm->localsids,
+ 1, vlib_buffer_length_in_chain (vm, b0));
+
+ vlib_increment_combined_counter
+ (&(sm->sr_ls_valid_counters), thread_index, ls1 - sm->localsids,
+ 1, vlib_buffer_length_in_chain (vm, b1));
+
+ vlib_increment_combined_counter
+ (&(sm->sr_ls_valid_counters), thread_index, ls2 - sm->localsids,
+ 1, vlib_buffer_length_in_chain (vm, b2));
+
+ vlib_increment_combined_counter
+ (&(sm->sr_ls_valid_counters), thread_index, ls3 - sm->localsids,
+ 1, vlib_buffer_length_in_chain (vm, b3));
+
+ vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
+ n_left_to_next, bi0, bi1, bi2, bi3,
+ next0, next1, next2, next3);
+ }
+
+ /* Single loop for potentially the last three packets */
+ while (n_left_from > 0 && n_left_to_next > 0)
+ {
+ u32 bi0;
+ vlib_buffer_t *b0;
+ ip6_header_t *ip0 = 0;
+ u32 next0 = SR_LOCALSID_NEXT_IP6_LOOKUP;
+ ip6_sr_localsid_t *ls0;
+
+ bi0 = from[0];
+ to_next[0] = bi0;
+ from += 1;
+ to_next += 1;
+ n_left_from -= 1;
+ n_left_to_next -= 1;
+
+ b0 = vlib_get_buffer (vm, bi0);
+ ip0 = vlib_buffer_get_current (b0);
+
+ /* Lookup the SR End behavior based on IP DA (adj) */
+ ls0 =
+ pool_elt_at_index (sm->localsids,
+ vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+
+ /* SRH processing */
+ end_un_processing (ip0, ls0);
+
+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ sr_localsid_trace_t *tr =
+ vlib_add_trace (vm, node, b0, sizeof (*tr));
+ tr->num_segments = 0;
+ clib_memcpy (tr->localsid.as_u8, ls0->localsid.as_u8,
+ sizeof (tr->localsid.as_u8));
+ tr->behavior = ls0->behavior;
+ }
+
+ vlib_increment_combined_counter
+ (&(sm->sr_ls_valid_counters), thread_index, ls0 - sm->localsids,
+ 1, vlib_buffer_length_in_chain (vm, b0));
+
+ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+ n_left_to_next, bi0, next0);
+ }
+ vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+ }
+ return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (sr_localsid_un_perf_node) = {
+ .function = sr_localsid_un_perf_fn,
+ .name = "sr-localsid-un-perf",
+ .vector_size = sizeof (u32),
+ .format_trace = format_sr_localsid_trace,
+ .type = VLIB_NODE_TYPE_INTERNAL,
+ .n_errors = SR_LOCALSID_N_ERROR,
+ .error_strings = sr_localsid_error_strings,
+ .n_next_nodes = SR_LOCALSID_N_NEXT,
+ .next_nodes = {
+#define _(s,n) [SR_LOCALSID_NEXT_##s] = n,
+ foreach_sr_localsid_next
+#undef _
+ },
+};
+/* *INDENT-ON* */
+
+static u8 *
+format_sr_dpo (u8 * s, va_list * args)
+{
+ index_t index = va_arg (*args, index_t);
+ CLIB_UNUSED (u32 indent) = va_arg (*args, u32);
+
+ return (format (s, "SR: localsid_index:[%d]", index));
+}
+
+const static dpo_vft_t sr_loc_vft = {
+ .dv_lock = sr_dpo_lock,
+ .dv_unlock = sr_dpo_unlock,
+ .dv_format = format_sr_dpo,
+};
+
+const static char *const sr_loc_ip6_nodes[] = {
+ "sr-localsid",
+ NULL,
+};
+
+const static char *const *const sr_loc_nodes[DPO_PROTO_NUM] = {
+ [DPO_PROTO_IP6] = sr_loc_ip6_nodes,
+};
+
+const static char *const sr_loc_d_ip6_nodes[] = {
+ "sr-localsid-d",
+ NULL,
+};
+
+const static char *const *const sr_loc_d_nodes[DPO_PROTO_NUM] = {
+ [DPO_PROTO_IP6] = sr_loc_d_ip6_nodes,
+};
+
+const static char *const sr_loc_un_ip6_nodes[] = {
+ "sr-localsid-un",
+ NULL,
+};
+
+const static char *const *const sr_loc_un_nodes[DPO_PROTO_NUM] = {
+ [DPO_PROTO_IP6] = sr_loc_un_ip6_nodes,
+};
+
+const static char *const sr_loc_un_perf_ip6_nodes[] = {
+ "sr-localsid-un-perf",
+ NULL,
+};
+
+const static char *const *const sr_loc_un_perf_nodes[DPO_PROTO_NUM] = {
+ [DPO_PROTO_IP6] = sr_loc_un_perf_ip6_nodes,
+};