examples/ip_pipeline/pipeline/pipeline_routing_be.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_malloc.h>
  13 #include <rte_ip.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_table_lpm.h>
  16 #include <rte_table_hash.h>
  17 #include <rte_pipeline.h>
  18
  19 #include "pipeline_routing_be.h"
  20 #include "pipeline_actions_common.h"
  21 #include "parser.h"
  22 #include "hash_func.h"
  23
  24 #define MPLS_LABEL(label, exp, s, ttl)                                  \
  25         (((((uint64_t) (label)) & 0xFFFFFLLU) << 12) |          \
  26         ((((uint64_t) (exp)) & 0x7LLU) << 9) |                          \
  27         ((((uint64_t) (s)) & 0x1LLU) << 8) |                            \
  28         (((uint64_t) (ttl)) & 0xFFLU))
  29
  30 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe,         \
  31         traffic_class, queue, color)                            \
  32         ((((uint64_t) (queue)) & 0x3) |                \
  33         ((((uint64_t) (traffic_class)) & 0x3) << 2) |  \
  34         ((((uint64_t) (color)) & 0x3) << 4) |          \
  35         ((((uint64_t) (subport)) & 0xFFFF) << 16) |    \
  36         ((((uint64_t) (pipe)) & 0xFFFFFFFF) << 32))
  37
  38
  39 /* Network Byte Order (NBO) */
  40 #define SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr, ethertype)       \
  41         (((uint64_t) macaddr) | (((uint64_t) rte_cpu_to_be_16(ethertype)) << 48))
  42
  43 #ifndef PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s
  44 #define PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s 256
  45 #endif
  46
  47 struct pipeline_routing {
  48         struct pipeline p;
  49         struct pipeline_routing_params params;
  50         pipeline_msg_req_handler custom_handlers[PIPELINE_ROUTING_MSG_REQS];
  51         uint64_t macaddr[PIPELINE_MAX_PORT_OUT];
  52 } __rte_cache_aligned;
  53
  54 /*
  55  * Message handlers
  56  */
  57 static void *
  58 pipeline_routing_msg_req_custom_handler(struct pipeline *p, void *msg);
  59
  60 static pipeline_msg_req_handler handlers[] = {
  61         [PIPELINE_MSG_REQ_PING] =
  62                 pipeline_msg_req_ping_handler,
  63         [PIPELINE_MSG_REQ_STATS_PORT_IN] =
  64                 pipeline_msg_req_stats_port_in_handler,
  65         [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
  66                 pipeline_msg_req_stats_port_out_handler,
  67         [PIPELINE_MSG_REQ_STATS_TABLE] =
  68                 pipeline_msg_req_stats_table_handler,
  69         [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
  70                 pipeline_msg_req_port_in_enable_handler,
  71         [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
  72                 pipeline_msg_req_port_in_disable_handler,
  73         [PIPELINE_MSG_REQ_CUSTOM] =
  74                 pipeline_routing_msg_req_custom_handler,
  75 };
  76
  77 static void *
  78 pipeline_routing_msg_req_route_add_handler(struct pipeline *p,
  79         void *msg);
  80
  81 static void *
  82 pipeline_routing_msg_req_route_del_handler(struct pipeline *p,
  83         void *msg);
  84
  85 static void *
  86 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
  87         void *msg);
  88
  89 static void *
  90 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
  91         void *msg);
  92
  93 static void *
  94 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p,
  95         void *msg);
  96
  97 static void *
  98 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p,
  99         void *msg);
 100
 101 static void *
 102 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p,
 103         void *msg);
 104
 105 static void *
 106 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p,
 107         void *msg);
 108
 109 static void *
 110 pipeline_routing_msg_req_set_macaddr_handler(struct pipeline *p,
 111         void *msg);
 112
 113 static pipeline_msg_req_handler custom_handlers[] = {
 114         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD] =
 115                 pipeline_routing_msg_req_route_add_handler,
 116         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL] =
 117                 pipeline_routing_msg_req_route_del_handler,
 118         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD_DEFAULT] =
 119                 pipeline_routing_msg_req_route_add_default_handler,
 120         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL_DEFAULT] =
 121                 pipeline_routing_msg_req_route_del_default_handler,
 122         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD] =
 123                 pipeline_routing_msg_req_arp_add_handler,
 124         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL] =
 125                 pipeline_routing_msg_req_arp_del_handler,
 126         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD_DEFAULT] =
 127                 pipeline_routing_msg_req_arp_add_default_handler,
 128         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL_DEFAULT] =
 129                 pipeline_routing_msg_req_arp_del_default_handler,
 130         [PIPELINE_ROUTING_MSG_REQ_SET_MACADDR] =
 131                 pipeline_routing_msg_req_set_macaddr_handler,
 132 };
 133
 134 /*
 135  * Routing table
 136  */
 137 struct routing_table_entry {
 138         struct rte_pipeline_table_entry head;
 139         uint32_t flags;
 140         uint32_t port_id; /* Output port ID */
 141         uint32_t ip; /* Next hop IP address (only valid for remote routes) */
 142
 143         /* ether_l2 */
 144         uint16_t data_offset;
 145         uint16_t ether_l2_length;
 146         uint64_t slab[4];
 147         uint16_t slab_offset[4];
 148 };
 149
 150 struct layout {
 151         uint16_t a;
 152         uint32_t b;
 153         uint16_t c;
 154 } __attribute__((__packed__));
 155
 156 #define MACADDR_DST_WRITE(slab_ptr, slab)                       \
 157 {                                                               \
 158         struct layout *dst = (struct layout *) (slab_ptr);      \
 159         struct layout *src = (struct layout *) &(slab);         \
 160                                                                 \
 161         dst->b = src->b;                                        \
 162         dst->c = src->c;                                        \
 163 }
 164
 165 static __rte_always_inline void
 166 pkt_work_routing(
 167         struct rte_mbuf *pkt,
 168         struct rte_pipeline_table_entry *table_entry,
 169         void *arg,
 170         int arp,
 171         int qinq,
 172         int qinq_sched,
 173         int mpls,
 174         int mpls_color_mark)
 175 {
 176         struct pipeline_routing *p_rt = arg;
 177
 178         struct routing_table_entry *entry =
 179                 (struct routing_table_entry *) table_entry;
 180
 181         struct ipv4_hdr *ip = (struct ipv4_hdr *)
 182                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.ip_hdr_offset);
 183
 184         enum rte_meter_color pkt_color = (enum rte_meter_color)
 185                 RTE_MBUF_METADATA_UINT32(pkt, p_rt->params.color_offset);
 186
 187         struct pipeline_routing_arp_key_ipv4 *arp_key =
 188                 (struct pipeline_routing_arp_key_ipv4 *)
 189                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.arp_key_offset);
 190
 191         uint64_t *slab0_ptr, *slab1_ptr, *slab2_ptr, *slab3_ptr, sched;
 192         uint32_t ip_da, nh_ip, port_id;
 193         uint16_t total_length, data_offset, ether_l2_length;
 194
 195         /* Read */
 196         total_length = rte_bswap16(ip->total_length);
 197         ip_da = ip->dst_addr;
 198         data_offset = entry->data_offset;
 199         ether_l2_length = entry->ether_l2_length;
 200         slab0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[0]);
 201         slab1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[1]);
 202         slab2_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[2]);
 203         slab3_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[3]);
 204
 205         if (arp) {
 206                 port_id = entry->port_id;
 207                 nh_ip = entry->ip;
 208                 if (entry->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 209                         nh_ip = ip_da;
 210         }
 211
 212         /* Compute */
 213         total_length += ether_l2_length;
 214
 215         if (qinq && qinq_sched) {
 216                 uint32_t dscp = ip->type_of_service >> 2;
 217                 uint32_t svlan, cvlan, tc, tc_q;
 218
 219                 if (qinq_sched == 1) {
 220                         uint64_t slab_qinq = rte_bswap64(entry->slab[0]);
 221
 222                         svlan = (slab_qinq >> 48) & 0xFFF;
 223                         cvlan = (slab_qinq >> 16) & 0xFFF;
 224                         tc = (dscp >> 2) & 0x3;
 225                         tc_q = dscp & 0x3;
 226                 } else {
 227                         uint32_t ip_src = rte_bswap32(ip->src_addr);
 228
 229                         svlan = 0;
 230                         cvlan = (ip_src >> 16) & 0xFFF;
 231                         tc = (ip_src >> 2) & 0x3;
 232                         tc_q = ip_src & 0x3;
 233                 }
 234                 sched = RTE_SCHED_PORT_HIERARCHY(svlan,
 235                         cvlan,
 236                         tc,
 237                         tc_q,
 238                         e_RTE_METER_GREEN);
 239         }
 240
 241         /* Write */
 242         pkt->data_off = data_offset;
 243         pkt->data_len = total_length;
 244         pkt->pkt_len = total_length;
 245
 246         if ((qinq == 0) && (mpls == 0)) {
 247                 *slab0_ptr = entry->slab[0];
 248
 249                 if (arp == 0)
 250                         MACADDR_DST_WRITE(slab1_ptr, entry->slab[1]);
 251         }
 252
 253         if (qinq) {
 254                 *slab0_ptr = entry->slab[0];
 255                 *slab1_ptr = entry->slab[1];
 256
 257                 if (arp == 0)
 258                         MACADDR_DST_WRITE(slab2_ptr, entry->slab[2]);
 259
 260                 if (qinq_sched) {
 261                         pkt->hash.sched.lo = sched & 0xFFFFFFFF;
 262                         pkt->hash.sched.hi = sched >> 32;
 263                 }
 264         }
 265
 266         if (mpls) {
 267                 if (mpls_color_mark) {
 268                         uint64_t mpls_exp = rte_bswap64(
 269                                 (MPLS_LABEL(0, pkt_color, 0, 0) << 32) |
 270                                 MPLS_LABEL(0, pkt_color, 0, 0));
 271
 272                         *slab0_ptr = entry->slab[0] | mpls_exp;
 273                         *slab1_ptr = entry->slab[1] | mpls_exp;
 274                         *slab2_ptr = entry->slab[2];
 275                 } else {
 276                         *slab0_ptr = entry->slab[0];
 277                         *slab1_ptr = entry->slab[1];
 278                         *slab2_ptr = entry->slab[2];
 279                 }
 280
 281                 if (arp == 0)
 282                         MACADDR_DST_WRITE(slab3_ptr, entry->slab[3]);
 283         }
 284
 285         if (arp) {
 286                 arp_key->port_id = port_id;
 287                 arp_key->ip = nh_ip;
 288         }
 289 }
 290
 291 static __rte_always_inline void
 292 pkt4_work_routing(
 293         struct rte_mbuf **pkts,
 294         struct rte_pipeline_table_entry **table_entries,
 295         void *arg,
 296         int arp,
 297         int qinq,
 298         int qinq_sched,
 299         int mpls,
 300         int mpls_color_mark)
 301 {
 302         struct pipeline_routing *p_rt = arg;
 303
 304         struct routing_table_entry *entry0 =
 305                 (struct routing_table_entry *) table_entries[0];
 306         struct routing_table_entry *entry1 =
 307                 (struct routing_table_entry *) table_entries[1];
 308         struct routing_table_entry *entry2 =
 309                 (struct routing_table_entry *) table_entries[2];
 310         struct routing_table_entry *entry3 =
 311                 (struct routing_table_entry *) table_entries[3];
 312
 313         struct ipv4_hdr *ip0 = (struct ipv4_hdr *)
 314                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 315                         p_rt->params.ip_hdr_offset);
 316         struct ipv4_hdr *ip1 = (struct ipv4_hdr *)
 317                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 318                         p_rt->params.ip_hdr_offset);
 319         struct ipv4_hdr *ip2 = (struct ipv4_hdr *)
 320                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 321                         p_rt->params.ip_hdr_offset);
 322         struct ipv4_hdr *ip3 = (struct ipv4_hdr *)
 323                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 324                         p_rt->params.ip_hdr_offset);
 325
 326         enum rte_meter_color pkt0_color = (enum rte_meter_color)
 327                 RTE_MBUF_METADATA_UINT32(pkts[0], p_rt->params.color_offset);
 328         enum rte_meter_color pkt1_color = (enum rte_meter_color)
 329                 RTE_MBUF_METADATA_UINT32(pkts[1], p_rt->params.color_offset);
 330         enum rte_meter_color pkt2_color = (enum rte_meter_color)
 331                 RTE_MBUF_METADATA_UINT32(pkts[2], p_rt->params.color_offset);
 332         enum rte_meter_color pkt3_color = (enum rte_meter_color)
 333                 RTE_MBUF_METADATA_UINT32(pkts[3], p_rt->params.color_offset);
 334
 335         struct pipeline_routing_arp_key_ipv4 *arp_key0 =
 336                 (struct pipeline_routing_arp_key_ipv4 *)
 337                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 338                         p_rt->params.arp_key_offset);
 339         struct pipeline_routing_arp_key_ipv4 *arp_key1 =
 340                 (struct pipeline_routing_arp_key_ipv4 *)
 341                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 342                         p_rt->params.arp_key_offset);
 343         struct pipeline_routing_arp_key_ipv4 *arp_key2 =
 344                 (struct pipeline_routing_arp_key_ipv4 *)
 345                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 346                         p_rt->params.arp_key_offset);
 347         struct pipeline_routing_arp_key_ipv4 *arp_key3 =
 348                 (struct pipeline_routing_arp_key_ipv4 *)
 349                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 350                         p_rt->params.arp_key_offset);
 351
 352         uint64_t *slab0_ptr0, *slab1_ptr0, *slab2_ptr0, *slab3_ptr0;
 353         uint64_t *slab0_ptr1, *slab1_ptr1, *slab2_ptr1, *slab3_ptr1;
 354         uint64_t *slab0_ptr2, *slab1_ptr2, *slab2_ptr2, *slab3_ptr2;
 355         uint64_t *slab0_ptr3, *slab1_ptr3, *slab2_ptr3, *slab3_ptr3;
 356         uint64_t sched0, sched1, sched2, sched3;
 357
 358         uint32_t ip_da0, nh_ip0, port_id0;
 359         uint32_t ip_da1, nh_ip1, port_id1;
 360         uint32_t ip_da2, nh_ip2, port_id2;
 361         uint32_t ip_da3, nh_ip3, port_id3;
 362
 363         uint16_t total_length0, data_offset0, ether_l2_length0;
 364         uint16_t total_length1, data_offset1, ether_l2_length1;
 365         uint16_t total_length2, data_offset2, ether_l2_length2;
 366         uint16_t total_length3, data_offset3, ether_l2_length3;
 367
 368         /* Read */
 369         total_length0 = rte_bswap16(ip0->total_length);
 370         total_length1 = rte_bswap16(ip1->total_length);
 371         total_length2 = rte_bswap16(ip2->total_length);
 372         total_length3 = rte_bswap16(ip3->total_length);
 373
 374         ip_da0 = ip0->dst_addr;
 375         ip_da1 = ip1->dst_addr;
 376         ip_da2 = ip2->dst_addr;
 377         ip_da3 = ip3->dst_addr;
 378
 379         data_offset0 = entry0->data_offset;
 380         data_offset1 = entry1->data_offset;
 381         data_offset2 = entry2->data_offset;
 382         data_offset3 = entry3->data_offset;
 383
 384         ether_l2_length0 = entry0->ether_l2_length;
 385         ether_l2_length1 = entry1->ether_l2_length;
 386         ether_l2_length2 = entry2->ether_l2_length;
 387         ether_l2_length3 = entry3->ether_l2_length;
 388
 389         slab0_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 390                 entry0->slab_offset[0]);
 391         slab1_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 392                 entry0->slab_offset[1]);
 393         slab2_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 394                 entry0->slab_offset[2]);
 395         slab3_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 396                 entry0->slab_offset[3]);
 397
 398         slab0_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 399                 entry1->slab_offset[0]);
 400         slab1_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 401                 entry1->slab_offset[1]);
 402         slab2_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 403                 entry1->slab_offset[2]);
 404         slab3_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 405                 entry1->slab_offset[3]);
 406
 407         slab0_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 408                 entry2->slab_offset[0]);
 409         slab1_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 410                 entry2->slab_offset[1]);
 411         slab2_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 412                 entry2->slab_offset[2]);
 413         slab3_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 414                 entry2->slab_offset[3]);
 415
 416         slab0_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 417                 entry3->slab_offset[0]);
 418         slab1_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 419                 entry3->slab_offset[1]);
 420         slab2_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 421                 entry3->slab_offset[2]);
 422         slab3_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 423                 entry3->slab_offset[3]);
 424
 425         if (arp) {
 426                 port_id0 = entry0->port_id;
 427                 nh_ip0 = entry0->ip;
 428                 if (entry0->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 429                         nh_ip0 = ip_da0;
 430
 431                 port_id1 = entry1->port_id;
 432                 nh_ip1 = entry1->ip;
 433                 if (entry1->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 434                         nh_ip1 = ip_da1;
 435
 436                 port_id2 = entry2->port_id;
 437                 nh_ip2 = entry2->ip;
 438                 if (entry2->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 439                         nh_ip2 = ip_da2;
 440
 441                 port_id3 = entry3->port_id;
 442                 nh_ip3 = entry3->ip;
 443                 if (entry3->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 444                         nh_ip3 = ip_da3;
 445         }
 446
 447         /* Compute */
 448         total_length0 += ether_l2_length0;
 449         total_length1 += ether_l2_length1;
 450         total_length2 += ether_l2_length2;
 451         total_length3 += ether_l2_length3;
 452
 453         if (qinq && qinq_sched) {
 454                 uint32_t dscp0 = ip0->type_of_service >> 2;
 455                 uint32_t dscp1 = ip1->type_of_service >> 2;
 456                 uint32_t dscp2 = ip2->type_of_service >> 2;
 457                 uint32_t dscp3 = ip3->type_of_service >> 2;
 458                 uint32_t svlan0, cvlan0, tc0, tc_q0;
 459                 uint32_t svlan1, cvlan1, tc1, tc_q1;
 460                 uint32_t svlan2, cvlan2, tc2, tc_q2;
 461                 uint32_t svlan3, cvlan3, tc3, tc_q3;
 462
 463                 if (qinq_sched == 1) {
 464                         uint64_t slab_qinq0 = rte_bswap64(entry0->slab[0]);
 465                         uint64_t slab_qinq1 = rte_bswap64(entry1->slab[0]);
 466                         uint64_t slab_qinq2 = rte_bswap64(entry2->slab[0]);
 467                         uint64_t slab_qinq3 = rte_bswap64(entry3->slab[0]);
 468
 469                         svlan0 = (slab_qinq0 >> 48) & 0xFFF;
 470                         svlan1 = (slab_qinq1 >> 48) & 0xFFF;
 471                         svlan2 = (slab_qinq2 >> 48) & 0xFFF;
 472                         svlan3 = (slab_qinq3 >> 48) & 0xFFF;
 473
 474                         cvlan0 = (slab_qinq0 >> 16) & 0xFFF;
 475                         cvlan1 = (slab_qinq1 >> 16) & 0xFFF;
 476                         cvlan2 = (slab_qinq2 >> 16) & 0xFFF;
 477                         cvlan3 = (slab_qinq3 >> 16) & 0xFFF;
 478
 479                         tc0 = (dscp0 >> 2) & 0x3;
 480                         tc1 = (dscp1 >> 2) & 0x3;
 481                         tc2 = (dscp2 >> 2) & 0x3;
 482                         tc3 = (dscp3 >> 2) & 0x3;
 483
 484                         tc_q0 = dscp0 & 0x3;
 485                         tc_q1 = dscp1 & 0x3;
 486                         tc_q2 = dscp2 & 0x3;
 487                         tc_q3 = dscp3 & 0x3;
 488                 } else {
 489                         uint32_t ip_src0 = rte_bswap32(ip0->src_addr);
 490                         uint32_t ip_src1 = rte_bswap32(ip1->src_addr);
 491                         uint32_t ip_src2 = rte_bswap32(ip2->src_addr);
 492                         uint32_t ip_src3 = rte_bswap32(ip3->src_addr);
 493
 494                         svlan0 = 0;
 495                         svlan1 = 0;
 496                         svlan2 = 0;
 497                         svlan3 = 0;
 498
 499                         cvlan0 = (ip_src0 >> 16) & 0xFFF;
 500                         cvlan1 = (ip_src1 >> 16) & 0xFFF;
 501                         cvlan2 = (ip_src2 >> 16) & 0xFFF;
 502                         cvlan3 = (ip_src3 >> 16) & 0xFFF;
 503
 504                         tc0 = (ip_src0 >> 2) & 0x3;
 505                         tc1 = (ip_src1 >> 2) & 0x3;
 506                         tc2 = (ip_src2 >> 2) & 0x3;
 507                         tc3 = (ip_src3 >> 2) & 0x3;
 508
 509                         tc_q0 = ip_src0 & 0x3;
 510                         tc_q1 = ip_src1 & 0x3;
 511                         tc_q2 = ip_src2 & 0x3;
 512                         tc_q3 = ip_src3 & 0x3;
 513                 }
 514
 515                 sched0 = RTE_SCHED_PORT_HIERARCHY(svlan0,
 516                         cvlan0,
 517                         tc0,
 518                         tc_q0,
 519                         e_RTE_METER_GREEN);
 520                 sched1 = RTE_SCHED_PORT_HIERARCHY(svlan1,
 521                         cvlan1,
 522                         tc1,
 523                         tc_q1,
 524                         e_RTE_METER_GREEN);
 525                 sched2 = RTE_SCHED_PORT_HIERARCHY(svlan2,
 526                         cvlan2,
 527                         tc2,
 528                         tc_q2,
 529                         e_RTE_METER_GREEN);
 530                 sched3 = RTE_SCHED_PORT_HIERARCHY(svlan3,
 531                         cvlan3,
 532                         tc3,
 533                         tc_q3,
 534                         e_RTE_METER_GREEN);
 535
 536         }
 537
 538         /* Write */
 539         pkts[0]->data_off = data_offset0;
 540         pkts[1]->data_off = data_offset1;
 541         pkts[2]->data_off = data_offset2;
 542         pkts[3]->data_off = data_offset3;
 543
 544         pkts[0]->data_len = total_length0;
 545         pkts[1]->data_len = total_length1;
 546         pkts[2]->data_len = total_length2;
 547         pkts[3]->data_len = total_length3;
 548
 549         pkts[0]->pkt_len = total_length0;
 550         pkts[1]->pkt_len = total_length1;
 551         pkts[2]->pkt_len = total_length2;
 552         pkts[3]->pkt_len = total_length3;
 553
 554         if ((qinq == 0) && (mpls == 0)) {
 555                 *slab0_ptr0 = entry0->slab[0];
 556                 *slab0_ptr1 = entry1->slab[0];
 557                 *slab0_ptr2 = entry2->slab[0];
 558                 *slab0_ptr3 = entry3->slab[0];
 559
 560                 if (arp == 0) {
 561                         MACADDR_DST_WRITE(slab1_ptr0, entry0->slab[1]);
 562                         MACADDR_DST_WRITE(slab1_ptr1, entry1->slab[1]);
 563                         MACADDR_DST_WRITE(slab1_ptr2, entry2->slab[1]);
 564                         MACADDR_DST_WRITE(slab1_ptr3, entry3->slab[1]);
 565                 }
 566         }
 567
 568         if (qinq) {
 569                 *slab0_ptr0 = entry0->slab[0];
 570                 *slab0_ptr1 = entry1->slab[0];
 571                 *slab0_ptr2 = entry2->slab[0];
 572                 *slab0_ptr3 = entry3->slab[0];
 573
 574                 *slab1_ptr0 = entry0->slab[1];
 575                 *slab1_ptr1 = entry1->slab[1];
 576                 *slab1_ptr2 = entry2->slab[1];
 577                 *slab1_ptr3 = entry3->slab[1];
 578
 579                 if (arp == 0) {
 580                         MACADDR_DST_WRITE(slab2_ptr0, entry0->slab[2]);
 581                         MACADDR_DST_WRITE(slab2_ptr1, entry1->slab[2]);
 582                         MACADDR_DST_WRITE(slab2_ptr2, entry2->slab[2]);
 583                         MACADDR_DST_WRITE(slab2_ptr3, entry3->slab[2]);
 584                 }
 585
 586                 if (qinq_sched) {
 587                         pkts[0]->hash.sched.lo = sched0 & 0xFFFFFFFF;
 588                         pkts[0]->hash.sched.hi = sched0 >> 32;
 589                         pkts[1]->hash.sched.lo = sched1 & 0xFFFFFFFF;
 590                         pkts[1]->hash.sched.hi = sched1 >> 32;
 591                         pkts[2]->hash.sched.lo = sched2 & 0xFFFFFFFF;
 592                         pkts[2]->hash.sched.hi = sched2 >> 32;
 593                         pkts[3]->hash.sched.lo = sched3 & 0xFFFFFFFF;
 594                         pkts[3]->hash.sched.hi = sched3 >> 32;
 595                 }
 596         }
 597
 598         if (mpls) {
 599                 if (mpls_color_mark) {
 600                         uint64_t mpls_exp0 = rte_bswap64(
 601                                 (MPLS_LABEL(0, pkt0_color, 0, 0) << 32) |
 602                                 MPLS_LABEL(0, pkt0_color, 0, 0));
 603                         uint64_t mpls_exp1 = rte_bswap64(
 604                                 (MPLS_LABEL(0, pkt1_color, 0, 0) << 32) |
 605                                 MPLS_LABEL(0, pkt1_color, 0, 0));
 606                         uint64_t mpls_exp2 = rte_bswap64(
 607                                 (MPLS_LABEL(0, pkt2_color, 0, 0) << 32) |
 608                                 MPLS_LABEL(0, pkt2_color, 0, 0));
 609                         uint64_t mpls_exp3 = rte_bswap64(
 610                                 (MPLS_LABEL(0, pkt3_color, 0, 0) << 32) |
 611                                 MPLS_LABEL(0, pkt3_color, 0, 0));
 612
 613                         *slab0_ptr0 = entry0->slab[0] | mpls_exp0;
 614                         *slab0_ptr1 = entry1->slab[0] | mpls_exp1;
 615                         *slab0_ptr2 = entry2->slab[0] | mpls_exp2;
 616                         *slab0_ptr3 = entry3->slab[0] | mpls_exp3;
 617
 618                         *slab1_ptr0 = entry0->slab[1] | mpls_exp0;
 619                         *slab1_ptr1 = entry1->slab[1] | mpls_exp1;
 620                         *slab1_ptr2 = entry2->slab[1] | mpls_exp2;
 621                         *slab1_ptr3 = entry3->slab[1] | mpls_exp3;
 622
 623                         *slab2_ptr0 = entry0->slab[2];
 624                         *slab2_ptr1 = entry1->slab[2];
 625                         *slab2_ptr2 = entry2->slab[2];
 626                         *slab2_ptr3 = entry3->slab[2];
 627                 } else {
 628                         *slab0_ptr0 = entry0->slab[0];
 629                         *slab0_ptr1 = entry1->slab[0];
 630                         *slab0_ptr2 = entry2->slab[0];
 631                         *slab0_ptr3 = entry3->slab[0];
 632
 633                         *slab1_ptr0 = entry0->slab[1];
 634                         *slab1_ptr1 = entry1->slab[1];
 635                         *slab1_ptr2 = entry2->slab[1];
 636                         *slab1_ptr3 = entry3->slab[1];
 637
 638                         *slab2_ptr0 = entry0->slab[2];
 639                         *slab2_ptr1 = entry1->slab[2];
 640                         *slab2_ptr2 = entry2->slab[2];
 641                         *slab2_ptr3 = entry3->slab[2];
 642                 }
 643
 644                 if (arp == 0) {
 645                         MACADDR_DST_WRITE(slab3_ptr0, entry0->slab[3]);
 646                         MACADDR_DST_WRITE(slab3_ptr1, entry1->slab[3]);
 647                         MACADDR_DST_WRITE(slab3_ptr2, entry2->slab[3]);
 648                         MACADDR_DST_WRITE(slab3_ptr3, entry3->slab[3]);
 649                 }
 650         }
 651
 652         if (arp) {
 653                 arp_key0->port_id = port_id0;
 654                 arp_key1->port_id = port_id1;
 655                 arp_key2->port_id = port_id2;
 656                 arp_key3->port_id = port_id3;
 657
 658                 arp_key0->ip = nh_ip0;
 659                 arp_key1->ip = nh_ip1;
 660                 arp_key2->ip = nh_ip2;
 661                 arp_key3->ip = nh_ip3;
 662         }
 663 }
 664
 665 #define PKT_WORK_ROUTING_ETHERNET(arp)                          \
 666 static inline void                                              \
 667 pkt_work_routing_ether_arp##arp(                                \
 668         struct rte_mbuf *pkt,                                   \
 669         struct rte_pipeline_table_entry *table_entry,           \
 670         void *arg)                                              \
 671 {                                                               \
 672         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 0, 0);\
 673 }
 674
 675 #define PKT4_WORK_ROUTING_ETHERNET(arp)                         \
 676 static inline void                                              \
 677 pkt4_work_routing_ether_arp##arp(                               \
 678         struct rte_mbuf **pkts,                                 \
 679         struct rte_pipeline_table_entry **table_entries,        \
 680         void *arg)                                              \
 681 {                                                               \
 682         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 0, 0);\
 683 }
 684
 685 #define routing_table_ah_hit_ether(arp)                         \
 686 PKT_WORK_ROUTING_ETHERNET(arp)                                  \
 687 PKT4_WORK_ROUTING_ETHERNET(arp)                                 \
 688 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_arp##arp,      \
 689         pkt_work_routing_ether_arp##arp,                        \
 690         pkt4_work_routing_ether_arp##arp)
 691
 692 routing_table_ah_hit_ether(0)
 693 routing_table_ah_hit_ether(1)
 694
 695 #define PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)              \
 696 static inline void                                              \
 697 pkt_work_routing_ether_qinq_sched##sched##_arp##arp(            \
 698         struct rte_mbuf *pkt,                                   \
 699         struct rte_pipeline_table_entry *table_entry,           \
 700         void *arg)                                              \
 701 {                                                               \
 702         pkt_work_routing(pkt, table_entry, arg, arp, 1, sched, 0, 0);\
 703 }
 704
 705 #define PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)             \
 706 static inline void                                              \
 707 pkt4_work_routing_ether_qinq_sched##sched##_arp##arp(           \
 708         struct rte_mbuf **pkts,                                 \
 709         struct rte_pipeline_table_entry **table_entries,        \
 710         void *arg)                                              \
 711 {                                                               \
 712         pkt4_work_routing(pkts, table_entries, arg, arp, 1, sched, 0, 0);\
 713 }
 714
 715 #define routing_table_ah_hit_ether_qinq(sched, arp)             \
 716 PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                      \
 717 PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                     \
 718 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_qinq_sched##sched##_arp##arp,\
 719         pkt_work_routing_ether_qinq_sched##sched##_arp##arp,    \
 720         pkt4_work_routing_ether_qinq_sched##sched##_arp##arp)
 721
 722 routing_table_ah_hit_ether_qinq(0, 0)
 723 routing_table_ah_hit_ether_qinq(1, 0)
 724 routing_table_ah_hit_ether_qinq(2, 0)
 725 routing_table_ah_hit_ether_qinq(0, 1)
 726 routing_table_ah_hit_ether_qinq(1, 1)
 727 routing_table_ah_hit_ether_qinq(2, 1)
 728
 729 #define PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)              \
 730 static inline void                                              \
 731 pkt_work_routing_ether_mpls_color##color##_arp##arp(            \
 732         struct rte_mbuf *pkt,                                   \
 733         struct rte_pipeline_table_entry *table_entry,           \
 734         void *arg)                                              \
 735 {                                                               \
 736         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 1, color);\
 737 }
 738
 739 #define PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)             \
 740 static inline void                                              \
 741 pkt4_work_routing_ether_mpls_color##color##_arp##arp(           \
 742         struct rte_mbuf **pkts,                                 \
 743         struct rte_pipeline_table_entry **table_entries,        \
 744         void *arg)                                              \
 745 {                                                               \
 746         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 1, color);\
 747 }
 748
 749 #define routing_table_ah_hit_ether_mpls(color, arp)             \
 750 PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)                      \
 751 PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)                     \
 752 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_mpls_color##color##_arp##arp,\
 753         pkt_work_routing_ether_mpls_color##color##_arp##arp,    \
 754         pkt4_work_routing_ether_mpls_color##color##_arp##arp)
 755
 756 routing_table_ah_hit_ether_mpls(0, 0)
 757 routing_table_ah_hit_ether_mpls(1, 0)
 758 routing_table_ah_hit_ether_mpls(0, 1)
 759 routing_table_ah_hit_ether_mpls(1, 1)
 760
 761 static rte_pipeline_table_action_handler_hit
 762 get_routing_table_ah_hit(struct pipeline_routing *p)
 763 {
 764         if (p->params.dbg_ah_disable)
 765                 return NULL;
 766
 767         switch (p->params.encap) {
 768         case PIPELINE_ROUTING_ENCAP_ETHERNET:
 769                 return (p->params.n_arp_entries) ?
 770                         routing_table_ah_hit_ether_arp1 :
 771                         routing_table_ah_hit_ether_arp0;
 772
 773         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
 774                 if (p->params.n_arp_entries)
 775                         switch (p->params.qinq_sched) {
 776                         case 0:
 777                                 return routing_table_ah_hit_ether_qinq_sched0_arp1;
 778                         case 1:
 779                                 return routing_table_ah_hit_ether_qinq_sched1_arp1;
 780                         case 2:
 781                                 return routing_table_ah_hit_ether_qinq_sched2_arp1;
 782                         default:
 783                                 return NULL;
 784                         }
 785                  else
 786                         switch (p->params.qinq_sched) {
 787                         case 0:
 788                                 return routing_table_ah_hit_ether_qinq_sched0_arp0;
 789                         case 1:
 790                                 return routing_table_ah_hit_ether_qinq_sched1_arp0;
 791                         case 2:
 792                                 return routing_table_ah_hit_ether_qinq_sched2_arp0;
 793                         default:
 794                                 return NULL;
 795                         }
 796
 797         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
 798                 if (p->params.n_arp_entries)
 799                         if (p->params.mpls_color_mark)
 800                                 return routing_table_ah_hit_ether_mpls_color1_arp1;
 801                         else
 802                                 return routing_table_ah_hit_ether_mpls_color0_arp1;
 803                 else
 804                         if (p->params.mpls_color_mark)
 805                                 return routing_table_ah_hit_ether_mpls_color1_arp0;
 806                         else
 807                                 return routing_table_ah_hit_ether_mpls_color0_arp0;
 808
 809         default:
 810                 return NULL;
 811         }
 812 }
 813
 814 /*
 815  * ARP table
 816  */
 817 struct arp_table_entry {
 818         struct rte_pipeline_table_entry head;
 819         uint64_t macaddr;
 820 };
 821
 822 /**
 823  * ARP table AH
 824  */
 825 static inline void
 826 pkt_work_arp(
 827         struct rte_mbuf *pkt,
 828         struct rte_pipeline_table_entry *table_entry,
 829         __rte_unused void *arg)
 830 {
 831         struct arp_table_entry *entry = (struct arp_table_entry *) table_entry;
 832
 833         /* Read */
 834         uint64_t macaddr_dst = entry->macaddr;
 835         uint64_t *slab_ptr = (uint64_t *) ((char *) pkt->buf_addr +
 836                 (pkt->data_off - 2));
 837
 838         /* Compute */
 839
 840         /* Write */
 841         MACADDR_DST_WRITE(slab_ptr, macaddr_dst);
 842 }
 843
 844 static inline void
 845 pkt4_work_arp(
 846         struct rte_mbuf **pkts,
 847         struct rte_pipeline_table_entry **table_entries,
 848         __rte_unused void *arg)
 849 {
 850         struct arp_table_entry *entry0 =
 851                 (struct arp_table_entry *) table_entries[0];
 852         struct arp_table_entry *entry1 =
 853                 (struct arp_table_entry *) table_entries[1];
 854         struct arp_table_entry *entry2 =
 855                 (struct arp_table_entry *) table_entries[2];
 856         struct arp_table_entry *entry3 =
 857                 (struct arp_table_entry *) table_entries[3];
 858
 859         /* Read */
 860         uint64_t macaddr_dst0 = entry0->macaddr;
 861         uint64_t macaddr_dst1 = entry1->macaddr;
 862         uint64_t macaddr_dst2 = entry2->macaddr;
 863         uint64_t macaddr_dst3 = entry3->macaddr;
 864
 865         uint64_t *slab_ptr0 = (uint64_t *) ((char *) pkts[0]->buf_addr +
 866                 (pkts[0]->data_off - 2));
 867         uint64_t *slab_ptr1 = (uint64_t *) ((char *) pkts[1]->buf_addr +
 868                 (pkts[1]->data_off - 2));
 869         uint64_t *slab_ptr2 = (uint64_t *) ((char *) pkts[2]->buf_addr +
 870                 (pkts[2]->data_off - 2));
 871         uint64_t *slab_ptr3 = (uint64_t *) ((char *) pkts[3]->buf_addr +
 872                 (pkts[3]->data_off - 2));
 873
 874         /* Compute */
 875
 876         /* Write */
 877         MACADDR_DST_WRITE(slab_ptr0, macaddr_dst0);
 878         MACADDR_DST_WRITE(slab_ptr1, macaddr_dst1);
 879         MACADDR_DST_WRITE(slab_ptr2, macaddr_dst2);
 880         MACADDR_DST_WRITE(slab_ptr3, macaddr_dst3);
 881 }
 882
 883 PIPELINE_TABLE_AH_HIT(arp_table_ah_hit,
 884         pkt_work_arp,
 885         pkt4_work_arp);
 886
 887 static rte_pipeline_table_action_handler_hit
 888 get_arp_table_ah_hit(struct pipeline_routing *p)
 889 {
 890         if (p->params.dbg_ah_disable)
 891                 return NULL;
 892
 893         return arp_table_ah_hit;
 894 }
 895
 896 /*
 897  * Argument parsing
 898  */
 899 int
 900 pipeline_routing_parse_args(struct pipeline_routing_params *p,
 901         struct pipeline_params *params)
 902 {
 903         uint32_t n_routes_present = 0;
 904         uint32_t port_local_dest_present = 0;
 905         uint32_t encap_present = 0;
 906         uint32_t qinq_sched_present = 0;
 907         uint32_t mpls_color_mark_present = 0;
 908         uint32_t n_arp_entries_present = 0;
 909         uint32_t ip_hdr_offset_present = 0;
 910         uint32_t arp_key_offset_present = 0;
 911         uint32_t color_offset_present = 0;
 912         uint32_t dbg_ah_disable_present = 0;
 913         uint32_t i;
 914
 915         /* default values */
 916         p->n_routes = PIPELINE_ROUTING_N_ROUTES_DEFAULT;
 917         p->port_local_dest = params->n_ports_out - 1;
 918         p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
 919         p->qinq_sched = 0;
 920         p->mpls_color_mark = 0;
 921         p->n_arp_entries = 0;
 922         p->dbg_ah_disable = 0;
 923
 924         for (i = 0; i < params->n_args; i++) {
 925                 char *arg_name = params->args_name[i];
 926                 char *arg_value = params->args_value[i];
 927
 928                 /* n_routes */
 929                 if (strcmp(arg_name, "n_routes") == 0) {
 930                         int status;
 931
 932                         PIPELINE_PARSE_ERR_DUPLICATE(
 933                                 n_routes_present == 0, params->name,
 934                                 arg_name);
 935                         n_routes_present = 1;
 936
 937                         status = parser_read_uint32(&p->n_routes,
 938                                 arg_value);
 939                         PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
 940                                 (p->n_routes != 0)), params->name,
 941                                 arg_name, arg_value);
 942                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 943                                 params->name, arg_name, arg_value);
 944
 945                         continue;
 946                 }
 947                 /* port_local_dest */
 948                 if (strcmp(arg_name, "port_local_dest") == 0) {
 949                         int status;
 950
 951                         PIPELINE_PARSE_ERR_DUPLICATE(
 952                                 port_local_dest_present == 0, params->name,
 953                                 arg_name);
 954                         port_local_dest_present = 1;
 955
 956                         status = parser_read_uint32(&p->port_local_dest,
 957                                 arg_value);
 958                         PIPELINE_PARSE_ERR_INV_VAL(((status == 0) &&
 959                                 (p->port_local_dest < params->n_ports_out)),
 960                                 params->name, arg_name, arg_value);
 961
 962                         continue;
 963                 }
 964
 965                 /* encap */
 966                 if (strcmp(arg_name, "encap") == 0) {
 967                         PIPELINE_PARSE_ERR_DUPLICATE(encap_present == 0,
 968                                 params->name, arg_name);
 969                         encap_present = 1;
 970
 971                         /* ethernet */
 972                         if (strcmp(arg_value, "ethernet") == 0) {
 973                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
 974                                 continue;
 975                         }
 976
 977                         /* ethernet_qinq */
 978                         if (strcmp(arg_value, "ethernet_qinq") == 0) {
 979                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ;
 980                                 continue;
 981                         }
 982
 983                         /* ethernet_mpls */
 984                         if (strcmp(arg_value, "ethernet_mpls") == 0) {
 985                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS;
 986                                 continue;
 987                         }
 988
 989                         /* any other */
 990                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
 991                                 arg_name, arg_value);
 992                 }
 993
 994                 /* qinq_sched */
 995                 if (strcmp(arg_name, "qinq_sched") == 0) {
 996                         int status;
 997
 998                         PIPELINE_PARSE_ERR_DUPLICATE(
 999                                 qinq_sched_present == 0, params->name,
1000                                 arg_name);
1001                         qinq_sched_present = 1;
1002
1003                         status = parser_read_arg_bool(arg_value);
1004                         if (status == -EINVAL) {
1005                                 if (strcmp(arg_value, "test") == 0) {
1006                                         p->qinq_sched = 2;
1007                                         continue;
1008                                 }
1009                         } else {
1010                                 p->qinq_sched = status;
1011                                 continue;
1012                         }
1013
1014                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1015                                 arg_name, arg_value);
1016                 }
1017
1018                 /* mpls_color_mark */
1019                 if (strcmp(arg_name, "mpls_color_mark") == 0) {
1020                         int status;
1021
1022                         PIPELINE_PARSE_ERR_DUPLICATE(
1023                                 mpls_color_mark_present == 0,
1024                                 params->name, arg_name);
1025                         mpls_color_mark_present = 1;
1026
1027
1028                         status = parser_read_arg_bool(arg_value);
1029                         if (status >= 0) {
1030                                 p->mpls_color_mark = status;
1031                                 continue;
1032                         }
1033
1034                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1035                                 arg_name, arg_value);
1036                 }
1037
1038                 /* n_arp_entries */
1039                 if (strcmp(arg_name, "n_arp_entries") == 0) {
1040                         int status;
1041
1042                         PIPELINE_PARSE_ERR_DUPLICATE(
1043                                 n_arp_entries_present == 0, params->name,
1044                                 arg_name);
1045                         n_arp_entries_present = 1;
1046
1047                         status = parser_read_uint32(&p->n_arp_entries,
1048                                 arg_value);
1049                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1050                                 params->name, arg_name, arg_value);
1051                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1052                                 params->name, arg_name, arg_value);
1053
1054                         continue;
1055                 }
1056
1057                 /* ip_hdr_offset */
1058                 if (strcmp(arg_name, "ip_hdr_offset") == 0) {
1059                         int status;
1060
1061                         PIPELINE_PARSE_ERR_DUPLICATE(
1062                                 ip_hdr_offset_present == 0, params->name,
1063                                 arg_name);
1064                         ip_hdr_offset_present = 1;
1065
1066                         status = parser_read_uint32(&p->ip_hdr_offset,
1067                                 arg_value);
1068                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1069                                 params->name, arg_name, arg_value);
1070                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1071                                 params->name, arg_name, arg_value);
1072
1073                         continue;
1074                 }
1075
1076                 /* arp_key_offset */
1077                 if (strcmp(arg_name, "arp_key_offset") == 0) {
1078                         int status;
1079
1080                         PIPELINE_PARSE_ERR_DUPLICATE(
1081                                 arp_key_offset_present == 0, params->name,
1082                                 arg_name);
1083                         arp_key_offset_present = 1;
1084
1085                         status = parser_read_uint32(&p->arp_key_offset,
1086                                 arg_value);
1087                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1088                                 params->name, arg_name, arg_value);
1089                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1090                                 params->name, arg_name, arg_value);
1091
1092                         continue;
1093                 }
1094
1095                 /* color_offset */
1096                 if (strcmp(arg_name, "color_offset") == 0) {
1097                         int status;
1098
1099                         PIPELINE_PARSE_ERR_DUPLICATE(
1100                                 color_offset_present == 0, params->name,
1101                                 arg_name);
1102                         color_offset_present = 1;
1103
1104                         status = parser_read_uint32(&p->color_offset,
1105                                 arg_value);
1106                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1107                                 params->name, arg_name, arg_value);
1108                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1109                                 params->name, arg_name, arg_value);
1110
1111                         continue;
1112                 }
1113
1114                 /* debug */
1115                 if (strcmp(arg_name, "dbg_ah_disable") == 0) {
1116                         int status;
1117
1118                         PIPELINE_PARSE_ERR_DUPLICATE(
1119                                 dbg_ah_disable_present == 0, params->name,
1120                                 arg_name);
1121                         dbg_ah_disable_present = 1;
1122
1123                         status = parser_read_arg_bool(arg_value);
1124                         if (status >= 0) {
1125                                 p->dbg_ah_disable = status;
1126                                 continue;
1127                         }
1128
1129                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1130                                 arg_name, arg_value);
1131
1132                         continue;
1133                 }
1134
1135                 /* any other */
1136                 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
1137         }
1138
1139         /* Check that mandatory arguments are present */
1140         PIPELINE_PARSE_ERR_MANDATORY(ip_hdr_offset_present, params->name,
1141                 "ip_hdr_offset");
1142
1143         /* Check relations between arguments */
1144         switch (p->encap) {
1145         case PIPELINE_ROUTING_ENCAP_ETHERNET:
1146                 PIPELINE_ARG_CHECK((!p->qinq_sched), "Parse error in "
1147                         "section \"%s\": encap = ethernet, therefore "
1148                         "qinq_sched = yes/test is not allowed",
1149                         params->name);
1150                 PIPELINE_ARG_CHECK((!p->mpls_color_mark), "Parse error "
1151                         "in section \"%s\": encap = ethernet, therefore "
1152                         "mpls_color_mark = yes is not allowed",
1153                         params->name);
1154                 PIPELINE_ARG_CHECK((!color_offset_present), "Parse error "
1155                         "in section \"%s\": encap = ethernet, therefore "
1156                         "color_offset is not allowed",
1157                         params->name);
1158                 break;
1159
1160         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
1161                 PIPELINE_ARG_CHECK((!p->mpls_color_mark), "Parse error "
1162                         "in section \"%s\": encap = ethernet_qinq, "
1163                         "therefore mpls_color_mark = yes is not allowed",
1164                         params->name);
1165                 PIPELINE_ARG_CHECK((!color_offset_present), "Parse error "
1166                         "in section \"%s\": encap = ethernet_qinq, "
1167                         "therefore color_offset is not allowed",
1168                         params->name);
1169                 break;
1170
1171         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
1172                 PIPELINE_ARG_CHECK((!p->qinq_sched), "Parse error in "
1173                         "section \"%s\": encap = ethernet_mpls, therefore "
1174                         "qinq_sched  = yes/test is not allowed",
1175                         params->name);
1176                 break;
1177         }
1178
1179         PIPELINE_ARG_CHECK((!(p->n_arp_entries &&
1180                 (!arp_key_offset_present))), "Parse error in section "
1181                         "\"%s\": n_arp_entries is set while "
1182                         "arp_key_offset is not set", params->name);
1183
1184         PIPELINE_ARG_CHECK((!((p->n_arp_entries == 0) &&
1185                 arp_key_offset_present)), "Parse error in section "
1186                         "\"%s\": arp_key_offset present while "
1187                         "n_arp_entries is not set", params->name);
1188
1189         return 0;
1190 }
1191
1192 static void *
1193 pipeline_routing_init(struct pipeline_params *params,
1194         __rte_unused void *arg)
1195 {
1196         struct pipeline *p;
1197         struct pipeline_routing *p_rt;
1198         uint32_t size, i;
1199
1200         /* Check input arguments */
1201         if ((params == NULL) ||
1202                 (params->n_ports_in == 0) ||
1203                 (params->n_ports_out == 0))
1204                 return NULL;
1205
1206         /* Memory allocation */
1207         size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_routing));
1208         p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
1209         p_rt = (struct pipeline_routing *) p;
1210         if (p == NULL)
1211                 return NULL;
1212
1213         strcpy(p->name, params->name);
1214         p->log_level = params->log_level;
1215
1216         PLOG(p, HIGH, "Routing");
1217
1218         /* Parse arguments */
1219         if (pipeline_routing_parse_args(&p_rt->params, params))
1220                 return NULL;
1221
1222         /* Pipeline */
1223         {
1224                 struct rte_pipeline_params pipeline_params = {
1225                         .name = params->name,
1226                         .socket_id = params->socket_id,
1227                         .offset_port_id = 0,
1228                 };
1229
1230                 p->p = rte_pipeline_create(&pipeline_params);
1231                 if (p->p == NULL) {
1232                         rte_free(p);
1233                         return NULL;
1234                 }
1235         }
1236
1237         /* Input ports */
1238         p->n_ports_in = params->n_ports_in;
1239         for (i = 0; i < p->n_ports_in; i++) {
1240                 struct rte_pipeline_port_in_params port_params = {
1241                         .ops = pipeline_port_in_params_get_ops(
1242                                 &params->port_in[i]),
1243                         .arg_create = pipeline_port_in_params_convert(
1244                                 &params->port_in[i]),
1245                         .f_action = NULL,
1246                         .arg_ah = NULL,
1247                         .burst_size = params->port_in[i].burst_size,
1248                 };
1249
1250                 int status = rte_pipeline_port_in_create(p->p,
1251                         &port_params,
1252                         &p->port_in_id[i]);
1253
1254                 if (status) {
1255                         rte_pipeline_free(p->p);
1256                         rte_free(p);
1257                         return NULL;
1258                 }
1259         }
1260
1261         /* Output ports */
1262         p->n_ports_out = params->n_ports_out;
1263         for (i = 0; i < p->n_ports_out; i++) {
1264                 struct rte_pipeline_port_out_params port_params = {
1265                         .ops = pipeline_port_out_params_get_ops(
1266                                 &params->port_out[i]),
1267                         .arg_create = pipeline_port_out_params_convert(
1268                                 &params->port_out[i]),
1269                         .f_action = NULL,
1270                         .arg_ah = NULL,
1271                 };
1272
1273                 int status = rte_pipeline_port_out_create(p->p,
1274                         &port_params,
1275                         &p->port_out_id[i]);
1276
1277                 if (status) {
1278                         rte_pipeline_free(p->p);
1279                         rte_free(p);
1280                         return NULL;
1281                 }
1282         }
1283
1284         /* Routing table */
1285         p->n_tables = 1;
1286         {
1287                 struct rte_table_lpm_params table_lpm_params = {
1288                         .name = p->name,
1289                         .n_rules = p_rt->params.n_routes,
1290                         .number_tbl8s = PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s,
1291                         .flags = 0,
1292                         .entry_unique_size = sizeof(struct routing_table_entry),
1293                         .offset = p_rt->params.ip_hdr_offset +
1294                                 __builtin_offsetof(struct ipv4_hdr, dst_addr),
1295                 };
1296
1297                 struct rte_pipeline_table_params table_params = {
1298                                 .ops = &rte_table_lpm_ops,
1299                                 .arg_create = &table_lpm_params,
1300                                 .f_action_hit = get_routing_table_ah_hit(p_rt),
1301                                 .f_action_miss = NULL,
1302                                 .arg_ah = p_rt,
1303                                 .action_data_size =
1304                                         sizeof(struct routing_table_entry) -
1305                                         sizeof(struct rte_pipeline_table_entry),
1306                         };
1307
1308                 int status;
1309
1310                 status = rte_pipeline_table_create(p->p,
1311                         &table_params,
1312                         &p->table_id[0]);
1313
1314                 if (status) {
1315                         rte_pipeline_free(p->p);
1316                         rte_free(p);
1317                         return NULL;
1318                 }
1319         }
1320
1321         /* ARP table configuration */
1322         if (p_rt->params.n_arp_entries) {
1323                 struct rte_table_hash_params table_arp_params = {
1324                         .name = p->name,
1325                         .key_size = 8,
1326                         .key_offset = p_rt->params.arp_key_offset,
1327                         .key_mask = NULL,
1328                         .n_keys = p_rt->params.n_arp_entries,
1329                         .n_buckets =
1330                                 rte_align32pow2(p_rt->params.n_arp_entries / 4),
1331                         .f_hash = hash_default_key8,
1332                         .seed = 0,
1333                 };
1334
1335                 struct rte_pipeline_table_params table_params = {
1336                         .ops = &rte_table_hash_key8_ext_ops,
1337                         .arg_create = &table_arp_params,
1338                         .f_action_hit = get_arp_table_ah_hit(p_rt),
1339                         .f_action_miss = NULL,
1340                         .arg_ah = p_rt,
1341                         .action_data_size = sizeof(struct arp_table_entry) -
1342                                 sizeof(struct rte_pipeline_table_entry),
1343                 };
1344
1345                 int status;
1346
1347                 status = rte_pipeline_table_create(p->p,
1348                         &table_params,
1349                         &p->table_id[1]);
1350
1351                 if (status) {
1352                         rte_pipeline_free(p->p);
1353                         rte_free(p);
1354                         return NULL;
1355                 }
1356
1357                 p->n_tables++;
1358         }
1359
1360         /* Connecting input ports to tables */
1361         for (i = 0; i < p->n_ports_in; i++) {
1362                 int status = rte_pipeline_port_in_connect_to_table(p->p,
1363                         p->port_in_id[i],
1364                         p->table_id[0]);
1365
1366                 if (status) {
1367                         rte_pipeline_free(p->p);
1368                         rte_free(p);
1369                         return NULL;
1370                 }
1371         }
1372
1373         /* Enable input ports */
1374         for (i = 0; i < p->n_ports_in; i++) {
1375                 int status = rte_pipeline_port_in_enable(p->p,
1376                         p->port_in_id[i]);
1377
1378                 if (status) {
1379                         rte_pipeline_free(p->p);
1380                         rte_free(p);
1381                         return NULL;
1382                 }
1383         }
1384
1385         /* Check pipeline consistency */
1386         if (rte_pipeline_check(p->p) < 0) {
1387                 rte_pipeline_free(p->p);
1388                 rte_free(p);
1389                 return NULL;
1390         }
1391
1392         /* Message queues */
1393         p->n_msgq = params->n_msgq;
1394         for (i = 0; i < p->n_msgq; i++)
1395                 p->msgq_in[i] = params->msgq_in[i];
1396         for (i = 0; i < p->n_msgq; i++)
1397                 p->msgq_out[i] = params->msgq_out[i];
1398
1399         /* Message handlers */
1400         memcpy(p->handlers, handlers, sizeof(p->handlers));
1401         memcpy(p_rt->custom_handlers,
1402                 custom_handlers,
1403                 sizeof(p_rt->custom_handlers));
1404
1405         return p;
1406 }
1407
1408 static int
1409 pipeline_routing_free(void *pipeline)
1410 {
1411         struct pipeline *p = (struct pipeline *) pipeline;
1412
1413         /* Check input arguments */
1414         if (p == NULL)
1415                 return -1;
1416
1417         /* Free resources */
1418         rte_pipeline_free(p->p);
1419         rte_free(p);
1420         return 0;
1421 }
1422
1423 static int
1424 pipeline_routing_timer(void *pipeline)
1425 {
1426         struct pipeline *p = (struct pipeline *) pipeline;
1427
1428         pipeline_msg_req_handle(p);
1429         rte_pipeline_flush(p->p);
1430
1431         return 0;
1432 }
1433
1434 void *
1435 pipeline_routing_msg_req_custom_handler(struct pipeline *p,
1436         void *msg)
1437 {
1438         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1439         struct pipeline_custom_msg_req *req = msg;
1440         pipeline_msg_req_handler f_handle;
1441
1442         f_handle = (req->subtype < PIPELINE_ROUTING_MSG_REQS) ?
1443                 p_rt->custom_handlers[req->subtype] :
1444                 pipeline_msg_req_invalid_handler;
1445
1446         if (f_handle == NULL)
1447                 f_handle = pipeline_msg_req_invalid_handler;
1448
1449         return f_handle(p, req);
1450 }
1451
1452 void *
1453 pipeline_routing_msg_req_route_add_handler(struct pipeline *p, void *msg)
1454 {
1455         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1456         struct pipeline_routing_route_add_msg_req *req = msg;
1457         struct pipeline_routing_route_add_msg_rsp *rsp = msg;
1458
1459         struct rte_table_lpm_key key = {
1460                 .ip = req->key.key.ipv4.ip,
1461                 .depth = req->key.key.ipv4.depth,
1462         };
1463
1464         struct routing_table_entry entry_arp0 = {
1465                 .head = {
1466                         .action = RTE_PIPELINE_ACTION_PORT,
1467                         {.port_id = p->port_out_id[req->data.port_id]},
1468                 },
1469
1470                 .flags = req->data.flags,
1471                 .port_id = req->data.port_id,
1472                 .ip = 0,
1473                 .data_offset = 0,
1474                 .ether_l2_length = 0,
1475                 .slab = {0},
1476                 .slab_offset = {0},
1477         };
1478
1479         struct routing_table_entry entry_arp1 = {
1480                 .head = {
1481                         .action = RTE_PIPELINE_ACTION_TABLE,
1482                         {.table_id = p->table_id[1]},
1483                 },
1484
1485                 .flags = req->data.flags,
1486                 .port_id = req->data.port_id,
1487                 .ip = rte_bswap32(req->data.ethernet.ip),
1488                 .data_offset = 0,
1489                 .ether_l2_length = 0,
1490                 .slab = {0},
1491                 .slab_offset = {0},
1492         };
1493
1494         struct rte_pipeline_table_entry *entry = (p_rt->params.n_arp_entries) ?
1495                 (struct rte_pipeline_table_entry *) &entry_arp1 :
1496                 (struct rte_pipeline_table_entry *) &entry_arp0;
1497
1498         if ((req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) ||
1499                 ((p_rt->params.n_arp_entries == 0) &&
1500                         (req->data.flags & PIPELINE_ROUTING_ROUTE_ARP)) ||
1501                 (p_rt->params.n_arp_entries &&
1502                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_ARP) == 0)) ||
1503                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1504                         (req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ)) ||
1505                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1506                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ) == 0)) ||
1507                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1508                         (req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS)) ||
1509                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1510                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS) == 0))) {
1511                 rsp->status = -1;
1512                 return rsp;
1513         }
1514
1515         /* Ether - ARP off */
1516         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1517                 (p_rt->params.n_arp_entries == 0)) {
1518                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1519                 uint64_t macaddr_dst;
1520                 uint64_t ethertype = ETHER_TYPE_IPv4;
1521
1522                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1523                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1524
1525                 entry_arp0.slab[0] =
1526                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype);
1527                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1528
1529                 entry_arp0.slab[1] = rte_bswap64(macaddr_dst);
1530                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1531
1532                 entry_arp0.data_offset = entry_arp0.slab_offset[1] + 2
1533                         - sizeof(struct rte_mbuf);
1534                 entry_arp0.ether_l2_length = 14;
1535         }
1536
1537         /* Ether - ARP on */
1538         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1539                 p_rt->params.n_arp_entries) {
1540                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1541                 uint64_t ethertype = ETHER_TYPE_IPv4;
1542
1543                 entry_arp1.slab[0] =
1544                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype);
1545                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1546
1547                 entry_arp1.data_offset = entry_arp1.slab_offset[0] - 6
1548                         - sizeof(struct rte_mbuf);
1549                 entry_arp1.ether_l2_length = 14;
1550         }
1551
1552         /* Ether QinQ - ARP off */
1553         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1554                 (p_rt->params.n_arp_entries == 0)) {
1555                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1556                 uint64_t macaddr_dst;
1557                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1558                 uint64_t ethertype_vlan = 0x8100;
1559                 uint64_t ethertype_qinq = 0x9100;
1560                 uint64_t svlan = req->data.l2.qinq.svlan;
1561                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1562
1563                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1564                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1565
1566                 entry_arp0.slab[0] = rte_bswap64((svlan << 48) |
1567                         (ethertype_vlan << 32) |
1568                         (cvlan << 16) |
1569                         ethertype_ipv4);
1570                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1571
1572                 entry_arp0.slab[1] =
1573                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_qinq);
1574                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1575
1576                 entry_arp0.slab[2] = rte_bswap64(macaddr_dst);
1577                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset - 3 * 8;
1578
1579                 entry_arp0.data_offset = entry_arp0.slab_offset[2] + 2
1580                         - sizeof(struct rte_mbuf);
1581                 entry_arp0.ether_l2_length = 22;
1582         }
1583
1584         /* Ether QinQ - ARP on */
1585         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1586                 p_rt->params.n_arp_entries) {
1587                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1588                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1589                 uint64_t ethertype_vlan = 0x8100;
1590                 uint64_t ethertype_qinq = 0x9100;
1591                 uint64_t svlan = req->data.l2.qinq.svlan;
1592                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1593
1594                 entry_arp1.slab[0] = rte_bswap64((svlan << 48) |
1595                         (ethertype_vlan << 32) |
1596                         (cvlan << 16) |
1597                         ethertype_ipv4);
1598                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1599
1600                 entry_arp1.slab[1] =
1601                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_qinq);
1602                 entry_arp1.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1603
1604                 entry_arp1.data_offset = entry_arp1.slab_offset[1] - 6
1605                         - sizeof(struct rte_mbuf);
1606                 entry_arp1.ether_l2_length = 22;
1607         }
1608
1609         /* Ether MPLS - ARP off */
1610         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1611                 (p_rt->params.n_arp_entries == 0)) {
1612                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1613                 uint64_t macaddr_dst;
1614                 uint64_t ethertype_mpls = 0x8847;
1615
1616                 uint64_t label0 = req->data.l2.mpls.labels[0];
1617                 uint64_t label1 = req->data.l2.mpls.labels[1];
1618                 uint64_t label2 = req->data.l2.mpls.labels[2];
1619                 uint64_t label3 = req->data.l2.mpls.labels[3];
1620                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1621
1622                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1623                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1624
1625                 switch (n_labels) {
1626                 case 1:
1627                         entry_arp0.slab[0] = 0;
1628                         entry_arp0.slab_offset[0] =
1629                                 p_rt->params.ip_hdr_offset - 8;
1630
1631                         entry_arp0.slab[1] = rte_bswap64(
1632                                 MPLS_LABEL(label0, 0, 1, 0));
1633                         entry_arp0.slab_offset[1] =
1634                                 p_rt->params.ip_hdr_offset - 8;
1635                         break;
1636
1637                 case 2:
1638                         entry_arp0.slab[0] = 0;
1639                         entry_arp0.slab_offset[0] =
1640                                 p_rt->params.ip_hdr_offset - 8;
1641
1642                         entry_arp0.slab[1] = rte_bswap64(
1643                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1644                                 MPLS_LABEL(label1, 0, 1, 0));
1645                         entry_arp0.slab_offset[1] =
1646                                 p_rt->params.ip_hdr_offset - 8;
1647                         break;
1648
1649                 case 3:
1650                         entry_arp0.slab[0] = rte_bswap64(
1651                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1652                                 MPLS_LABEL(label2, 0, 1, 0));
1653                         entry_arp0.slab_offset[0] =
1654                                 p_rt->params.ip_hdr_offset - 8;
1655
1656                         entry_arp0.slab[1] = rte_bswap64(
1657                                 MPLS_LABEL(label0, 0, 0, 0));
1658                         entry_arp0.slab_offset[1] =
1659                                 p_rt->params.ip_hdr_offset - 2 * 8;
1660                         break;
1661
1662                 case 4:
1663                         entry_arp0.slab[0] = rte_bswap64(
1664                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1665                                 MPLS_LABEL(label3, 0, 1, 0));
1666                         entry_arp0.slab_offset[0] =
1667                                 p_rt->params.ip_hdr_offset - 8;
1668
1669                         entry_arp0.slab[1] = rte_bswap64(
1670                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1671                                 MPLS_LABEL(label1, 0, 0, 0));
1672                         entry_arp0.slab_offset[1] =
1673                                 p_rt->params.ip_hdr_offset - 2 * 8;
1674                         break;
1675
1676                 default:
1677                         rsp->status = -1;
1678                         return rsp;
1679                 }
1680
1681                 entry_arp0.slab[2] =
1682                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_mpls);
1683                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset -
1684                         (n_labels * 4 + 8);
1685
1686                 entry_arp0.slab[3] = rte_bswap64(macaddr_dst);
1687                 entry_arp0.slab_offset[3] = p_rt->params.ip_hdr_offset -
1688                         (n_labels * 4 + 2 * 8);
1689
1690                 entry_arp0.data_offset = entry_arp0.slab_offset[3] + 2
1691                         - sizeof(struct rte_mbuf);
1692                 entry_arp0.ether_l2_length = n_labels * 4 + 14;
1693         }
1694
1695         /* Ether MPLS - ARP on */
1696         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1697                 p_rt->params.n_arp_entries) {
1698                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1699                 uint64_t ethertype_mpls = 0x8847;
1700
1701                 uint64_t label0 = req->data.l2.mpls.labels[0];
1702                 uint64_t label1 = req->data.l2.mpls.labels[1];
1703                 uint64_t label2 = req->data.l2.mpls.labels[2];
1704                 uint64_t label3 = req->data.l2.mpls.labels[3];
1705                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1706
1707                 switch (n_labels) {
1708                 case 1:
1709                         entry_arp1.slab[0] = 0;
1710                         entry_arp1.slab_offset[0] =
1711                                 p_rt->params.ip_hdr_offset - 8;
1712
1713                         entry_arp1.slab[1] = rte_bswap64(
1714                                 MPLS_LABEL(label0, 0, 1, 0));
1715                         entry_arp1.slab_offset[1] =
1716                                 p_rt->params.ip_hdr_offset - 8;
1717                         break;
1718
1719                 case 2:
1720                         entry_arp1.slab[0] = 0;
1721                         entry_arp1.slab_offset[0] =
1722                                 p_rt->params.ip_hdr_offset - 8;
1723
1724                         entry_arp1.slab[1] = rte_bswap64(
1725                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1726                                 MPLS_LABEL(label1, 0, 1, 0));
1727                         entry_arp1.slab_offset[1] =
1728                                 p_rt->params.ip_hdr_offset - 8;
1729                         break;
1730
1731                 case 3:
1732                         entry_arp1.slab[0] = rte_bswap64(
1733                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1734                                 MPLS_LABEL(label2, 0, 1, 0));
1735                         entry_arp1.slab_offset[0] =
1736                                 p_rt->params.ip_hdr_offset - 8;
1737
1738                         entry_arp1.slab[1] = rte_bswap64(
1739                                 MPLS_LABEL(label0, 0, 0, 0));
1740                         entry_arp1.slab_offset[1] =
1741                                 p_rt->params.ip_hdr_offset - 2 * 8;
1742                         break;
1743
1744                 case 4:
1745                         entry_arp1.slab[0] = rte_bswap64(
1746                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1747                                 MPLS_LABEL(label3, 0, 1, 0));
1748                         entry_arp1.slab_offset[0] =
1749                                 p_rt->params.ip_hdr_offset - 8;
1750
1751                         entry_arp1.slab[1] = rte_bswap64(
1752                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1753                                 MPLS_LABEL(label1, 0, 0, 0));
1754                         entry_arp1.slab_offset[1] =
1755                                 p_rt->params.ip_hdr_offset - 2 * 8;
1756                         break;
1757
1758                 default:
1759                         rsp->status = -1;
1760                         return rsp;
1761                 }
1762
1763                 entry_arp1.slab[2] =
1764                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_mpls);
1765                 entry_arp1.slab_offset[2] = p_rt->params.ip_hdr_offset -
1766                         (n_labels * 4 + 8);
1767
1768                 entry_arp1.data_offset = entry_arp1.slab_offset[2] - 6
1769                         - sizeof(struct rte_mbuf);
1770                 entry_arp1.ether_l2_length = n_labels * 4 + 14;
1771         }
1772
1773         rsp->status = rte_pipeline_table_entry_add(p->p,
1774                 p->table_id[0],
1775                 &key,
1776                 entry,
1777                 &rsp->key_found,
1778                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1779
1780         return rsp;
1781 }
1782
1783 void *
1784 pipeline_routing_msg_req_route_del_handler(struct pipeline *p, void *msg)
1785 {
1786         struct pipeline_routing_route_delete_msg_req *req = msg;
1787         struct pipeline_routing_route_delete_msg_rsp *rsp = msg;
1788
1789         struct rte_table_lpm_key key = {
1790                 .ip = req->key.key.ipv4.ip,
1791                 .depth = req->key.key.ipv4.depth,
1792         };
1793
1794         if (req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) {
1795                 rsp->status = -1;
1796                 return rsp;
1797         }
1798
1799         rsp->status = rte_pipeline_table_entry_delete(p->p,
1800                 p->table_id[0],
1801                 &key,
1802                 &rsp->key_found,
1803                 NULL);
1804
1805         return rsp;
1806 }
1807
1808 void *
1809 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
1810         void *msg)
1811 {
1812         struct pipeline_routing_route_add_default_msg_req *req = msg;
1813         struct pipeline_routing_route_add_default_msg_rsp *rsp = msg;
1814
1815         struct routing_table_entry default_entry = {
1816                 .head = {
1817                         .action = RTE_PIPELINE_ACTION_PORT,
1818                         {.port_id = p->port_out_id[req->port_id]},
1819                 },
1820
1821                 .flags = 0,
1822                 .port_id = 0,
1823                 .ip = 0,
1824         };
1825
1826         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1827                 p->table_id[0],
1828                 (struct rte_pipeline_table_entry *) &default_entry,
1829                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1830
1831         return rsp;
1832 }
1833
1834 void *
1835 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
1836         void *msg)
1837 {
1838         struct pipeline_routing_route_delete_default_msg_rsp *rsp = msg;
1839
1840         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1841                 p->table_id[0],
1842                 NULL);
1843
1844         return rsp;
1845 }
1846
1847 void *
1848 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p, void *msg)
1849 {
1850         struct pipeline_routing_arp_add_msg_req *req = msg;
1851         struct pipeline_routing_arp_add_msg_rsp *rsp = msg;
1852
1853         struct pipeline_routing_arp_key_ipv4 key = {
1854                 .port_id = req->key.key.ipv4.port_id,
1855                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1856         };
1857
1858         struct arp_table_entry entry = {
1859                 .head = {
1860                         .action = RTE_PIPELINE_ACTION_PORT,
1861                         {.port_id = p->port_out_id[req->key.key.ipv4.port_id]},
1862                 },
1863
1864                 .macaddr = 0, /* set below */
1865         };
1866
1867         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1868                 rsp->status = -1;
1869                 return rsp;
1870         }
1871
1872         entry.macaddr = *((uint64_t *)&(req->macaddr));
1873         entry.macaddr = entry.macaddr << 16;
1874
1875         rsp->status = rte_pipeline_table_entry_add(p->p,
1876                 p->table_id[1],
1877                 &key,
1878                 (struct rte_pipeline_table_entry *) &entry,
1879                 &rsp->key_found,
1880                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1881
1882         return rsp;
1883 }
1884
1885 void *
1886 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p, void *msg)
1887 {
1888         struct pipeline_routing_arp_delete_msg_req *req = msg;
1889         struct pipeline_routing_arp_delete_msg_rsp *rsp = msg;
1890
1891         struct pipeline_routing_arp_key_ipv4 key = {
1892                 .port_id = req->key.key.ipv4.port_id,
1893                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1894         };
1895
1896         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1897                 rsp->status = -1;
1898                 return rsp;
1899         }
1900
1901         rsp->status = rte_pipeline_table_entry_delete(p->p,
1902                 p->table_id[1],
1903                 &key,
1904                 &rsp->key_found,
1905                 NULL);
1906
1907         return rsp;
1908 }
1909
1910 void *
1911 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p, void *msg)
1912 {
1913         struct pipeline_routing_arp_add_default_msg_req *req = msg;
1914         struct pipeline_routing_arp_add_default_msg_rsp *rsp = msg;
1915
1916         struct arp_table_entry default_entry = {
1917                 .head = {
1918                         .action = RTE_PIPELINE_ACTION_PORT,
1919                         {.port_id = p->port_out_id[req->port_id]},
1920                 },
1921
1922                 .macaddr = 0,
1923         };
1924
1925         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1926                 p->table_id[1],
1927                 (struct rte_pipeline_table_entry *) &default_entry,
1928                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1929
1930         return rsp;
1931 }
1932
1933 void *
1934 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p, void *msg)
1935 {
1936         struct pipeline_routing_arp_delete_default_msg_rsp *rsp = msg;
1937
1938         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1939                 p->table_id[1],
1940                 NULL);
1941
1942         return rsp;
1943 }
1944
1945 void *
1946 pipeline_routing_msg_req_set_macaddr_handler(struct pipeline *p, void *msg)
1947 {
1948         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1949         struct pipeline_routing_set_macaddr_msg_req *req = msg;
1950         struct pipeline_routing_set_macaddr_msg_rsp *rsp = msg;
1951         uint32_t port_id;
1952
1953         for (port_id = 0; port_id < p->n_ports_out; port_id++)
1954                 p_rt->macaddr[port_id] = req->macaddr[port_id];
1955
1956         rsp->status = 0;
1957
1958         return rsp;
1959 }
1960
1961 struct pipeline_be_ops pipeline_routing_be_ops = {
1962         .f_init = pipeline_routing_init,
1963         .f_free = pipeline_routing_free,
1964         .f_run = NULL,
1965         .f_timer = pipeline_routing_timer,
1966 };