examples/ip_pipeline/pipeline/pipeline_routing_be.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <unistd.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_malloc.h>
  42 #include <rte_ip.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_table_lpm.h>
  45 #include <rte_table_hash.h>
  46 #include <rte_pipeline.h>
  47
  48 #include "pipeline_routing_be.h"
  49 #include "pipeline_actions_common.h"
  50 #include "parser.h"
  51 #include "hash_func.h"
  52
  53 #define MPLS_LABEL(label, exp, s, ttl)                                  \
  54         (((((uint64_t) (label)) & 0xFFFFFLLU) << 12) |          \
  55         ((((uint64_t) (exp)) & 0x7LLU) << 9) |                          \
  56         ((((uint64_t) (s)) & 0x1LLU) << 8) |                            \
  57         (((uint64_t) (ttl)) & 0xFFLU))
  58
  59 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe,         \
  60         traffic_class, queue, color)                            \
  61         ((((uint64_t) (queue)) & 0x3) |                \
  62         ((((uint64_t) (traffic_class)) & 0x3) << 2) |  \
  63         ((((uint64_t) (color)) & 0x3) << 4) |          \
  64         ((((uint64_t) (subport)) & 0xFFFF) << 16) |    \
  65         ((((uint64_t) (pipe)) & 0xFFFFFFFF) << 32))
  66
  67
  68 #define MAC_SRC_DEFAULT 0x112233445566ULL
  69
  70 #ifndef PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s
  71 #define PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s 256
  72 #endif
  73
  74 struct pipeline_routing {
  75         struct pipeline p;
  76         struct pipeline_routing_params params;
  77         pipeline_msg_req_handler custom_handlers[PIPELINE_ROUTING_MSG_REQS];
  78 } __rte_cache_aligned;
  79
  80 /*
  81  * Message handlers
  82  */
  83 static void *
  84 pipeline_routing_msg_req_custom_handler(struct pipeline *p, void *msg);
  85
  86 static pipeline_msg_req_handler handlers[] = {
  87         [PIPELINE_MSG_REQ_PING] =
  88                 pipeline_msg_req_ping_handler,
  89         [PIPELINE_MSG_REQ_STATS_PORT_IN] =
  90                 pipeline_msg_req_stats_port_in_handler,
  91         [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
  92                 pipeline_msg_req_stats_port_out_handler,
  93         [PIPELINE_MSG_REQ_STATS_TABLE] =
  94                 pipeline_msg_req_stats_table_handler,
  95         [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
  96                 pipeline_msg_req_port_in_enable_handler,
  97         [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
  98                 pipeline_msg_req_port_in_disable_handler,
  99         [PIPELINE_MSG_REQ_CUSTOM] =
 100                 pipeline_routing_msg_req_custom_handler,
 101 };
 102
 103 static void *
 104 pipeline_routing_msg_req_route_add_handler(struct pipeline *p,
 105         void *msg);
 106
 107 static void *
 108 pipeline_routing_msg_req_route_del_handler(struct pipeline *p,
 109         void *msg);
 110
 111 static void *
 112 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
 113         void *msg);
 114
 115 static void *
 116 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
 117         void *msg);
 118
 119 static void *
 120 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p,
 121         void *msg);
 122
 123 static void *
 124 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p,
 125         void *msg);
 126
 127 static void *
 128 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p,
 129         void *msg);
 130
 131 static void *
 132 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p,
 133         void *msg);
 134
 135 static pipeline_msg_req_handler custom_handlers[] = {
 136         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD] =
 137                 pipeline_routing_msg_req_route_add_handler,
 138         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL] =
 139                 pipeline_routing_msg_req_route_del_handler,
 140         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD_DEFAULT] =
 141                 pipeline_routing_msg_req_route_add_default_handler,
 142         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL_DEFAULT] =
 143                 pipeline_routing_msg_req_route_del_default_handler,
 144         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD] =
 145                 pipeline_routing_msg_req_arp_add_handler,
 146         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL] =
 147                 pipeline_routing_msg_req_arp_del_handler,
 148         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD_DEFAULT] =
 149                 pipeline_routing_msg_req_arp_add_default_handler,
 150         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL_DEFAULT] =
 151                 pipeline_routing_msg_req_arp_del_default_handler,
 152 };
 153
 154 /*
 155  * Routing table
 156  */
 157 struct routing_table_entry {
 158         struct rte_pipeline_table_entry head;
 159         uint32_t flags;
 160         uint32_t port_id; /* Output port ID */
 161         uint32_t ip; /* Next hop IP address (only valid for remote routes) */
 162
 163         /* ether_l2 */
 164         uint16_t data_offset;
 165         uint16_t ether_l2_length;
 166         uint64_t slab[4];
 167         uint16_t slab_offset[4];
 168 };
 169
 170 struct layout {
 171         uint16_t a;
 172         uint32_t b;
 173         uint16_t c;
 174 } __attribute__((__packed__));
 175
 176 #define MACADDR_DST_WRITE(slab_ptr, slab)                       \
 177 {                                                               \
 178         struct layout *dst = (struct layout *) (slab_ptr);      \
 179         struct layout *src = (struct layout *) &(slab);         \
 180                                                                 \
 181         dst->b = src->b;                                        \
 182         dst->c = src->c;                                        \
 183 }
 184
 185 static inline __attribute__((always_inline)) void
 186 pkt_work_routing(
 187         struct rte_mbuf *pkt,
 188         struct rte_pipeline_table_entry *table_entry,
 189         void *arg,
 190         int arp,
 191         int qinq,
 192         int qinq_sched,
 193         int mpls,
 194         int mpls_color_mark)
 195 {
 196         struct pipeline_routing *p_rt = arg;
 197
 198         struct routing_table_entry *entry =
 199                 (struct routing_table_entry *) table_entry;
 200
 201         struct ipv4_hdr *ip = (struct ipv4_hdr *)
 202                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.ip_hdr_offset);
 203
 204         enum rte_meter_color pkt_color = (enum rte_meter_color)
 205                 RTE_MBUF_METADATA_UINT32(pkt, p_rt->params.color_offset);
 206
 207         struct pipeline_routing_arp_key_ipv4 *arp_key =
 208                 (struct pipeline_routing_arp_key_ipv4 *)
 209                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.arp_key_offset);
 210
 211         uint64_t *slab0_ptr, *slab1_ptr, *slab2_ptr, *slab3_ptr, sched;
 212         uint32_t ip_da, nh_ip, port_id;
 213         uint16_t total_length, data_offset, ether_l2_length;
 214
 215         /* Read */
 216         total_length = rte_bswap16(ip->total_length);
 217         ip_da = ip->dst_addr;
 218         data_offset = entry->data_offset;
 219         ether_l2_length = entry->ether_l2_length;
 220         slab0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[0]);
 221         slab1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[1]);
 222         slab2_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[2]);
 223         slab3_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[3]);
 224
 225         if (arp) {
 226                 port_id = entry->port_id;
 227                 nh_ip = entry->ip;
 228                 if (entry->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 229                         nh_ip = ip_da;
 230         }
 231
 232         /* Compute */
 233         total_length += ether_l2_length;
 234
 235         if (qinq && qinq_sched) {
 236                 uint32_t dscp = ip->type_of_service >> 2;
 237                 uint32_t svlan, cvlan, tc, tc_q;
 238
 239                 if (qinq_sched == 1) {
 240                         uint64_t slab_qinq = rte_bswap64(entry->slab[0]);
 241
 242                         svlan = (slab_qinq >> 48) & 0xFFF;
 243                         cvlan = (slab_qinq >> 16) & 0xFFF;
 244                         tc = (dscp >> 2) & 0x3;
 245                         tc_q = dscp & 0x3;
 246                 } else {
 247                         uint32_t ip_src = rte_bswap32(ip->src_addr);
 248
 249                         svlan = 0;
 250                         cvlan = (ip_src >> 16) & 0xFFF;
 251                         tc = (ip_src >> 2) & 0x3;
 252                         tc_q = ip_src & 0x3;
 253                 }
 254                 sched = RTE_SCHED_PORT_HIERARCHY(svlan,
 255                         cvlan,
 256                         tc,
 257                         tc_q,
 258                         e_RTE_METER_GREEN);
 259         }
 260
 261         /* Write */
 262         pkt->data_off = data_offset;
 263         pkt->data_len = total_length;
 264         pkt->pkt_len = total_length;
 265
 266         if ((qinq == 0) && (mpls == 0)) {
 267                 *slab0_ptr = entry->slab[0];
 268
 269                 if (arp == 0)
 270                         MACADDR_DST_WRITE(slab1_ptr, entry->slab[1]);
 271         }
 272
 273         if (qinq) {
 274                 *slab0_ptr = entry->slab[0];
 275                 *slab1_ptr = entry->slab[1];
 276
 277                 if (arp == 0)
 278                         MACADDR_DST_WRITE(slab2_ptr, entry->slab[2]);
 279
 280                 if (qinq_sched) {
 281                         pkt->hash.sched.lo = sched & 0xFFFFFFFF;
 282                         pkt->hash.sched.hi = sched >> 32;
 283                 }
 284         }
 285
 286         if (mpls) {
 287                 if (mpls_color_mark) {
 288                         uint64_t mpls_exp = rte_bswap64(
 289                                 (MPLS_LABEL(0, pkt_color, 0, 0) << 32) |
 290                                 MPLS_LABEL(0, pkt_color, 0, 0));
 291
 292                         *slab0_ptr = entry->slab[0] | mpls_exp;
 293                         *slab1_ptr = entry->slab[1] | mpls_exp;
 294                         *slab2_ptr = entry->slab[2];
 295                 } else {
 296                         *slab0_ptr = entry->slab[0];
 297                         *slab1_ptr = entry->slab[1];
 298                         *slab2_ptr = entry->slab[2];
 299                 }
 300
 301                 if (arp == 0)
 302                         MACADDR_DST_WRITE(slab3_ptr, entry->slab[3]);
 303         }
 304
 305         if (arp) {
 306                 arp_key->port_id = port_id;
 307                 arp_key->ip = nh_ip;
 308         }
 309 }
 310
 311 static inline __attribute__((always_inline)) void
 312 pkt4_work_routing(
 313         struct rte_mbuf **pkts,
 314         struct rte_pipeline_table_entry **table_entries,
 315         void *arg,
 316         int arp,
 317         int qinq,
 318         int qinq_sched,
 319         int mpls,
 320         int mpls_color_mark)
 321 {
 322         struct pipeline_routing *p_rt = arg;
 323
 324         struct routing_table_entry *entry0 =
 325                 (struct routing_table_entry *) table_entries[0];
 326         struct routing_table_entry *entry1 =
 327                 (struct routing_table_entry *) table_entries[1];
 328         struct routing_table_entry *entry2 =
 329                 (struct routing_table_entry *) table_entries[2];
 330         struct routing_table_entry *entry3 =
 331                 (struct routing_table_entry *) table_entries[3];
 332
 333         struct ipv4_hdr *ip0 = (struct ipv4_hdr *)
 334                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 335                         p_rt->params.ip_hdr_offset);
 336         struct ipv4_hdr *ip1 = (struct ipv4_hdr *)
 337                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 338                         p_rt->params.ip_hdr_offset);
 339         struct ipv4_hdr *ip2 = (struct ipv4_hdr *)
 340                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 341                         p_rt->params.ip_hdr_offset);
 342         struct ipv4_hdr *ip3 = (struct ipv4_hdr *)
 343                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 344                         p_rt->params.ip_hdr_offset);
 345
 346         enum rte_meter_color pkt0_color = (enum rte_meter_color)
 347                 RTE_MBUF_METADATA_UINT32(pkts[0], p_rt->params.color_offset);
 348         enum rte_meter_color pkt1_color = (enum rte_meter_color)
 349                 RTE_MBUF_METADATA_UINT32(pkts[1], p_rt->params.color_offset);
 350         enum rte_meter_color pkt2_color = (enum rte_meter_color)
 351                 RTE_MBUF_METADATA_UINT32(pkts[2], p_rt->params.color_offset);
 352         enum rte_meter_color pkt3_color = (enum rte_meter_color)
 353                 RTE_MBUF_METADATA_UINT32(pkts[3], p_rt->params.color_offset);
 354
 355         struct pipeline_routing_arp_key_ipv4 *arp_key0 =
 356                 (struct pipeline_routing_arp_key_ipv4 *)
 357                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 358                         p_rt->params.arp_key_offset);
 359         struct pipeline_routing_arp_key_ipv4 *arp_key1 =
 360                 (struct pipeline_routing_arp_key_ipv4 *)
 361                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 362                         p_rt->params.arp_key_offset);
 363         struct pipeline_routing_arp_key_ipv4 *arp_key2 =
 364                 (struct pipeline_routing_arp_key_ipv4 *)
 365                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 366                         p_rt->params.arp_key_offset);
 367         struct pipeline_routing_arp_key_ipv4 *arp_key3 =
 368                 (struct pipeline_routing_arp_key_ipv4 *)
 369                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 370                         p_rt->params.arp_key_offset);
 371
 372         uint64_t *slab0_ptr0, *slab1_ptr0, *slab2_ptr0, *slab3_ptr0;
 373         uint64_t *slab0_ptr1, *slab1_ptr1, *slab2_ptr1, *slab3_ptr1;
 374         uint64_t *slab0_ptr2, *slab1_ptr2, *slab2_ptr2, *slab3_ptr2;
 375         uint64_t *slab0_ptr3, *slab1_ptr3, *slab2_ptr3, *slab3_ptr3;
 376         uint64_t sched0, sched1, sched2, sched3;
 377
 378         uint32_t ip_da0, nh_ip0, port_id0;
 379         uint32_t ip_da1, nh_ip1, port_id1;
 380         uint32_t ip_da2, nh_ip2, port_id2;
 381         uint32_t ip_da3, nh_ip3, port_id3;
 382
 383         uint16_t total_length0, data_offset0, ether_l2_length0;
 384         uint16_t total_length1, data_offset1, ether_l2_length1;
 385         uint16_t total_length2, data_offset2, ether_l2_length2;
 386         uint16_t total_length3, data_offset3, ether_l2_length3;
 387
 388         /* Read */
 389         total_length0 = rte_bswap16(ip0->total_length);
 390         total_length1 = rte_bswap16(ip1->total_length);
 391         total_length2 = rte_bswap16(ip2->total_length);
 392         total_length3 = rte_bswap16(ip3->total_length);
 393
 394         ip_da0 = ip0->dst_addr;
 395         ip_da1 = ip1->dst_addr;
 396         ip_da2 = ip2->dst_addr;
 397         ip_da3 = ip3->dst_addr;
 398
 399         data_offset0 = entry0->data_offset;
 400         data_offset1 = entry1->data_offset;
 401         data_offset2 = entry2->data_offset;
 402         data_offset3 = entry3->data_offset;
 403
 404         ether_l2_length0 = entry0->ether_l2_length;
 405         ether_l2_length1 = entry1->ether_l2_length;
 406         ether_l2_length2 = entry2->ether_l2_length;
 407         ether_l2_length3 = entry3->ether_l2_length;
 408
 409         slab0_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 410                 entry0->slab_offset[0]);
 411         slab1_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 412                 entry0->slab_offset[1]);
 413         slab2_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 414                 entry0->slab_offset[2]);
 415         slab3_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 416                 entry0->slab_offset[3]);
 417
 418         slab0_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 419                 entry1->slab_offset[0]);
 420         slab1_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 421                 entry1->slab_offset[1]);
 422         slab2_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 423                 entry1->slab_offset[2]);
 424         slab3_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 425                 entry1->slab_offset[3]);
 426
 427         slab0_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 428                 entry2->slab_offset[0]);
 429         slab1_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 430                 entry2->slab_offset[1]);
 431         slab2_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 432                 entry2->slab_offset[2]);
 433         slab3_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 434                 entry2->slab_offset[3]);
 435
 436         slab0_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 437                 entry3->slab_offset[0]);
 438         slab1_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 439                 entry3->slab_offset[1]);
 440         slab2_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 441                 entry3->slab_offset[2]);
 442         slab3_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 443                 entry3->slab_offset[3]);
 444
 445         if (arp) {
 446                 port_id0 = entry0->port_id;
 447                 nh_ip0 = entry0->ip;
 448                 if (entry0->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 449                         nh_ip0 = ip_da0;
 450
 451                 port_id1 = entry1->port_id;
 452                 nh_ip1 = entry1->ip;
 453                 if (entry1->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 454                         nh_ip1 = ip_da1;
 455
 456                 port_id2 = entry2->port_id;
 457                 nh_ip2 = entry2->ip;
 458                 if (entry2->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 459                         nh_ip2 = ip_da2;
 460
 461                 port_id3 = entry3->port_id;
 462                 nh_ip3 = entry3->ip;
 463                 if (entry3->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 464                         nh_ip3 = ip_da3;
 465         }
 466
 467         /* Compute */
 468         total_length0 += ether_l2_length0;
 469         total_length1 += ether_l2_length1;
 470         total_length2 += ether_l2_length2;
 471         total_length3 += ether_l2_length3;
 472
 473         if (qinq && qinq_sched) {
 474                 uint32_t dscp0 = ip0->type_of_service >> 2;
 475                 uint32_t dscp1 = ip1->type_of_service >> 2;
 476                 uint32_t dscp2 = ip2->type_of_service >> 2;
 477                 uint32_t dscp3 = ip3->type_of_service >> 2;
 478                 uint32_t svlan0, cvlan0, tc0, tc_q0;
 479                 uint32_t svlan1, cvlan1, tc1, tc_q1;
 480                 uint32_t svlan2, cvlan2, tc2, tc_q2;
 481                 uint32_t svlan3, cvlan3, tc3, tc_q3;
 482
 483                 if (qinq_sched == 1) {
 484                         uint64_t slab_qinq0 = rte_bswap64(entry0->slab[0]);
 485                         uint64_t slab_qinq1 = rte_bswap64(entry1->slab[0]);
 486                         uint64_t slab_qinq2 = rte_bswap64(entry2->slab[0]);
 487                         uint64_t slab_qinq3 = rte_bswap64(entry3->slab[0]);
 488
 489                         svlan0 = (slab_qinq0 >> 48) & 0xFFF;
 490                         svlan1 = (slab_qinq1 >> 48) & 0xFFF;
 491                         svlan2 = (slab_qinq2 >> 48) & 0xFFF;
 492                         svlan3 = (slab_qinq3 >> 48) & 0xFFF;
 493
 494                         cvlan0 = (slab_qinq0 >> 16) & 0xFFF;
 495                         cvlan1 = (slab_qinq1 >> 16) & 0xFFF;
 496                         cvlan2 = (slab_qinq2 >> 16) & 0xFFF;
 497                         cvlan3 = (slab_qinq3 >> 16) & 0xFFF;
 498
 499                         tc0 = (dscp0 >> 2) & 0x3;
 500                         tc1 = (dscp1 >> 2) & 0x3;
 501                         tc2 = (dscp2 >> 2) & 0x3;
 502                         tc3 = (dscp3 >> 2) & 0x3;
 503
 504                         tc_q0 = dscp0 & 0x3;
 505                         tc_q1 = dscp1 & 0x3;
 506                         tc_q2 = dscp2 & 0x3;
 507                         tc_q3 = dscp3 & 0x3;
 508                 } else {
 509                         uint32_t ip_src0 = rte_bswap32(ip0->src_addr);
 510                         uint32_t ip_src1 = rte_bswap32(ip1->src_addr);
 511                         uint32_t ip_src2 = rte_bswap32(ip2->src_addr);
 512                         uint32_t ip_src3 = rte_bswap32(ip3->src_addr);
 513
 514                         svlan0 = 0;
 515                         svlan1 = 0;
 516                         svlan2 = 0;
 517                         svlan3 = 0;
 518
 519                         cvlan0 = (ip_src0 >> 16) & 0xFFF;
 520                         cvlan1 = (ip_src1 >> 16) & 0xFFF;
 521                         cvlan2 = (ip_src2 >> 16) & 0xFFF;
 522                         cvlan3 = (ip_src3 >> 16) & 0xFFF;
 523
 524                         tc0 = (ip_src0 >> 2) & 0x3;
 525                         tc1 = (ip_src1 >> 2) & 0x3;
 526                         tc2 = (ip_src2 >> 2) & 0x3;
 527                         tc3 = (ip_src3 >> 2) & 0x3;
 528
 529                         tc_q0 = ip_src0 & 0x3;
 530                         tc_q1 = ip_src1 & 0x3;
 531                         tc_q2 = ip_src2 & 0x3;
 532                         tc_q3 = ip_src3 & 0x3;
 533                 }
 534
 535                 sched0 = RTE_SCHED_PORT_HIERARCHY(svlan0,
 536                         cvlan0,
 537                         tc0,
 538                         tc_q0,
 539                         e_RTE_METER_GREEN);
 540                 sched1 = RTE_SCHED_PORT_HIERARCHY(svlan1,
 541                         cvlan1,
 542                         tc1,
 543                         tc_q1,
 544                         e_RTE_METER_GREEN);
 545                 sched2 = RTE_SCHED_PORT_HIERARCHY(svlan2,
 546                         cvlan2,
 547                         tc2,
 548                         tc_q2,
 549                         e_RTE_METER_GREEN);
 550                 sched3 = RTE_SCHED_PORT_HIERARCHY(svlan3,
 551                         cvlan3,
 552                         tc3,
 553                         tc_q3,
 554                         e_RTE_METER_GREEN);
 555
 556         }
 557
 558         /* Write */
 559         pkts[0]->data_off = data_offset0;
 560         pkts[1]->data_off = data_offset1;
 561         pkts[2]->data_off = data_offset2;
 562         pkts[3]->data_off = data_offset3;
 563
 564         pkts[0]->data_len = total_length0;
 565         pkts[1]->data_len = total_length1;
 566         pkts[2]->data_len = total_length2;
 567         pkts[3]->data_len = total_length3;
 568
 569         pkts[0]->pkt_len = total_length0;
 570         pkts[1]->pkt_len = total_length1;
 571         pkts[2]->pkt_len = total_length2;
 572         pkts[3]->pkt_len = total_length3;
 573
 574         if ((qinq == 0) && (mpls == 0)) {
 575                 *slab0_ptr0 = entry0->slab[0];
 576                 *slab0_ptr1 = entry1->slab[0];
 577                 *slab0_ptr2 = entry2->slab[0];
 578                 *slab0_ptr3 = entry3->slab[0];
 579
 580                 if (arp == 0) {
 581                         MACADDR_DST_WRITE(slab1_ptr0, entry0->slab[1]);
 582                         MACADDR_DST_WRITE(slab1_ptr1, entry1->slab[1]);
 583                         MACADDR_DST_WRITE(slab1_ptr2, entry2->slab[1]);
 584                         MACADDR_DST_WRITE(slab1_ptr3, entry3->slab[1]);
 585                 }
 586         }
 587
 588         if (qinq) {
 589                 *slab0_ptr0 = entry0->slab[0];
 590                 *slab0_ptr1 = entry1->slab[0];
 591                 *slab0_ptr2 = entry2->slab[0];
 592                 *slab0_ptr3 = entry3->slab[0];
 593
 594                 *slab1_ptr0 = entry0->slab[1];
 595                 *slab1_ptr1 = entry1->slab[1];
 596                 *slab1_ptr2 = entry2->slab[1];
 597                 *slab1_ptr3 = entry3->slab[1];
 598
 599                 if (arp == 0) {
 600                         MACADDR_DST_WRITE(slab2_ptr0, entry0->slab[2]);
 601                         MACADDR_DST_WRITE(slab2_ptr1, entry1->slab[2]);
 602                         MACADDR_DST_WRITE(slab2_ptr2, entry2->slab[2]);
 603                         MACADDR_DST_WRITE(slab2_ptr3, entry3->slab[2]);
 604                 }
 605
 606                 if (qinq_sched) {
 607                         pkts[0]->hash.sched.lo = sched0 & 0xFFFFFFFF;
 608                         pkts[0]->hash.sched.hi = sched0 >> 32;
 609                         pkts[1]->hash.sched.lo = sched1 & 0xFFFFFFFF;
 610                         pkts[1]->hash.sched.hi = sched1 >> 32;
 611                         pkts[2]->hash.sched.lo = sched2 & 0xFFFFFFFF;
 612                         pkts[2]->hash.sched.hi = sched2 >> 32;
 613                         pkts[3]->hash.sched.lo = sched3 & 0xFFFFFFFF;
 614                         pkts[3]->hash.sched.hi = sched3 >> 32;
 615                 }
 616         }
 617
 618         if (mpls) {
 619                 if (mpls_color_mark) {
 620                         uint64_t mpls_exp0 = rte_bswap64(
 621                                 (MPLS_LABEL(0, pkt0_color, 0, 0) << 32) |
 622                                 MPLS_LABEL(0, pkt0_color, 0, 0));
 623                         uint64_t mpls_exp1 = rte_bswap64(
 624                                 (MPLS_LABEL(0, pkt1_color, 0, 0) << 32) |
 625                                 MPLS_LABEL(0, pkt1_color, 0, 0));
 626                         uint64_t mpls_exp2 = rte_bswap64(
 627                                 (MPLS_LABEL(0, pkt2_color, 0, 0) << 32) |
 628                                 MPLS_LABEL(0, pkt2_color, 0, 0));
 629                         uint64_t mpls_exp3 = rte_bswap64(
 630                                 (MPLS_LABEL(0, pkt3_color, 0, 0) << 32) |
 631                                 MPLS_LABEL(0, pkt3_color, 0, 0));
 632
 633                         *slab0_ptr0 = entry0->slab[0] | mpls_exp0;
 634                         *slab0_ptr1 = entry1->slab[0] | mpls_exp1;
 635                         *slab0_ptr2 = entry2->slab[0] | mpls_exp2;
 636                         *slab0_ptr3 = entry3->slab[0] | mpls_exp3;
 637
 638                         *slab1_ptr0 = entry0->slab[1] | mpls_exp0;
 639                         *slab1_ptr1 = entry1->slab[1] | mpls_exp1;
 640                         *slab1_ptr2 = entry2->slab[1] | mpls_exp2;
 641                         *slab1_ptr3 = entry3->slab[1] | mpls_exp3;
 642
 643                         *slab2_ptr0 = entry0->slab[2];
 644                         *slab2_ptr1 = entry1->slab[2];
 645                         *slab2_ptr2 = entry2->slab[2];
 646                         *slab2_ptr3 = entry3->slab[2];
 647                 } else {
 648                         *slab0_ptr0 = entry0->slab[0];
 649                         *slab0_ptr1 = entry1->slab[0];
 650                         *slab0_ptr2 = entry2->slab[0];
 651                         *slab0_ptr3 = entry3->slab[0];
 652
 653                         *slab1_ptr0 = entry0->slab[1];
 654                         *slab1_ptr1 = entry1->slab[1];
 655                         *slab1_ptr2 = entry2->slab[1];
 656                         *slab1_ptr3 = entry3->slab[1];
 657
 658                         *slab2_ptr0 = entry0->slab[2];
 659                         *slab2_ptr1 = entry1->slab[2];
 660                         *slab2_ptr2 = entry2->slab[2];
 661                         *slab2_ptr3 = entry3->slab[2];
 662                 }
 663
 664                 if (arp == 0) {
 665                         MACADDR_DST_WRITE(slab3_ptr0, entry0->slab[3]);
 666                         MACADDR_DST_WRITE(slab3_ptr1, entry1->slab[3]);
 667                         MACADDR_DST_WRITE(slab3_ptr2, entry2->slab[3]);
 668                         MACADDR_DST_WRITE(slab3_ptr3, entry3->slab[3]);
 669                 }
 670         }
 671
 672         if (arp) {
 673                 arp_key0->port_id = port_id0;
 674                 arp_key1->port_id = port_id1;
 675                 arp_key2->port_id = port_id2;
 676                 arp_key3->port_id = port_id3;
 677
 678                 arp_key0->ip = nh_ip0;
 679                 arp_key1->ip = nh_ip1;
 680                 arp_key2->ip = nh_ip2;
 681                 arp_key3->ip = nh_ip3;
 682         }
 683 }
 684
 685 #define PKT_WORK_ROUTING_ETHERNET(arp)                          \
 686 static inline void                                              \
 687 pkt_work_routing_ether_arp##arp(                                \
 688         struct rte_mbuf *pkt,                                   \
 689         struct rte_pipeline_table_entry *table_entry,           \
 690         void *arg)                                              \
 691 {                                                               \
 692         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 0, 0);\
 693 }
 694
 695 #define PKT4_WORK_ROUTING_ETHERNET(arp)                         \
 696 static inline void                                              \
 697 pkt4_work_routing_ether_arp##arp(                               \
 698         struct rte_mbuf **pkts,                                 \
 699         struct rte_pipeline_table_entry **table_entries,        \
 700         void *arg)                                              \
 701 {                                                               \
 702         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 0, 0);\
 703 }
 704
 705 #define routing_table_ah_hit_ether(arp)                         \
 706 PKT_WORK_ROUTING_ETHERNET(arp)                                  \
 707 PKT4_WORK_ROUTING_ETHERNET(arp)                                 \
 708 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_arp##arp,      \
 709         pkt_work_routing_ether_arp##arp,                        \
 710         pkt4_work_routing_ether_arp##arp)
 711
 712 routing_table_ah_hit_ether(0)
 713 routing_table_ah_hit_ether(1)
 714
 715 #define PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)              \
 716 static inline void                                              \
 717 pkt_work_routing_ether_qinq_sched##sched##_arp##arp(            \
 718         struct rte_mbuf *pkt,                                   \
 719         struct rte_pipeline_table_entry *table_entry,           \
 720         void *arg)                                              \
 721 {                                                               \
 722         pkt_work_routing(pkt, table_entry, arg, arp, 1, sched, 0, 0);\
 723 }
 724
 725 #define PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)             \
 726 static inline void                                              \
 727 pkt4_work_routing_ether_qinq_sched##sched##_arp##arp(           \
 728         struct rte_mbuf **pkts,                                 \
 729         struct rte_pipeline_table_entry **table_entries,        \
 730         void *arg)                                              \
 731 {                                                               \
 732         pkt4_work_routing(pkts, table_entries, arg, arp, 1, sched, 0, 0);\
 733 }
 734
 735 #define routing_table_ah_hit_ether_qinq(sched, arp)             \
 736 PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                      \
 737 PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                     \
 738 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_qinq_sched##sched##_arp##arp,\
 739         pkt_work_routing_ether_qinq_sched##sched##_arp##arp,    \
 740         pkt4_work_routing_ether_qinq_sched##sched##_arp##arp)
 741
 742 routing_table_ah_hit_ether_qinq(0, 0)
 743 routing_table_ah_hit_ether_qinq(1, 0)
 744 routing_table_ah_hit_ether_qinq(2, 0)
 745 routing_table_ah_hit_ether_qinq(0, 1)
 746 routing_table_ah_hit_ether_qinq(1, 1)
 747 routing_table_ah_hit_ether_qinq(2, 1)
 748
 749 #define PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)              \
 750 static inline void                                              \
 751 pkt_work_routing_ether_mpls_color##color##_arp##arp(            \
 752         struct rte_mbuf *pkt,                                   \
 753         struct rte_pipeline_table_entry *table_entry,           \
 754         void *arg)                                              \
 755 {                                                               \
 756         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 1, color);\
 757 }
 758
 759 #define PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)             \
 760 static inline void                                              \
 761 pkt4_work_routing_ether_mpls_color##color##_arp##arp(           \
 762         struct rte_mbuf **pkts,                                 \
 763         struct rte_pipeline_table_entry **table_entries,        \
 764         void *arg)                                              \
 765 {                                                               \
 766         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 1, color);\
 767 }
 768
 769 #define routing_table_ah_hit_ether_mpls(color, arp)             \
 770 PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)                      \
 771 PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)                     \
 772 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_mpls_color##color##_arp##arp,\
 773         pkt_work_routing_ether_mpls_color##color##_arp##arp,    \
 774         pkt4_work_routing_ether_mpls_color##color##_arp##arp)
 775
 776 routing_table_ah_hit_ether_mpls(0, 0)
 777 routing_table_ah_hit_ether_mpls(1, 0)
 778 routing_table_ah_hit_ether_mpls(0, 1)
 779 routing_table_ah_hit_ether_mpls(1, 1)
 780
 781 static rte_pipeline_table_action_handler_hit
 782 get_routing_table_ah_hit(struct pipeline_routing *p)
 783 {
 784         if (p->params.dbg_ah_disable)
 785                 return NULL;
 786
 787         switch (p->params.encap) {
 788         case PIPELINE_ROUTING_ENCAP_ETHERNET:
 789                 return (p->params.n_arp_entries) ?
 790                         routing_table_ah_hit_ether_arp1 :
 791                         routing_table_ah_hit_ether_arp0;
 792
 793         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
 794                 if (p->params.n_arp_entries)
 795                         switch (p->params.qinq_sched) {
 796                         case 0:
 797                                 return routing_table_ah_hit_ether_qinq_sched0_arp1;
 798                         case 1:
 799                                 return routing_table_ah_hit_ether_qinq_sched1_arp1;
 800                         case 2:
 801                                 return routing_table_ah_hit_ether_qinq_sched2_arp1;
 802                         default:
 803                                 return NULL;
 804                         }
 805                  else
 806                         switch (p->params.qinq_sched) {
 807                         case 0:
 808                                 return routing_table_ah_hit_ether_qinq_sched0_arp0;
 809                         case 1:
 810                                 return routing_table_ah_hit_ether_qinq_sched1_arp0;
 811                         case 2:
 812                                 return routing_table_ah_hit_ether_qinq_sched2_arp0;
 813                         default:
 814                                 return NULL;
 815                         }
 816
 817         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
 818                 if (p->params.n_arp_entries)
 819                         if (p->params.mpls_color_mark)
 820                                 return routing_table_ah_hit_ether_mpls_color1_arp1;
 821                         else
 822                                 return routing_table_ah_hit_ether_mpls_color0_arp1;
 823                 else
 824                         if (p->params.mpls_color_mark)
 825                                 return routing_table_ah_hit_ether_mpls_color1_arp0;
 826                         else
 827                                 return routing_table_ah_hit_ether_mpls_color0_arp0;
 828
 829         default:
 830                 return NULL;
 831         }
 832 }
 833
 834 /*
 835  * ARP table
 836  */
 837 struct arp_table_entry {
 838         struct rte_pipeline_table_entry head;
 839         uint64_t macaddr;
 840 };
 841
 842 /**
 843  * ARP table AH
 844  */
 845 static inline void
 846 pkt_work_arp(
 847         struct rte_mbuf *pkt,
 848         struct rte_pipeline_table_entry *table_entry,
 849         __rte_unused void *arg)
 850 {
 851         struct arp_table_entry *entry = (struct arp_table_entry *) table_entry;
 852
 853         /* Read */
 854         uint64_t macaddr_dst = entry->macaddr;
 855         uint64_t *slab_ptr = (uint64_t *) ((char *) pkt->buf_addr +
 856                 (pkt->data_off - 2));
 857
 858         /* Compute */
 859
 860         /* Write */
 861         MACADDR_DST_WRITE(slab_ptr, macaddr_dst);
 862 }
 863
 864 static inline void
 865 pkt4_work_arp(
 866         struct rte_mbuf **pkts,
 867         struct rte_pipeline_table_entry **table_entries,
 868         __rte_unused void *arg)
 869 {
 870         struct arp_table_entry *entry0 =
 871                 (struct arp_table_entry *) table_entries[0];
 872         struct arp_table_entry *entry1 =
 873                 (struct arp_table_entry *) table_entries[1];
 874         struct arp_table_entry *entry2 =
 875                 (struct arp_table_entry *) table_entries[2];
 876         struct arp_table_entry *entry3 =
 877                 (struct arp_table_entry *) table_entries[3];
 878
 879         /* Read */
 880         uint64_t macaddr_dst0 = entry0->macaddr;
 881         uint64_t macaddr_dst1 = entry1->macaddr;
 882         uint64_t macaddr_dst2 = entry2->macaddr;
 883         uint64_t macaddr_dst3 = entry3->macaddr;
 884
 885         uint64_t *slab_ptr0 = (uint64_t *) ((char *) pkts[0]->buf_addr +
 886                 (pkts[0]->data_off - 2));
 887         uint64_t *slab_ptr1 = (uint64_t *) ((char *) pkts[1]->buf_addr +
 888                 (pkts[1]->data_off - 2));
 889         uint64_t *slab_ptr2 = (uint64_t *) ((char *) pkts[2]->buf_addr +
 890                 (pkts[2]->data_off - 2));
 891         uint64_t *slab_ptr3 = (uint64_t *) ((char *) pkts[3]->buf_addr +
 892                 (pkts[3]->data_off - 2));
 893
 894         /* Compute */
 895
 896         /* Write */
 897         MACADDR_DST_WRITE(slab_ptr0, macaddr_dst0);
 898         MACADDR_DST_WRITE(slab_ptr1, macaddr_dst1);
 899         MACADDR_DST_WRITE(slab_ptr2, macaddr_dst2);
 900         MACADDR_DST_WRITE(slab_ptr3, macaddr_dst3);
 901 }
 902
 903 PIPELINE_TABLE_AH_HIT(arp_table_ah_hit,
 904         pkt_work_arp,
 905         pkt4_work_arp);
 906
 907 static rte_pipeline_table_action_handler_hit
 908 get_arp_table_ah_hit(struct pipeline_routing *p)
 909 {
 910         if (p->params.dbg_ah_disable)
 911                 return NULL;
 912
 913         return arp_table_ah_hit;
 914 }
 915
 916 /*
 917  * Argument parsing
 918  */
 919 int
 920 pipeline_routing_parse_args(struct pipeline_routing_params *p,
 921         struct pipeline_params *params)
 922 {
 923         uint32_t n_routes_present = 0;
 924         uint32_t encap_present = 0;
 925         uint32_t qinq_sched_present = 0;
 926         uint32_t mpls_color_mark_present = 0;
 927         uint32_t n_arp_entries_present = 0;
 928         uint32_t ip_hdr_offset_present = 0;
 929         uint32_t arp_key_offset_present = 0;
 930         uint32_t color_offset_present = 0;
 931         uint32_t dbg_ah_disable_present = 0;
 932         uint32_t i;
 933
 934         /* default values */
 935         p->n_routes = PIPELINE_ROUTING_N_ROUTES_DEFAULT;
 936         p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
 937         p->qinq_sched = 0;
 938         p->mpls_color_mark = 0;
 939         p->n_arp_entries = 0;
 940         p->dbg_ah_disable = 0;
 941
 942         for (i = 0; i < params->n_args; i++) {
 943                 char *arg_name = params->args_name[i];
 944                 char *arg_value = params->args_value[i];
 945
 946                 /* n_routes */
 947                 if (strcmp(arg_name, "n_routes") == 0) {
 948                         int status;
 949
 950                         PIPELINE_PARSE_ERR_DUPLICATE(
 951                                 n_routes_present == 0, params->name,
 952                                 arg_name);
 953                         n_routes_present = 1;
 954
 955                         status = parser_read_uint32(&p->n_routes,
 956                                 arg_value);
 957                         PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
 958                                 (p->n_routes != 0)), params->name,
 959                                 arg_name, arg_value);
 960                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 961                                 params->name, arg_name, arg_value);
 962
 963                         continue;
 964                 }
 965
 966                 /* encap */
 967                 if (strcmp(arg_name, "encap") == 0) {
 968                         PIPELINE_PARSE_ERR_DUPLICATE(encap_present == 0,
 969                                 params->name, arg_name);
 970                         encap_present = 1;
 971
 972                         /* ethernet */
 973                         if (strcmp(arg_value, "ethernet") == 0) {
 974                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
 975                                 continue;
 976                         }
 977
 978                         /* ethernet_qinq */
 979                         if (strcmp(arg_value, "ethernet_qinq") == 0) {
 980                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ;
 981                                 continue;
 982                         }
 983
 984                         /* ethernet_mpls */
 985                         if (strcmp(arg_value, "ethernet_mpls") == 0) {
 986                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS;
 987                                 continue;
 988                         }
 989
 990                         /* any other */
 991                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
 992                                 arg_name, arg_value);
 993                 }
 994
 995                 /* qinq_sched */
 996                 if (strcmp(arg_name, "qinq_sched") == 0) {
 997                         int status;
 998
 999                         PIPELINE_PARSE_ERR_DUPLICATE(
1000                                 qinq_sched_present == 0, params->name,
1001                                 arg_name);
1002                         qinq_sched_present = 1;
1003
1004                         status = parser_read_arg_bool(arg_value);
1005                         if (status == -EINVAL) {
1006                                 if (strcmp(arg_value, "test") == 0) {
1007                                         p->qinq_sched = 2;
1008                                         continue;
1009                                 }
1010                         } else {
1011                                 p->qinq_sched = status;
1012                                 continue;
1013                         }
1014
1015                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1016                                 arg_name, arg_value);
1017                 }
1018
1019                 /* mpls_color_mark */
1020                 if (strcmp(arg_name, "mpls_color_mark") == 0) {
1021                         int status;
1022
1023                         PIPELINE_PARSE_ERR_DUPLICATE(
1024                                 mpls_color_mark_present == 0,
1025                                 params->name, arg_name);
1026                         mpls_color_mark_present = 1;
1027
1028
1029                         status = parser_read_arg_bool(arg_value);
1030                         if (status >= 0) {
1031                                 p->mpls_color_mark = status;
1032                                 continue;
1033                         }
1034
1035                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1036                                 arg_name, arg_value);
1037                 }
1038
1039                 /* n_arp_entries */
1040                 if (strcmp(arg_name, "n_arp_entries") == 0) {
1041                         int status;
1042
1043                         PIPELINE_PARSE_ERR_DUPLICATE(
1044                                 n_arp_entries_present == 0, params->name,
1045                                 arg_name);
1046                         n_arp_entries_present = 1;
1047
1048                         status = parser_read_uint32(&p->n_arp_entries,
1049                                 arg_value);
1050                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1051                                 params->name, arg_name, arg_value);
1052                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1053                                 params->name, arg_name, arg_value);
1054
1055                         continue;
1056                 }
1057
1058                 /* ip_hdr_offset */
1059                 if (strcmp(arg_name, "ip_hdr_offset") == 0) {
1060                         int status;
1061
1062                         PIPELINE_PARSE_ERR_DUPLICATE(
1063                                 ip_hdr_offset_present == 0, params->name,
1064                                 arg_name);
1065                         ip_hdr_offset_present = 1;
1066
1067                         status = parser_read_uint32(&p->ip_hdr_offset,
1068                                 arg_value);
1069                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1070                                 params->name, arg_name, arg_value);
1071                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1072                                 params->name, arg_name, arg_value);
1073
1074                         continue;
1075                 }
1076
1077                 /* arp_key_offset */
1078                 if (strcmp(arg_name, "arp_key_offset") == 0) {
1079                         int status;
1080
1081                         PIPELINE_PARSE_ERR_DUPLICATE(
1082                                 arp_key_offset_present == 0, params->name,
1083                                 arg_name);
1084                         arp_key_offset_present = 1;
1085
1086                         status = parser_read_uint32(&p->arp_key_offset,
1087                                 arg_value);
1088                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1089                                 params->name, arg_name, arg_value);
1090                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1091                                 params->name, arg_name, arg_value);
1092
1093                         continue;
1094                 }
1095
1096                 /* color_offset */
1097                 if (strcmp(arg_name, "color_offset") == 0) {
1098                         int status;
1099
1100                         PIPELINE_PARSE_ERR_DUPLICATE(
1101                                 color_offset_present == 0, params->name,
1102                                 arg_name);
1103                         color_offset_present = 1;
1104
1105                         status = parser_read_uint32(&p->color_offset,
1106                                 arg_value);
1107                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1108                                 params->name, arg_name, arg_value);
1109                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1110                                 params->name, arg_name, arg_value);
1111
1112                         continue;
1113                 }
1114
1115                 /* debug */
1116                 if (strcmp(arg_name, "dbg_ah_disable") == 0) {
1117                         int status;
1118
1119                         PIPELINE_PARSE_ERR_DUPLICATE(
1120                                 dbg_ah_disable_present == 0, params->name,
1121                                 arg_name);
1122                         dbg_ah_disable_present = 1;
1123
1124                         status = parser_read_arg_bool(arg_value);
1125                         if (status >= 0) {
1126                                 p->dbg_ah_disable = status;
1127                                 continue;
1128                         }
1129
1130                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1131                                 arg_name, arg_value);
1132
1133                         continue;
1134                 }
1135
1136                 /* any other */
1137                 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
1138         }
1139
1140         /* Check that mandatory arguments are present */
1141         PIPELINE_PARSE_ERR_MANDATORY(ip_hdr_offset_present, params->name,
1142                 "ip_hdr_offset");
1143
1144         /* Check relations between arguments */
1145         switch (p->encap) {
1146         case PIPELINE_ROUTING_ENCAP_ETHERNET:
1147                 PIPELINE_ARG_CHECK((!p->qinq_sched), "Parse error in "
1148                         "section \"%s\": encap = ethernet, therefore "
1149                         "qinq_sched = yes/test is not allowed",
1150                         params->name);
1151                 PIPELINE_ARG_CHECK((!p->mpls_color_mark), "Parse error "
1152                         "in section \"%s\": encap = ethernet, therefore "
1153                         "mpls_color_mark = yes is not allowed",
1154                         params->name);
1155                 PIPELINE_ARG_CHECK((!color_offset_present), "Parse error "
1156                         "in section \"%s\": encap = ethernet, therefore "
1157                         "color_offset is not allowed",
1158                         params->name);
1159                 break;
1160
1161         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
1162                 PIPELINE_ARG_CHECK((!p->mpls_color_mark), "Parse error "
1163                         "in section \"%s\": encap = ethernet_qinq, "
1164                         "therefore mpls_color_mark = yes is not allowed",
1165                         params->name);
1166                 PIPELINE_ARG_CHECK((!color_offset_present), "Parse error "
1167                         "in section \"%s\": encap = ethernet_qinq, "
1168                         "therefore color_offset is not allowed",
1169                         params->name);
1170                 break;
1171
1172         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
1173                 PIPELINE_ARG_CHECK((!p->qinq_sched), "Parse error in "
1174                         "section \"%s\": encap = ethernet_mpls, therefore "
1175                         "qinq_sched  = yes/test is not allowed",
1176                         params->name);
1177                 break;
1178         }
1179
1180         PIPELINE_ARG_CHECK((!(p->n_arp_entries &&
1181                 (!arp_key_offset_present))), "Parse error in section "
1182                         "\"%s\": n_arp_entries is set while "
1183                         "arp_key_offset is not set", params->name);
1184
1185         PIPELINE_ARG_CHECK((!((p->n_arp_entries == 0) &&
1186                 arp_key_offset_present)), "Parse error in section "
1187                         "\"%s\": arp_key_offset present while "
1188                         "n_arp_entries is not set", params->name);
1189
1190         return 0;
1191 }
1192
1193 static void *
1194 pipeline_routing_init(struct pipeline_params *params,
1195         __rte_unused void *arg)
1196 {
1197         struct pipeline *p;
1198         struct pipeline_routing *p_rt;
1199         uint32_t size, i;
1200
1201         /* Check input arguments */
1202         if ((params == NULL) ||
1203                 (params->n_ports_in == 0) ||
1204                 (params->n_ports_out == 0))
1205                 return NULL;
1206
1207         /* Memory allocation */
1208         size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_routing));
1209         p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
1210         p_rt = (struct pipeline_routing *) p;
1211         if (p == NULL)
1212                 return NULL;
1213
1214         strcpy(p->name, params->name);
1215         p->log_level = params->log_level;
1216
1217         PLOG(p, HIGH, "Routing");
1218
1219         /* Parse arguments */
1220         if (pipeline_routing_parse_args(&p_rt->params, params))
1221                 return NULL;
1222
1223         /* Pipeline */
1224         {
1225                 struct rte_pipeline_params pipeline_params = {
1226                         .name = params->name,
1227                         .socket_id = params->socket_id,
1228                         .offset_port_id = 0,
1229                 };
1230
1231                 p->p = rte_pipeline_create(&pipeline_params);
1232                 if (p->p == NULL) {
1233                         rte_free(p);
1234                         return NULL;
1235                 }
1236         }
1237
1238         /* Input ports */
1239         p->n_ports_in = params->n_ports_in;
1240         for (i = 0; i < p->n_ports_in; i++) {
1241                 struct rte_pipeline_port_in_params port_params = {
1242                         .ops = pipeline_port_in_params_get_ops(
1243                                 &params->port_in[i]),
1244                         .arg_create = pipeline_port_in_params_convert(
1245                                 &params->port_in[i]),
1246                         .f_action = NULL,
1247                         .arg_ah = NULL,
1248                         .burst_size = params->port_in[i].burst_size,
1249                 };
1250
1251                 int status = rte_pipeline_port_in_create(p->p,
1252                         &port_params,
1253                         &p->port_in_id[i]);
1254
1255                 if (status) {
1256                         rte_pipeline_free(p->p);
1257                         rte_free(p);
1258                         return NULL;
1259                 }
1260         }
1261
1262         /* Output ports */
1263         p->n_ports_out = params->n_ports_out;
1264         for (i = 0; i < p->n_ports_out; i++) {
1265                 struct rte_pipeline_port_out_params port_params = {
1266                         .ops = pipeline_port_out_params_get_ops(
1267                                 &params->port_out[i]),
1268                         .arg_create = pipeline_port_out_params_convert(
1269                                 &params->port_out[i]),
1270                         .f_action = NULL,
1271                         .arg_ah = NULL,
1272                 };
1273
1274                 int status = rte_pipeline_port_out_create(p->p,
1275                         &port_params,
1276                         &p->port_out_id[i]);
1277
1278                 if (status) {
1279                         rte_pipeline_free(p->p);
1280                         rte_free(p);
1281                         return NULL;
1282                 }
1283         }
1284
1285         /* Routing table */
1286         p->n_tables = 1;
1287         {
1288                 struct rte_table_lpm_params table_lpm_params = {
1289                         .name = p->name,
1290                         .n_rules = p_rt->params.n_routes,
1291                         .number_tbl8s = PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s,
1292                         .flags = 0,
1293                         .entry_unique_size = sizeof(struct routing_table_entry),
1294                         .offset = p_rt->params.ip_hdr_offset +
1295                                 __builtin_offsetof(struct ipv4_hdr, dst_addr),
1296                 };
1297
1298                 struct rte_pipeline_table_params table_params = {
1299                                 .ops = &rte_table_lpm_ops,
1300                                 .arg_create = &table_lpm_params,
1301                                 .f_action_hit = get_routing_table_ah_hit(p_rt),
1302                                 .f_action_miss = NULL,
1303                                 .arg_ah = p_rt,
1304                                 .action_data_size =
1305                                         sizeof(struct routing_table_entry) -
1306                                         sizeof(struct rte_pipeline_table_entry),
1307                         };
1308
1309                 int status;
1310
1311                 status = rte_pipeline_table_create(p->p,
1312                         &table_params,
1313                         &p->table_id[0]);
1314
1315                 if (status) {
1316                         rte_pipeline_free(p->p);
1317                         rte_free(p);
1318                         return NULL;
1319                 }
1320         }
1321
1322         /* ARP table configuration */
1323         if (p_rt->params.n_arp_entries) {
1324                 struct rte_table_hash_key8_ext_params table_arp_params = {
1325                         .n_entries = p_rt->params.n_arp_entries,
1326                         .n_entries_ext = p_rt->params.n_arp_entries,
1327                         .f_hash = hash_default_key8,
1328                         .seed = 0,
1329                         .signature_offset = 0, /* Unused */
1330                         .key_offset = p_rt->params.arp_key_offset,
1331                 };
1332
1333                 struct rte_pipeline_table_params table_params = {
1334                         .ops = &rte_table_hash_key8_ext_dosig_ops,
1335                         .arg_create = &table_arp_params,
1336                         .f_action_hit = get_arp_table_ah_hit(p_rt),
1337                         .f_action_miss = NULL,
1338                         .arg_ah = p_rt,
1339                         .action_data_size = sizeof(struct arp_table_entry) -
1340                                 sizeof(struct rte_pipeline_table_entry),
1341                 };
1342
1343                 int status;
1344
1345                 status = rte_pipeline_table_create(p->p,
1346                         &table_params,
1347                         &p->table_id[1]);
1348
1349                 if (status) {
1350                         rte_pipeline_free(p->p);
1351                         rte_free(p);
1352                         return NULL;
1353                 }
1354
1355                 p->n_tables++;
1356         }
1357
1358         /* Connecting input ports to tables */
1359         for (i = 0; i < p->n_ports_in; i++) {
1360                 int status = rte_pipeline_port_in_connect_to_table(p->p,
1361                         p->port_in_id[i],
1362                         p->table_id[0]);
1363
1364                 if (status) {
1365                         rte_pipeline_free(p->p);
1366                         rte_free(p);
1367                         return NULL;
1368                 }
1369         }
1370
1371         /* Enable input ports */
1372         for (i = 0; i < p->n_ports_in; i++) {
1373                 int status = rte_pipeline_port_in_enable(p->p,
1374                         p->port_in_id[i]);
1375
1376                 if (status) {
1377                         rte_pipeline_free(p->p);
1378                         rte_free(p);
1379                         return NULL;
1380                 }
1381         }
1382
1383         /* Check pipeline consistency */
1384         if (rte_pipeline_check(p->p) < 0) {
1385                 rte_pipeline_free(p->p);
1386                 rte_free(p);
1387                 return NULL;
1388         }
1389
1390         /* Message queues */
1391         p->n_msgq = params->n_msgq;
1392         for (i = 0; i < p->n_msgq; i++)
1393                 p->msgq_in[i] = params->msgq_in[i];
1394         for (i = 0; i < p->n_msgq; i++)
1395                 p->msgq_out[i] = params->msgq_out[i];
1396
1397         /* Message handlers */
1398         memcpy(p->handlers, handlers, sizeof(p->handlers));
1399         memcpy(p_rt->custom_handlers,
1400                 custom_handlers,
1401                 sizeof(p_rt->custom_handlers));
1402
1403         return p;
1404 }
1405
1406 static int
1407 pipeline_routing_free(void *pipeline)
1408 {
1409         struct pipeline *p = (struct pipeline *) pipeline;
1410
1411         /* Check input arguments */
1412         if (p == NULL)
1413                 return -1;
1414
1415         /* Free resources */
1416         rte_pipeline_free(p->p);
1417         rte_free(p);
1418         return 0;
1419 }
1420
1421 static int
1422 pipeline_routing_track(void *pipeline,
1423         __rte_unused uint32_t port_in,
1424         uint32_t *port_out)
1425 {
1426         struct pipeline *p = (struct pipeline *) pipeline;
1427
1428         /* Check input arguments */
1429         if ((p == NULL) ||
1430                 (port_in >= p->n_ports_in) ||
1431                 (port_out == NULL))
1432                 return -1;
1433
1434         if (p->n_ports_in == 1) {
1435                 *port_out = 0;
1436                 return 0;
1437         }
1438
1439         return -1;
1440 }
1441
1442 static int
1443 pipeline_routing_timer(void *pipeline)
1444 {
1445         struct pipeline *p = (struct pipeline *) pipeline;
1446
1447         pipeline_msg_req_handle(p);
1448         rte_pipeline_flush(p->p);
1449
1450         return 0;
1451 }
1452
1453 void *
1454 pipeline_routing_msg_req_custom_handler(struct pipeline *p,
1455         void *msg)
1456 {
1457         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1458         struct pipeline_custom_msg_req *req = msg;
1459         pipeline_msg_req_handler f_handle;
1460
1461         f_handle = (req->subtype < PIPELINE_ROUTING_MSG_REQS) ?
1462                 p_rt->custom_handlers[req->subtype] :
1463                 pipeline_msg_req_invalid_handler;
1464
1465         if (f_handle == NULL)
1466                 f_handle = pipeline_msg_req_invalid_handler;
1467
1468         return f_handle(p, req);
1469 }
1470
1471 void *
1472 pipeline_routing_msg_req_route_add_handler(struct pipeline *p, void *msg)
1473 {
1474         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1475         struct pipeline_routing_route_add_msg_req *req = msg;
1476         struct pipeline_routing_route_add_msg_rsp *rsp = msg;
1477
1478         struct rte_table_lpm_key key = {
1479                 .ip = req->key.key.ipv4.ip,
1480                 .depth = req->key.key.ipv4.depth,
1481         };
1482
1483         struct routing_table_entry entry_arp0 = {
1484                 .head = {
1485                         .action = RTE_PIPELINE_ACTION_PORT,
1486                         {.port_id = p->port_out_id[req->data.port_id]},
1487                 },
1488
1489                 .flags = req->data.flags,
1490                 .port_id = req->data.port_id,
1491                 .ip = 0,
1492                 .data_offset = 0,
1493                 .ether_l2_length = 0,
1494                 .slab = {0},
1495                 .slab_offset = {0},
1496         };
1497
1498         struct routing_table_entry entry_arp1 = {
1499                 .head = {
1500                         .action = RTE_PIPELINE_ACTION_TABLE,
1501                         {.table_id = p->table_id[1]},
1502                 },
1503
1504                 .flags = req->data.flags,
1505                 .port_id = req->data.port_id,
1506                 .ip = rte_bswap32(req->data.ethernet.ip),
1507                 .data_offset = 0,
1508                 .ether_l2_length = 0,
1509                 .slab = {0},
1510                 .slab_offset = {0},
1511         };
1512
1513         struct rte_pipeline_table_entry *entry = (p_rt->params.n_arp_entries) ?
1514                 (struct rte_pipeline_table_entry *) &entry_arp1 :
1515                 (struct rte_pipeline_table_entry *) &entry_arp0;
1516
1517         if ((req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) ||
1518                 ((p_rt->params.n_arp_entries == 0) &&
1519                         (req->data.flags & PIPELINE_ROUTING_ROUTE_ARP)) ||
1520                 (p_rt->params.n_arp_entries &&
1521                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_ARP) == 0)) ||
1522                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1523                         (req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ)) ||
1524                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1525                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ) == 0)) ||
1526                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1527                         (req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS)) ||
1528                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1529                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS) == 0))) {
1530                 rsp->status = -1;
1531                 return rsp;
1532         }
1533
1534         /* Ether - ARP off */
1535         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1536                 (p_rt->params.n_arp_entries == 0)) {
1537                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1538                 uint64_t macaddr_dst;
1539                 uint64_t ethertype = ETHER_TYPE_IPv4;
1540
1541                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1542                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1543
1544                 entry_arp0.slab[0] =
1545                         rte_bswap64((macaddr_src << 16) | ethertype);
1546                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1547
1548                 entry_arp0.slab[1] = rte_bswap64(macaddr_dst);
1549                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1550
1551                 entry_arp0.data_offset = entry_arp0.slab_offset[1] + 2
1552                         - sizeof(struct rte_mbuf);
1553                 entry_arp0.ether_l2_length = 14;
1554         }
1555
1556         /* Ether - ARP on */
1557         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1558                 p_rt->params.n_arp_entries) {
1559                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1560                 uint64_t ethertype = ETHER_TYPE_IPv4;
1561
1562                 entry_arp1.slab[0] = rte_bswap64((macaddr_src << 16) |
1563                         ethertype);
1564                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1565
1566                 entry_arp1.data_offset = entry_arp1.slab_offset[0] - 6
1567                         - sizeof(struct rte_mbuf);
1568                 entry_arp1.ether_l2_length = 14;
1569         }
1570
1571         /* Ether QinQ - ARP off */
1572         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1573                 (p_rt->params.n_arp_entries == 0)) {
1574                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1575                 uint64_t macaddr_dst;
1576                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1577                 uint64_t ethertype_vlan = 0x8100;
1578                 uint64_t ethertype_qinq = 0x9100;
1579                 uint64_t svlan = req->data.l2.qinq.svlan;
1580                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1581
1582                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1583                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1584
1585                 entry_arp0.slab[0] = rte_bswap64((svlan << 48) |
1586                         (ethertype_vlan << 32) |
1587                         (cvlan << 16) |
1588                         ethertype_ipv4);
1589                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1590
1591                 entry_arp0.slab[1] = rte_bswap64((macaddr_src << 16) |
1592                         ethertype_qinq);
1593                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1594
1595                 entry_arp0.slab[2] = rte_bswap64(macaddr_dst);
1596                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset - 3 * 8;
1597
1598                 entry_arp0.data_offset = entry_arp0.slab_offset[2] + 2
1599                         - sizeof(struct rte_mbuf);
1600                 entry_arp0.ether_l2_length = 22;
1601         }
1602
1603         /* Ether QinQ - ARP on */
1604         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1605                 p_rt->params.n_arp_entries) {
1606                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1607                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1608                 uint64_t ethertype_vlan = 0x8100;
1609                 uint64_t ethertype_qinq = 0x9100;
1610                 uint64_t svlan = req->data.l2.qinq.svlan;
1611                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1612
1613                 entry_arp1.slab[0] = rte_bswap64((svlan << 48) |
1614                         (ethertype_vlan << 32) |
1615                         (cvlan << 16) |
1616                         ethertype_ipv4);
1617                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1618
1619                 entry_arp1.slab[1] = rte_bswap64((macaddr_src << 16) |
1620                         ethertype_qinq);
1621                 entry_arp1.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1622
1623                 entry_arp1.data_offset = entry_arp1.slab_offset[1] - 6
1624                         - sizeof(struct rte_mbuf);
1625                 entry_arp1.ether_l2_length = 22;
1626         }
1627
1628         /* Ether MPLS - ARP off */
1629         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1630                 (p_rt->params.n_arp_entries == 0)) {
1631                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1632                 uint64_t macaddr_dst;
1633                 uint64_t ethertype_mpls = 0x8847;
1634
1635                 uint64_t label0 = req->data.l2.mpls.labels[0];
1636                 uint64_t label1 = req->data.l2.mpls.labels[1];
1637                 uint64_t label2 = req->data.l2.mpls.labels[2];
1638                 uint64_t label3 = req->data.l2.mpls.labels[3];
1639                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1640
1641                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1642                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1643
1644                 switch (n_labels) {
1645                 case 1:
1646                         entry_arp0.slab[0] = 0;
1647                         entry_arp0.slab_offset[0] =
1648                                 p_rt->params.ip_hdr_offset - 8;
1649
1650                         entry_arp0.slab[1] = rte_bswap64(
1651                                 MPLS_LABEL(label0, 0, 1, 0));
1652                         entry_arp0.slab_offset[1] =
1653                                 p_rt->params.ip_hdr_offset - 8;
1654                         break;
1655
1656                 case 2:
1657                         entry_arp0.slab[0] = 0;
1658                         entry_arp0.slab_offset[0] =
1659                                 p_rt->params.ip_hdr_offset - 8;
1660
1661                         entry_arp0.slab[1] = rte_bswap64(
1662                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1663                                 MPLS_LABEL(label1, 0, 1, 0));
1664                         entry_arp0.slab_offset[1] =
1665                                 p_rt->params.ip_hdr_offset - 8;
1666                         break;
1667
1668                 case 3:
1669                         entry_arp0.slab[0] = rte_bswap64(
1670                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1671                                 MPLS_LABEL(label2, 0, 1, 0));
1672                         entry_arp0.slab_offset[0] =
1673                                 p_rt->params.ip_hdr_offset - 8;
1674
1675                         entry_arp0.slab[1] = rte_bswap64(
1676                                 MPLS_LABEL(label0, 0, 0, 0));
1677                         entry_arp0.slab_offset[1] =
1678                                 p_rt->params.ip_hdr_offset - 2 * 8;
1679                         break;
1680
1681                 case 4:
1682                         entry_arp0.slab[0] = rte_bswap64(
1683                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1684                                 MPLS_LABEL(label3, 0, 1, 0));
1685                         entry_arp0.slab_offset[0] =
1686                                 p_rt->params.ip_hdr_offset - 8;
1687
1688                         entry_arp0.slab[1] = rte_bswap64(
1689                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1690                                 MPLS_LABEL(label1, 0, 0, 0));
1691                         entry_arp0.slab_offset[1] =
1692                                 p_rt->params.ip_hdr_offset - 2 * 8;
1693                         break;
1694
1695                 default:
1696                         rsp->status = -1;
1697                         return rsp;
1698                 }
1699
1700                 entry_arp0.slab[2] = rte_bswap64((macaddr_src << 16) |
1701                         ethertype_mpls);
1702                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset -
1703                         (n_labels * 4 + 8);
1704
1705                 entry_arp0.slab[3] = rte_bswap64(macaddr_dst);
1706                 entry_arp0.slab_offset[3] = p_rt->params.ip_hdr_offset -
1707                         (n_labels * 4 + 2 * 8);
1708
1709                 entry_arp0.data_offset = entry_arp0.slab_offset[3] + 2
1710                         - sizeof(struct rte_mbuf);
1711                 entry_arp0.ether_l2_length = n_labels * 4 + 14;
1712         }
1713
1714         /* Ether MPLS - ARP on */
1715         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1716                 p_rt->params.n_arp_entries) {
1717                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1718                 uint64_t ethertype_mpls = 0x8847;
1719
1720                 uint64_t label0 = req->data.l2.mpls.labels[0];
1721                 uint64_t label1 = req->data.l2.mpls.labels[1];
1722                 uint64_t label2 = req->data.l2.mpls.labels[2];
1723                 uint64_t label3 = req->data.l2.mpls.labels[3];
1724                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1725
1726                 switch (n_labels) {
1727                 case 1:
1728                         entry_arp1.slab[0] = 0;
1729                         entry_arp1.slab_offset[0] =
1730                                 p_rt->params.ip_hdr_offset - 8;
1731
1732                         entry_arp1.slab[1] = rte_bswap64(
1733                                 MPLS_LABEL(label0, 0, 1, 0));
1734                         entry_arp1.slab_offset[1] =
1735                                 p_rt->params.ip_hdr_offset - 8;
1736                         break;
1737
1738                 case 2:
1739                         entry_arp1.slab[0] = 0;
1740                         entry_arp1.slab_offset[0] =
1741                                 p_rt->params.ip_hdr_offset - 8;
1742
1743                         entry_arp1.slab[1] = rte_bswap64(
1744                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1745                                 MPLS_LABEL(label1, 0, 1, 0));
1746                         entry_arp1.slab_offset[1] =
1747                                 p_rt->params.ip_hdr_offset - 8;
1748                         break;
1749
1750                 case 3:
1751                         entry_arp1.slab[0] = rte_bswap64(
1752                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1753                                 MPLS_LABEL(label2, 0, 1, 0));
1754                         entry_arp1.slab_offset[0] =
1755                                 p_rt->params.ip_hdr_offset - 8;
1756
1757                         entry_arp1.slab[1] = rte_bswap64(
1758                                 MPLS_LABEL(label0, 0, 0, 0));
1759                         entry_arp1.slab_offset[1] =
1760                                 p_rt->params.ip_hdr_offset - 2 * 8;
1761                         break;
1762
1763                 case 4:
1764                         entry_arp1.slab[0] = rte_bswap64(
1765                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1766                                 MPLS_LABEL(label3, 0, 1, 0));
1767                         entry_arp1.slab_offset[0] =
1768                                 p_rt->params.ip_hdr_offset - 8;
1769
1770                         entry_arp1.slab[1] = rte_bswap64(
1771                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1772                                 MPLS_LABEL(label1, 0, 0, 0));
1773                         entry_arp1.slab_offset[1] =
1774                                 p_rt->params.ip_hdr_offset - 2 * 8;
1775                         break;
1776
1777                 default:
1778                         rsp->status = -1;
1779                         return rsp;
1780                 }
1781
1782                 entry_arp1.slab[2] = rte_bswap64((macaddr_src << 16) |
1783                         ethertype_mpls);
1784                 entry_arp1.slab_offset[2] = p_rt->params.ip_hdr_offset -
1785                         (n_labels * 4 + 8);
1786
1787                 entry_arp1.data_offset = entry_arp1.slab_offset[2] - 6
1788                         - sizeof(struct rte_mbuf);
1789                 entry_arp1.ether_l2_length = n_labels * 4 + 14;
1790         }
1791
1792         rsp->status = rte_pipeline_table_entry_add(p->p,
1793                 p->table_id[0],
1794                 &key,
1795                 entry,
1796                 &rsp->key_found,
1797                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1798
1799         return rsp;
1800 }
1801
1802 void *
1803 pipeline_routing_msg_req_route_del_handler(struct pipeline *p, void *msg)
1804 {
1805         struct pipeline_routing_route_delete_msg_req *req = msg;
1806         struct pipeline_routing_route_delete_msg_rsp *rsp = msg;
1807
1808         struct rte_table_lpm_key key = {
1809                 .ip = req->key.key.ipv4.ip,
1810                 .depth = req->key.key.ipv4.depth,
1811         };
1812
1813         if (req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) {
1814                 rsp->status = -1;
1815                 return rsp;
1816         }
1817
1818         rsp->status = rte_pipeline_table_entry_delete(p->p,
1819                 p->table_id[0],
1820                 &key,
1821                 &rsp->key_found,
1822                 NULL);
1823
1824         return rsp;
1825 }
1826
1827 void *
1828 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
1829         void *msg)
1830 {
1831         struct pipeline_routing_route_add_default_msg_req *req = msg;
1832         struct pipeline_routing_route_add_default_msg_rsp *rsp = msg;
1833
1834         struct routing_table_entry default_entry = {
1835                 .head = {
1836                         .action = RTE_PIPELINE_ACTION_PORT,
1837                         {.port_id = p->port_out_id[req->port_id]},
1838                 },
1839
1840                 .flags = 0,
1841                 .port_id = 0,
1842                 .ip = 0,
1843         };
1844
1845         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1846                 p->table_id[0],
1847                 (struct rte_pipeline_table_entry *) &default_entry,
1848                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1849
1850         return rsp;
1851 }
1852
1853 void *
1854 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
1855         void *msg)
1856 {
1857         struct pipeline_routing_route_delete_default_msg_rsp *rsp = msg;
1858
1859         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1860                 p->table_id[0],
1861                 NULL);
1862
1863         return rsp;
1864 }
1865
1866 void *
1867 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p, void *msg)
1868 {
1869         struct pipeline_routing_arp_add_msg_req *req = msg;
1870         struct pipeline_routing_arp_add_msg_rsp *rsp = msg;
1871
1872         struct pipeline_routing_arp_key_ipv4 key = {
1873                 .port_id = req->key.key.ipv4.port_id,
1874                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1875         };
1876
1877         struct arp_table_entry entry = {
1878                 .head = {
1879                         .action = RTE_PIPELINE_ACTION_PORT,
1880                         {.port_id = p->port_out_id[req->key.key.ipv4.port_id]},
1881                 },
1882
1883                 .macaddr = 0, /* set below */
1884         };
1885
1886         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1887                 rsp->status = -1;
1888                 return rsp;
1889         }
1890
1891         entry.macaddr = *((uint64_t *)&(req->macaddr));
1892         entry.macaddr = entry.macaddr << 16;
1893
1894         rsp->status = rte_pipeline_table_entry_add(p->p,
1895                 p->table_id[1],
1896                 &key,
1897                 (struct rte_pipeline_table_entry *) &entry,
1898                 &rsp->key_found,
1899                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1900
1901         return rsp;
1902 }
1903
1904 void *
1905 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p, void *msg)
1906 {
1907         struct pipeline_routing_arp_delete_msg_req *req = msg;
1908         struct pipeline_routing_arp_delete_msg_rsp *rsp = msg;
1909
1910         struct pipeline_routing_arp_key_ipv4 key = {
1911                 .port_id = req->key.key.ipv4.port_id,
1912                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1913         };
1914
1915         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1916                 rsp->status = -1;
1917                 return rsp;
1918         }
1919
1920         rsp->status = rte_pipeline_table_entry_delete(p->p,
1921                 p->table_id[1],
1922                 &key,
1923                 &rsp->key_found,
1924                 NULL);
1925
1926         return rsp;
1927 }
1928
1929 void *
1930 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p, void *msg)
1931 {
1932         struct pipeline_routing_arp_add_default_msg_req *req = msg;
1933         struct pipeline_routing_arp_add_default_msg_rsp *rsp = msg;
1934
1935         struct arp_table_entry default_entry = {
1936                 .head = {
1937                         .action = RTE_PIPELINE_ACTION_PORT,
1938                         {.port_id = p->port_out_id[req->port_id]},
1939                 },
1940
1941                 .macaddr = 0,
1942         };
1943
1944         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1945                 p->table_id[1],
1946                 (struct rte_pipeline_table_entry *) &default_entry,
1947                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1948
1949         return rsp;
1950 }
1951
1952 void *
1953 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p, void *msg)
1954 {
1955         struct pipeline_routing_arp_delete_default_msg_rsp *rsp = msg;
1956
1957         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1958                 p->table_id[1],
1959                 NULL);
1960
1961         return rsp;
1962 }
1963
1964 struct pipeline_be_ops pipeline_routing_be_ops = {
1965         .f_init = pipeline_routing_init,
1966         .f_free = pipeline_routing_free,
1967         .f_run = NULL,
1968         .f_timer = pipeline_routing_timer,
1969         .f_track = pipeline_routing_track,
1970 };