examples/ip_pipeline/pipeline/pipeline_routing_be.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <unistd.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_malloc.h>
  42 #include <rte_ip.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_table_lpm.h>
  45 #include <rte_table_hash.h>
  46 #include <rte_pipeline.h>
  47
  48 #include "pipeline_routing_be.h"
  49 #include "pipeline_actions_common.h"
  50 #include "parser.h"
  51 #include "hash_func.h"
  52
  53 #define MPLS_LABEL(label, exp, s, ttl)                                  \
  54         (((((uint64_t) (label)) & 0xFFFFFLLU) << 12) |          \
  55         ((((uint64_t) (exp)) & 0x7LLU) << 9) |                          \
  56         ((((uint64_t) (s)) & 0x1LLU) << 8) |                            \
  57         (((uint64_t) (ttl)) & 0xFFLU))
  58
  59 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe,         \
  60         traffic_class, queue, color)                            \
  61         ((((uint64_t) (queue)) & 0x3) |                \
  62         ((((uint64_t) (traffic_class)) & 0x3) << 2) |  \
  63         ((((uint64_t) (color)) & 0x3) << 4) |          \
  64         ((((uint64_t) (subport)) & 0xFFFF) << 16) |    \
  65         ((((uint64_t) (pipe)) & 0xFFFFFFFF) << 32))
  66
  67
  68 /* Network Byte Order (NBO) */
  69 #define SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr, ethertype)       \
  70         (((uint64_t) macaddr) | (((uint64_t) rte_cpu_to_be_16(ethertype)) << 48))
  71
  72 #ifndef PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s
  73 #define PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s 256
  74 #endif
  75
  76 struct pipeline_routing {
  77         struct pipeline p;
  78         struct pipeline_routing_params params;
  79         pipeline_msg_req_handler custom_handlers[PIPELINE_ROUTING_MSG_REQS];
  80         uint64_t macaddr[PIPELINE_MAX_PORT_OUT];
  81 } __rte_cache_aligned;
  82
  83 /*
  84  * Message handlers
  85  */
  86 static void *
  87 pipeline_routing_msg_req_custom_handler(struct pipeline *p, void *msg);
  88
  89 static pipeline_msg_req_handler handlers[] = {
  90         [PIPELINE_MSG_REQ_PING] =
  91                 pipeline_msg_req_ping_handler,
  92         [PIPELINE_MSG_REQ_STATS_PORT_IN] =
  93                 pipeline_msg_req_stats_port_in_handler,
  94         [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
  95                 pipeline_msg_req_stats_port_out_handler,
  96         [PIPELINE_MSG_REQ_STATS_TABLE] =
  97                 pipeline_msg_req_stats_table_handler,
  98         [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
  99                 pipeline_msg_req_port_in_enable_handler,
 100         [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
 101                 pipeline_msg_req_port_in_disable_handler,
 102         [PIPELINE_MSG_REQ_CUSTOM] =
 103                 pipeline_routing_msg_req_custom_handler,
 104 };
 105
 106 static void *
 107 pipeline_routing_msg_req_route_add_handler(struct pipeline *p,
 108         void *msg);
 109
 110 static void *
 111 pipeline_routing_msg_req_route_del_handler(struct pipeline *p,
 112         void *msg);
 113
 114 static void *
 115 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
 116         void *msg);
 117
 118 static void *
 119 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
 120         void *msg);
 121
 122 static void *
 123 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p,
 124         void *msg);
 125
 126 static void *
 127 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p,
 128         void *msg);
 129
 130 static void *
 131 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p,
 132         void *msg);
 133
 134 static void *
 135 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p,
 136         void *msg);
 137
 138 static void *
 139 pipeline_routing_msg_req_set_macaddr_handler(struct pipeline *p,
 140         void *msg);
 141
 142 static pipeline_msg_req_handler custom_handlers[] = {
 143         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD] =
 144                 pipeline_routing_msg_req_route_add_handler,
 145         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL] =
 146                 pipeline_routing_msg_req_route_del_handler,
 147         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD_DEFAULT] =
 148                 pipeline_routing_msg_req_route_add_default_handler,
 149         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL_DEFAULT] =
 150                 pipeline_routing_msg_req_route_del_default_handler,
 151         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD] =
 152                 pipeline_routing_msg_req_arp_add_handler,
 153         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL] =
 154                 pipeline_routing_msg_req_arp_del_handler,
 155         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD_DEFAULT] =
 156                 pipeline_routing_msg_req_arp_add_default_handler,
 157         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL_DEFAULT] =
 158                 pipeline_routing_msg_req_arp_del_default_handler,
 159         [PIPELINE_ROUTING_MSG_REQ_SET_MACADDR] =
 160                 pipeline_routing_msg_req_set_macaddr_handler,
 161 };
 162
 163 /*
 164  * Routing table
 165  */
 166 struct routing_table_entry {
 167         struct rte_pipeline_table_entry head;
 168         uint32_t flags;
 169         uint32_t port_id; /* Output port ID */
 170         uint32_t ip; /* Next hop IP address (only valid for remote routes) */
 171
 172         /* ether_l2 */
 173         uint16_t data_offset;
 174         uint16_t ether_l2_length;
 175         uint64_t slab[4];
 176         uint16_t slab_offset[4];
 177 };
 178
 179 struct layout {
 180         uint16_t a;
 181         uint32_t b;
 182         uint16_t c;
 183 } __attribute__((__packed__));
 184
 185 #define MACADDR_DST_WRITE(slab_ptr, slab)                       \
 186 {                                                               \
 187         struct layout *dst = (struct layout *) (slab_ptr);      \
 188         struct layout *src = (struct layout *) &(slab);         \
 189                                                                 \
 190         dst->b = src->b;                                        \
 191         dst->c = src->c;                                        \
 192 }
 193
 194 static inline __attribute__((always_inline)) void
 195 pkt_work_routing(
 196         struct rte_mbuf *pkt,
 197         struct rte_pipeline_table_entry *table_entry,
 198         void *arg,
 199         int arp,
 200         int qinq,
 201         int qinq_sched,
 202         int mpls,
 203         int mpls_color_mark)
 204 {
 205         struct pipeline_routing *p_rt = arg;
 206
 207         struct routing_table_entry *entry =
 208                 (struct routing_table_entry *) table_entry;
 209
 210         struct ipv4_hdr *ip = (struct ipv4_hdr *)
 211                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.ip_hdr_offset);
 212
 213         enum rte_meter_color pkt_color = (enum rte_meter_color)
 214                 RTE_MBUF_METADATA_UINT32(pkt, p_rt->params.color_offset);
 215
 216         struct pipeline_routing_arp_key_ipv4 *arp_key =
 217                 (struct pipeline_routing_arp_key_ipv4 *)
 218                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.arp_key_offset);
 219
 220         uint64_t *slab0_ptr, *slab1_ptr, *slab2_ptr, *slab3_ptr, sched;
 221         uint32_t ip_da, nh_ip, port_id;
 222         uint16_t total_length, data_offset, ether_l2_length;
 223
 224         /* Read */
 225         total_length = rte_bswap16(ip->total_length);
 226         ip_da = ip->dst_addr;
 227         data_offset = entry->data_offset;
 228         ether_l2_length = entry->ether_l2_length;
 229         slab0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[0]);
 230         slab1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[1]);
 231         slab2_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[2]);
 232         slab3_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[3]);
 233
 234         if (arp) {
 235                 port_id = entry->port_id;
 236                 nh_ip = entry->ip;
 237                 if (entry->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 238                         nh_ip = ip_da;
 239         }
 240
 241         /* Compute */
 242         total_length += ether_l2_length;
 243
 244         if (qinq && qinq_sched) {
 245                 uint32_t dscp = ip->type_of_service >> 2;
 246                 uint32_t svlan, cvlan, tc, tc_q;
 247
 248                 if (qinq_sched == 1) {
 249                         uint64_t slab_qinq = rte_bswap64(entry->slab[0]);
 250
 251                         svlan = (slab_qinq >> 48) & 0xFFF;
 252                         cvlan = (slab_qinq >> 16) & 0xFFF;
 253                         tc = (dscp >> 2) & 0x3;
 254                         tc_q = dscp & 0x3;
 255                 } else {
 256                         uint32_t ip_src = rte_bswap32(ip->src_addr);
 257
 258                         svlan = 0;
 259                         cvlan = (ip_src >> 16) & 0xFFF;
 260                         tc = (ip_src >> 2) & 0x3;
 261                         tc_q = ip_src & 0x3;
 262                 }
 263                 sched = RTE_SCHED_PORT_HIERARCHY(svlan,
 264                         cvlan,
 265                         tc,
 266                         tc_q,
 267                         e_RTE_METER_GREEN);
 268         }
 269
 270         /* Write */
 271         pkt->data_off = data_offset;
 272         pkt->data_len = total_length;
 273         pkt->pkt_len = total_length;
 274
 275         if ((qinq == 0) && (mpls == 0)) {
 276                 *slab0_ptr = entry->slab[0];
 277
 278                 if (arp == 0)
 279                         MACADDR_DST_WRITE(slab1_ptr, entry->slab[1]);
 280         }
 281
 282         if (qinq) {
 283                 *slab0_ptr = entry->slab[0];
 284                 *slab1_ptr = entry->slab[1];
 285
 286                 if (arp == 0)
 287                         MACADDR_DST_WRITE(slab2_ptr, entry->slab[2]);
 288
 289                 if (qinq_sched) {
 290                         pkt->hash.sched.lo = sched & 0xFFFFFFFF;
 291                         pkt->hash.sched.hi = sched >> 32;
 292                 }
 293         }
 294
 295         if (mpls) {
 296                 if (mpls_color_mark) {
 297                         uint64_t mpls_exp = rte_bswap64(
 298                                 (MPLS_LABEL(0, pkt_color, 0, 0) << 32) |
 299                                 MPLS_LABEL(0, pkt_color, 0, 0));
 300
 301                         *slab0_ptr = entry->slab[0] | mpls_exp;
 302                         *slab1_ptr = entry->slab[1] | mpls_exp;
 303                         *slab2_ptr = entry->slab[2];
 304                 } else {
 305                         *slab0_ptr = entry->slab[0];
 306                         *slab1_ptr = entry->slab[1];
 307                         *slab2_ptr = entry->slab[2];
 308                 }
 309
 310                 if (arp == 0)
 311                         MACADDR_DST_WRITE(slab3_ptr, entry->slab[3]);
 312         }
 313
 314         if (arp) {
 315                 arp_key->port_id = port_id;
 316                 arp_key->ip = nh_ip;
 317         }
 318 }
 319
 320 static inline __attribute__((always_inline)) void
 321 pkt4_work_routing(
 322         struct rte_mbuf **pkts,
 323         struct rte_pipeline_table_entry **table_entries,
 324         void *arg,
 325         int arp,
 326         int qinq,
 327         int qinq_sched,
 328         int mpls,
 329         int mpls_color_mark)
 330 {
 331         struct pipeline_routing *p_rt = arg;
 332
 333         struct routing_table_entry *entry0 =
 334                 (struct routing_table_entry *) table_entries[0];
 335         struct routing_table_entry *entry1 =
 336                 (struct routing_table_entry *) table_entries[1];
 337         struct routing_table_entry *entry2 =
 338                 (struct routing_table_entry *) table_entries[2];
 339         struct routing_table_entry *entry3 =
 340                 (struct routing_table_entry *) table_entries[3];
 341
 342         struct ipv4_hdr *ip0 = (struct ipv4_hdr *)
 343                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 344                         p_rt->params.ip_hdr_offset);
 345         struct ipv4_hdr *ip1 = (struct ipv4_hdr *)
 346                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 347                         p_rt->params.ip_hdr_offset);
 348         struct ipv4_hdr *ip2 = (struct ipv4_hdr *)
 349                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 350                         p_rt->params.ip_hdr_offset);
 351         struct ipv4_hdr *ip3 = (struct ipv4_hdr *)
 352                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 353                         p_rt->params.ip_hdr_offset);
 354
 355         enum rte_meter_color pkt0_color = (enum rte_meter_color)
 356                 RTE_MBUF_METADATA_UINT32(pkts[0], p_rt->params.color_offset);
 357         enum rte_meter_color pkt1_color = (enum rte_meter_color)
 358                 RTE_MBUF_METADATA_UINT32(pkts[1], p_rt->params.color_offset);
 359         enum rte_meter_color pkt2_color = (enum rte_meter_color)
 360                 RTE_MBUF_METADATA_UINT32(pkts[2], p_rt->params.color_offset);
 361         enum rte_meter_color pkt3_color = (enum rte_meter_color)
 362                 RTE_MBUF_METADATA_UINT32(pkts[3], p_rt->params.color_offset);
 363
 364         struct pipeline_routing_arp_key_ipv4 *arp_key0 =
 365                 (struct pipeline_routing_arp_key_ipv4 *)
 366                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 367                         p_rt->params.arp_key_offset);
 368         struct pipeline_routing_arp_key_ipv4 *arp_key1 =
 369                 (struct pipeline_routing_arp_key_ipv4 *)
 370                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 371                         p_rt->params.arp_key_offset);
 372         struct pipeline_routing_arp_key_ipv4 *arp_key2 =
 373                 (struct pipeline_routing_arp_key_ipv4 *)
 374                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 375                         p_rt->params.arp_key_offset);
 376         struct pipeline_routing_arp_key_ipv4 *arp_key3 =
 377                 (struct pipeline_routing_arp_key_ipv4 *)
 378                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 379                         p_rt->params.arp_key_offset);
 380
 381         uint64_t *slab0_ptr0, *slab1_ptr0, *slab2_ptr0, *slab3_ptr0;
 382         uint64_t *slab0_ptr1, *slab1_ptr1, *slab2_ptr1, *slab3_ptr1;
 383         uint64_t *slab0_ptr2, *slab1_ptr2, *slab2_ptr2, *slab3_ptr2;
 384         uint64_t *slab0_ptr3, *slab1_ptr3, *slab2_ptr3, *slab3_ptr3;
 385         uint64_t sched0, sched1, sched2, sched3;
 386
 387         uint32_t ip_da0, nh_ip0, port_id0;
 388         uint32_t ip_da1, nh_ip1, port_id1;
 389         uint32_t ip_da2, nh_ip2, port_id2;
 390         uint32_t ip_da3, nh_ip3, port_id3;
 391
 392         uint16_t total_length0, data_offset0, ether_l2_length0;
 393         uint16_t total_length1, data_offset1, ether_l2_length1;
 394         uint16_t total_length2, data_offset2, ether_l2_length2;
 395         uint16_t total_length3, data_offset3, ether_l2_length3;
 396
 397         /* Read */
 398         total_length0 = rte_bswap16(ip0->total_length);
 399         total_length1 = rte_bswap16(ip1->total_length);
 400         total_length2 = rte_bswap16(ip2->total_length);
 401         total_length3 = rte_bswap16(ip3->total_length);
 402
 403         ip_da0 = ip0->dst_addr;
 404         ip_da1 = ip1->dst_addr;
 405         ip_da2 = ip2->dst_addr;
 406         ip_da3 = ip3->dst_addr;
 407
 408         data_offset0 = entry0->data_offset;
 409         data_offset1 = entry1->data_offset;
 410         data_offset2 = entry2->data_offset;
 411         data_offset3 = entry3->data_offset;
 412
 413         ether_l2_length0 = entry0->ether_l2_length;
 414         ether_l2_length1 = entry1->ether_l2_length;
 415         ether_l2_length2 = entry2->ether_l2_length;
 416         ether_l2_length3 = entry3->ether_l2_length;
 417
 418         slab0_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 419                 entry0->slab_offset[0]);
 420         slab1_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 421                 entry0->slab_offset[1]);
 422         slab2_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 423                 entry0->slab_offset[2]);
 424         slab3_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 425                 entry0->slab_offset[3]);
 426
 427         slab0_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 428                 entry1->slab_offset[0]);
 429         slab1_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 430                 entry1->slab_offset[1]);
 431         slab2_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 432                 entry1->slab_offset[2]);
 433         slab3_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 434                 entry1->slab_offset[3]);
 435
 436         slab0_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 437                 entry2->slab_offset[0]);
 438         slab1_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 439                 entry2->slab_offset[1]);
 440         slab2_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 441                 entry2->slab_offset[2]);
 442         slab3_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 443                 entry2->slab_offset[3]);
 444
 445         slab0_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 446                 entry3->slab_offset[0]);
 447         slab1_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 448                 entry3->slab_offset[1]);
 449         slab2_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 450                 entry3->slab_offset[2]);
 451         slab3_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 452                 entry3->slab_offset[3]);
 453
 454         if (arp) {
 455                 port_id0 = entry0->port_id;
 456                 nh_ip0 = entry0->ip;
 457                 if (entry0->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 458                         nh_ip0 = ip_da0;
 459
 460                 port_id1 = entry1->port_id;
 461                 nh_ip1 = entry1->ip;
 462                 if (entry1->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 463                         nh_ip1 = ip_da1;
 464
 465                 port_id2 = entry2->port_id;
 466                 nh_ip2 = entry2->ip;
 467                 if (entry2->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 468                         nh_ip2 = ip_da2;
 469
 470                 port_id3 = entry3->port_id;
 471                 nh_ip3 = entry3->ip;
 472                 if (entry3->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 473                         nh_ip3 = ip_da3;
 474         }
 475
 476         /* Compute */
 477         total_length0 += ether_l2_length0;
 478         total_length1 += ether_l2_length1;
 479         total_length2 += ether_l2_length2;
 480         total_length3 += ether_l2_length3;
 481
 482         if (qinq && qinq_sched) {
 483                 uint32_t dscp0 = ip0->type_of_service >> 2;
 484                 uint32_t dscp1 = ip1->type_of_service >> 2;
 485                 uint32_t dscp2 = ip2->type_of_service >> 2;
 486                 uint32_t dscp3 = ip3->type_of_service >> 2;
 487                 uint32_t svlan0, cvlan0, tc0, tc_q0;
 488                 uint32_t svlan1, cvlan1, tc1, tc_q1;
 489                 uint32_t svlan2, cvlan2, tc2, tc_q2;
 490                 uint32_t svlan3, cvlan3, tc3, tc_q3;
 491
 492                 if (qinq_sched == 1) {
 493                         uint64_t slab_qinq0 = rte_bswap64(entry0->slab[0]);
 494                         uint64_t slab_qinq1 = rte_bswap64(entry1->slab[0]);
 495                         uint64_t slab_qinq2 = rte_bswap64(entry2->slab[0]);
 496                         uint64_t slab_qinq3 = rte_bswap64(entry3->slab[0]);
 497
 498                         svlan0 = (slab_qinq0 >> 48) & 0xFFF;
 499                         svlan1 = (slab_qinq1 >> 48) & 0xFFF;
 500                         svlan2 = (slab_qinq2 >> 48) & 0xFFF;
 501                         svlan3 = (slab_qinq3 >> 48) & 0xFFF;
 502
 503                         cvlan0 = (slab_qinq0 >> 16) & 0xFFF;
 504                         cvlan1 = (slab_qinq1 >> 16) & 0xFFF;
 505                         cvlan2 = (slab_qinq2 >> 16) & 0xFFF;
 506                         cvlan3 = (slab_qinq3 >> 16) & 0xFFF;
 507
 508                         tc0 = (dscp0 >> 2) & 0x3;
 509                         tc1 = (dscp1 >> 2) & 0x3;
 510                         tc2 = (dscp2 >> 2) & 0x3;
 511                         tc3 = (dscp3 >> 2) & 0x3;
 512
 513                         tc_q0 = dscp0 & 0x3;
 514                         tc_q1 = dscp1 & 0x3;
 515                         tc_q2 = dscp2 & 0x3;
 516                         tc_q3 = dscp3 & 0x3;
 517                 } else {
 518                         uint32_t ip_src0 = rte_bswap32(ip0->src_addr);
 519                         uint32_t ip_src1 = rte_bswap32(ip1->src_addr);
 520                         uint32_t ip_src2 = rte_bswap32(ip2->src_addr);
 521                         uint32_t ip_src3 = rte_bswap32(ip3->src_addr);
 522
 523                         svlan0 = 0;
 524                         svlan1 = 0;
 525                         svlan2 = 0;
 526                         svlan3 = 0;
 527
 528                         cvlan0 = (ip_src0 >> 16) & 0xFFF;
 529                         cvlan1 = (ip_src1 >> 16) & 0xFFF;
 530                         cvlan2 = (ip_src2 >> 16) & 0xFFF;
 531                         cvlan3 = (ip_src3 >> 16) & 0xFFF;
 532
 533                         tc0 = (ip_src0 >> 2) & 0x3;
 534                         tc1 = (ip_src1 >> 2) & 0x3;
 535                         tc2 = (ip_src2 >> 2) & 0x3;
 536                         tc3 = (ip_src3 >> 2) & 0x3;
 537
 538                         tc_q0 = ip_src0 & 0x3;
 539                         tc_q1 = ip_src1 & 0x3;
 540                         tc_q2 = ip_src2 & 0x3;
 541                         tc_q3 = ip_src3 & 0x3;
 542                 }
 543
 544                 sched0 = RTE_SCHED_PORT_HIERARCHY(svlan0,
 545                         cvlan0,
 546                         tc0,
 547                         tc_q0,
 548                         e_RTE_METER_GREEN);
 549                 sched1 = RTE_SCHED_PORT_HIERARCHY(svlan1,
 550                         cvlan1,
 551                         tc1,
 552                         tc_q1,
 553                         e_RTE_METER_GREEN);
 554                 sched2 = RTE_SCHED_PORT_HIERARCHY(svlan2,
 555                         cvlan2,
 556                         tc2,
 557                         tc_q2,
 558                         e_RTE_METER_GREEN);
 559                 sched3 = RTE_SCHED_PORT_HIERARCHY(svlan3,
 560                         cvlan3,
 561                         tc3,
 562                         tc_q3,
 563                         e_RTE_METER_GREEN);
 564
 565         }
 566
 567         /* Write */
 568         pkts[0]->data_off = data_offset0;
 569         pkts[1]->data_off = data_offset1;
 570         pkts[2]->data_off = data_offset2;
 571         pkts[3]->data_off = data_offset3;
 572
 573         pkts[0]->data_len = total_length0;
 574         pkts[1]->data_len = total_length1;
 575         pkts[2]->data_len = total_length2;
 576         pkts[3]->data_len = total_length3;
 577
 578         pkts[0]->pkt_len = total_length0;
 579         pkts[1]->pkt_len = total_length1;
 580         pkts[2]->pkt_len = total_length2;
 581         pkts[3]->pkt_len = total_length3;
 582
 583         if ((qinq == 0) && (mpls == 0)) {
 584                 *slab0_ptr0 = entry0->slab[0];
 585                 *slab0_ptr1 = entry1->slab[0];
 586                 *slab0_ptr2 = entry2->slab[0];
 587                 *slab0_ptr3 = entry3->slab[0];
 588
 589                 if (arp == 0) {
 590                         MACADDR_DST_WRITE(slab1_ptr0, entry0->slab[1]);
 591                         MACADDR_DST_WRITE(slab1_ptr1, entry1->slab[1]);
 592                         MACADDR_DST_WRITE(slab1_ptr2, entry2->slab[1]);
 593                         MACADDR_DST_WRITE(slab1_ptr3, entry3->slab[1]);
 594                 }
 595         }
 596
 597         if (qinq) {
 598                 *slab0_ptr0 = entry0->slab[0];
 599                 *slab0_ptr1 = entry1->slab[0];
 600                 *slab0_ptr2 = entry2->slab[0];
 601                 *slab0_ptr3 = entry3->slab[0];
 602
 603                 *slab1_ptr0 = entry0->slab[1];
 604                 *slab1_ptr1 = entry1->slab[1];
 605                 *slab1_ptr2 = entry2->slab[1];
 606                 *slab1_ptr3 = entry3->slab[1];
 607
 608                 if (arp == 0) {
 609                         MACADDR_DST_WRITE(slab2_ptr0, entry0->slab[2]);
 610                         MACADDR_DST_WRITE(slab2_ptr1, entry1->slab[2]);
 611                         MACADDR_DST_WRITE(slab2_ptr2, entry2->slab[2]);
 612                         MACADDR_DST_WRITE(slab2_ptr3, entry3->slab[2]);
 613                 }
 614
 615                 if (qinq_sched) {
 616                         pkts[0]->hash.sched.lo = sched0 & 0xFFFFFFFF;
 617                         pkts[0]->hash.sched.hi = sched0 >> 32;
 618                         pkts[1]->hash.sched.lo = sched1 & 0xFFFFFFFF;
 619                         pkts[1]->hash.sched.hi = sched1 >> 32;
 620                         pkts[2]->hash.sched.lo = sched2 & 0xFFFFFFFF;
 621                         pkts[2]->hash.sched.hi = sched2 >> 32;
 622                         pkts[3]->hash.sched.lo = sched3 & 0xFFFFFFFF;
 623                         pkts[3]->hash.sched.hi = sched3 >> 32;
 624                 }
 625         }
 626
 627         if (mpls) {
 628                 if (mpls_color_mark) {
 629                         uint64_t mpls_exp0 = rte_bswap64(
 630                                 (MPLS_LABEL(0, pkt0_color, 0, 0) << 32) |
 631                                 MPLS_LABEL(0, pkt0_color, 0, 0));
 632                         uint64_t mpls_exp1 = rte_bswap64(
 633                                 (MPLS_LABEL(0, pkt1_color, 0, 0) << 32) |
 634                                 MPLS_LABEL(0, pkt1_color, 0, 0));
 635                         uint64_t mpls_exp2 = rte_bswap64(
 636                                 (MPLS_LABEL(0, pkt2_color, 0, 0) << 32) |
 637                                 MPLS_LABEL(0, pkt2_color, 0, 0));
 638                         uint64_t mpls_exp3 = rte_bswap64(
 639                                 (MPLS_LABEL(0, pkt3_color, 0, 0) << 32) |
 640                                 MPLS_LABEL(0, pkt3_color, 0, 0));
 641
 642                         *slab0_ptr0 = entry0->slab[0] | mpls_exp0;
 643                         *slab0_ptr1 = entry1->slab[0] | mpls_exp1;
 644                         *slab0_ptr2 = entry2->slab[0] | mpls_exp2;
 645                         *slab0_ptr3 = entry3->slab[0] | mpls_exp3;
 646
 647                         *slab1_ptr0 = entry0->slab[1] | mpls_exp0;
 648                         *slab1_ptr1 = entry1->slab[1] | mpls_exp1;
 649                         *slab1_ptr2 = entry2->slab[1] | mpls_exp2;
 650                         *slab1_ptr3 = entry3->slab[1] | mpls_exp3;
 651
 652                         *slab2_ptr0 = entry0->slab[2];
 653                         *slab2_ptr1 = entry1->slab[2];
 654                         *slab2_ptr2 = entry2->slab[2];
 655                         *slab2_ptr3 = entry3->slab[2];
 656                 } else {
 657                         *slab0_ptr0 = entry0->slab[0];
 658                         *slab0_ptr1 = entry1->slab[0];
 659                         *slab0_ptr2 = entry2->slab[0];
 660                         *slab0_ptr3 = entry3->slab[0];
 661
 662                         *slab1_ptr0 = entry0->slab[1];
 663                         *slab1_ptr1 = entry1->slab[1];
 664                         *slab1_ptr2 = entry2->slab[1];
 665                         *slab1_ptr3 = entry3->slab[1];
 666
 667                         *slab2_ptr0 = entry0->slab[2];
 668                         *slab2_ptr1 = entry1->slab[2];
 669                         *slab2_ptr2 = entry2->slab[2];
 670                         *slab2_ptr3 = entry3->slab[2];
 671                 }
 672
 673                 if (arp == 0) {
 674                         MACADDR_DST_WRITE(slab3_ptr0, entry0->slab[3]);
 675                         MACADDR_DST_WRITE(slab3_ptr1, entry1->slab[3]);
 676                         MACADDR_DST_WRITE(slab3_ptr2, entry2->slab[3]);
 677                         MACADDR_DST_WRITE(slab3_ptr3, entry3->slab[3]);
 678                 }
 679         }
 680
 681         if (arp) {
 682                 arp_key0->port_id = port_id0;
 683                 arp_key1->port_id = port_id1;
 684                 arp_key2->port_id = port_id2;
 685                 arp_key3->port_id = port_id3;
 686
 687                 arp_key0->ip = nh_ip0;
 688                 arp_key1->ip = nh_ip1;
 689                 arp_key2->ip = nh_ip2;
 690                 arp_key3->ip = nh_ip3;
 691         }
 692 }
 693
 694 #define PKT_WORK_ROUTING_ETHERNET(arp)                          \
 695 static inline void                                              \
 696 pkt_work_routing_ether_arp##arp(                                \
 697         struct rte_mbuf *pkt,                                   \
 698         struct rte_pipeline_table_entry *table_entry,           \
 699         void *arg)                                              \
 700 {                                                               \
 701         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 0, 0);\
 702 }
 703
 704 #define PKT4_WORK_ROUTING_ETHERNET(arp)                         \
 705 static inline void                                              \
 706 pkt4_work_routing_ether_arp##arp(                               \
 707         struct rte_mbuf **pkts,                                 \
 708         struct rte_pipeline_table_entry **table_entries,        \
 709         void *arg)                                              \
 710 {                                                               \
 711         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 0, 0);\
 712 }
 713
 714 #define routing_table_ah_hit_ether(arp)                         \
 715 PKT_WORK_ROUTING_ETHERNET(arp)                                  \
 716 PKT4_WORK_ROUTING_ETHERNET(arp)                                 \
 717 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_arp##arp,      \
 718         pkt_work_routing_ether_arp##arp,                        \
 719         pkt4_work_routing_ether_arp##arp)
 720
 721 routing_table_ah_hit_ether(0)
 722 routing_table_ah_hit_ether(1)
 723
 724 #define PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)              \
 725 static inline void                                              \
 726 pkt_work_routing_ether_qinq_sched##sched##_arp##arp(            \
 727         struct rte_mbuf *pkt,                                   \
 728         struct rte_pipeline_table_entry *table_entry,           \
 729         void *arg)                                              \
 730 {                                                               \
 731         pkt_work_routing(pkt, table_entry, arg, arp, 1, sched, 0, 0);\
 732 }
 733
 734 #define PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)             \
 735 static inline void                                              \
 736 pkt4_work_routing_ether_qinq_sched##sched##_arp##arp(           \
 737         struct rte_mbuf **pkts,                                 \
 738         struct rte_pipeline_table_entry **table_entries,        \
 739         void *arg)                                              \
 740 {                                                               \
 741         pkt4_work_routing(pkts, table_entries, arg, arp, 1, sched, 0, 0);\
 742 }
 743
 744 #define routing_table_ah_hit_ether_qinq(sched, arp)             \
 745 PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                      \
 746 PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                     \
 747 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_qinq_sched##sched##_arp##arp,\
 748         pkt_work_routing_ether_qinq_sched##sched##_arp##arp,    \
 749         pkt4_work_routing_ether_qinq_sched##sched##_arp##arp)
 750
 751 routing_table_ah_hit_ether_qinq(0, 0)
 752 routing_table_ah_hit_ether_qinq(1, 0)
 753 routing_table_ah_hit_ether_qinq(2, 0)
 754 routing_table_ah_hit_ether_qinq(0, 1)
 755 routing_table_ah_hit_ether_qinq(1, 1)
 756 routing_table_ah_hit_ether_qinq(2, 1)
 757
 758 #define PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)              \
 759 static inline void                                              \
 760 pkt_work_routing_ether_mpls_color##color##_arp##arp(            \
 761         struct rte_mbuf *pkt,                                   \
 762         struct rte_pipeline_table_entry *table_entry,           \
 763         void *arg)                                              \
 764 {                                                               \
 765         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 1, color);\
 766 }
 767
 768 #define PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)             \
 769 static inline void                                              \
 770 pkt4_work_routing_ether_mpls_color##color##_arp##arp(           \
 771         struct rte_mbuf **pkts,                                 \
 772         struct rte_pipeline_table_entry **table_entries,        \
 773         void *arg)                                              \
 774 {                                                               \
 775         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 1, color);\
 776 }
 777
 778 #define routing_table_ah_hit_ether_mpls(color, arp)             \
 779 PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)                      \
 780 PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)                     \
 781 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_mpls_color##color##_arp##arp,\
 782         pkt_work_routing_ether_mpls_color##color##_arp##arp,    \
 783         pkt4_work_routing_ether_mpls_color##color##_arp##arp)
 784
 785 routing_table_ah_hit_ether_mpls(0, 0)
 786 routing_table_ah_hit_ether_mpls(1, 0)
 787 routing_table_ah_hit_ether_mpls(0, 1)
 788 routing_table_ah_hit_ether_mpls(1, 1)
 789
 790 static rte_pipeline_table_action_handler_hit
 791 get_routing_table_ah_hit(struct pipeline_routing *p)
 792 {
 793         if (p->params.dbg_ah_disable)
 794                 return NULL;
 795
 796         switch (p->params.encap) {
 797         case PIPELINE_ROUTING_ENCAP_ETHERNET:
 798                 return (p->params.n_arp_entries) ?
 799                         routing_table_ah_hit_ether_arp1 :
 800                         routing_table_ah_hit_ether_arp0;
 801
 802         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
 803                 if (p->params.n_arp_entries)
 804                         switch (p->params.qinq_sched) {
 805                         case 0:
 806                                 return routing_table_ah_hit_ether_qinq_sched0_arp1;
 807                         case 1:
 808                                 return routing_table_ah_hit_ether_qinq_sched1_arp1;
 809                         case 2:
 810                                 return routing_table_ah_hit_ether_qinq_sched2_arp1;
 811                         default:
 812                                 return NULL;
 813                         }
 814                  else
 815                         switch (p->params.qinq_sched) {
 816                         case 0:
 817                                 return routing_table_ah_hit_ether_qinq_sched0_arp0;
 818                         case 1:
 819                                 return routing_table_ah_hit_ether_qinq_sched1_arp0;
 820                         case 2:
 821                                 return routing_table_ah_hit_ether_qinq_sched2_arp0;
 822                         default:
 823                                 return NULL;
 824                         }
 825
 826         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
 827                 if (p->params.n_arp_entries)
 828                         if (p->params.mpls_color_mark)
 829                                 return routing_table_ah_hit_ether_mpls_color1_arp1;
 830                         else
 831                                 return routing_table_ah_hit_ether_mpls_color0_arp1;
 832                 else
 833                         if (p->params.mpls_color_mark)
 834                                 return routing_table_ah_hit_ether_mpls_color1_arp0;
 835                         else
 836                                 return routing_table_ah_hit_ether_mpls_color0_arp0;
 837
 838         default:
 839                 return NULL;
 840         }
 841 }
 842
 843 /*
 844  * ARP table
 845  */
 846 struct arp_table_entry {
 847         struct rte_pipeline_table_entry head;
 848         uint64_t macaddr;
 849 };
 850
 851 /**
 852  * ARP table AH
 853  */
 854 static inline void
 855 pkt_work_arp(
 856         struct rte_mbuf *pkt,
 857         struct rte_pipeline_table_entry *table_entry,
 858         __rte_unused void *arg)
 859 {
 860         struct arp_table_entry *entry = (struct arp_table_entry *) table_entry;
 861
 862         /* Read */
 863         uint64_t macaddr_dst = entry->macaddr;
 864         uint64_t *slab_ptr = (uint64_t *) ((char *) pkt->buf_addr +
 865                 (pkt->data_off - 2));
 866
 867         /* Compute */
 868
 869         /* Write */
 870         MACADDR_DST_WRITE(slab_ptr, macaddr_dst);
 871 }
 872
 873 static inline void
 874 pkt4_work_arp(
 875         struct rte_mbuf **pkts,
 876         struct rte_pipeline_table_entry **table_entries,
 877         __rte_unused void *arg)
 878 {
 879         struct arp_table_entry *entry0 =
 880                 (struct arp_table_entry *) table_entries[0];
 881         struct arp_table_entry *entry1 =
 882                 (struct arp_table_entry *) table_entries[1];
 883         struct arp_table_entry *entry2 =
 884                 (struct arp_table_entry *) table_entries[2];
 885         struct arp_table_entry *entry3 =
 886                 (struct arp_table_entry *) table_entries[3];
 887
 888         /* Read */
 889         uint64_t macaddr_dst0 = entry0->macaddr;
 890         uint64_t macaddr_dst1 = entry1->macaddr;
 891         uint64_t macaddr_dst2 = entry2->macaddr;
 892         uint64_t macaddr_dst3 = entry3->macaddr;
 893
 894         uint64_t *slab_ptr0 = (uint64_t *) ((char *) pkts[0]->buf_addr +
 895                 (pkts[0]->data_off - 2));
 896         uint64_t *slab_ptr1 = (uint64_t *) ((char *) pkts[1]->buf_addr +
 897                 (pkts[1]->data_off - 2));
 898         uint64_t *slab_ptr2 = (uint64_t *) ((char *) pkts[2]->buf_addr +
 899                 (pkts[2]->data_off - 2));
 900         uint64_t *slab_ptr3 = (uint64_t *) ((char *) pkts[3]->buf_addr +
 901                 (pkts[3]->data_off - 2));
 902
 903         /* Compute */
 904
 905         /* Write */
 906         MACADDR_DST_WRITE(slab_ptr0, macaddr_dst0);
 907         MACADDR_DST_WRITE(slab_ptr1, macaddr_dst1);
 908         MACADDR_DST_WRITE(slab_ptr2, macaddr_dst2);
 909         MACADDR_DST_WRITE(slab_ptr3, macaddr_dst3);
 910 }
 911
 912 PIPELINE_TABLE_AH_HIT(arp_table_ah_hit,
 913         pkt_work_arp,
 914         pkt4_work_arp);
 915
 916 static rte_pipeline_table_action_handler_hit
 917 get_arp_table_ah_hit(struct pipeline_routing *p)
 918 {
 919         if (p->params.dbg_ah_disable)
 920                 return NULL;
 921
 922         return arp_table_ah_hit;
 923 }
 924
 925 /*
 926  * Argument parsing
 927  */
 928 int
 929 pipeline_routing_parse_args(struct pipeline_routing_params *p,
 930         struct pipeline_params *params)
 931 {
 932         uint32_t n_routes_present = 0;
 933         uint32_t port_local_dest_present = 0;
 934         uint32_t encap_present = 0;
 935         uint32_t qinq_sched_present = 0;
 936         uint32_t mpls_color_mark_present = 0;
 937         uint32_t n_arp_entries_present = 0;
 938         uint32_t ip_hdr_offset_present = 0;
 939         uint32_t arp_key_offset_present = 0;
 940         uint32_t color_offset_present = 0;
 941         uint32_t dbg_ah_disable_present = 0;
 942         uint32_t i;
 943
 944         /* default values */
 945         p->n_routes = PIPELINE_ROUTING_N_ROUTES_DEFAULT;
 946         p->port_local_dest = params->n_ports_out - 1;
 947         p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
 948         p->qinq_sched = 0;
 949         p->mpls_color_mark = 0;
 950         p->n_arp_entries = 0;
 951         p->dbg_ah_disable = 0;
 952
 953         for (i = 0; i < params->n_args; i++) {
 954                 char *arg_name = params->args_name[i];
 955                 char *arg_value = params->args_value[i];
 956
 957                 /* n_routes */
 958                 if (strcmp(arg_name, "n_routes") == 0) {
 959                         int status;
 960
 961                         PIPELINE_PARSE_ERR_DUPLICATE(
 962                                 n_routes_present == 0, params->name,
 963                                 arg_name);
 964                         n_routes_present = 1;
 965
 966                         status = parser_read_uint32(&p->n_routes,
 967                                 arg_value);
 968                         PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
 969                                 (p->n_routes != 0)), params->name,
 970                                 arg_name, arg_value);
 971                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 972                                 params->name, arg_name, arg_value);
 973
 974                         continue;
 975                 }
 976                 /* port_local_dest */
 977                 if (strcmp(arg_name, "port_local_dest") == 0) {
 978                         int status;
 979
 980                         PIPELINE_PARSE_ERR_DUPLICATE(
 981                                 port_local_dest_present == 0, params->name,
 982                                 arg_name);
 983                         port_local_dest_present = 1;
 984
 985                         status = parser_read_uint32(&p->port_local_dest,
 986                                 arg_value);
 987                         PIPELINE_PARSE_ERR_INV_VAL(((status == 0) &&
 988                                 (p->port_local_dest < params->n_ports_out)),
 989                                 params->name, arg_name, arg_value);
 990
 991                         continue;
 992                 }
 993
 994                 /* encap */
 995                 if (strcmp(arg_name, "encap") == 0) {
 996                         PIPELINE_PARSE_ERR_DUPLICATE(encap_present == 0,
 997                                 params->name, arg_name);
 998                         encap_present = 1;
 999
1000                         /* ethernet */
1001                         if (strcmp(arg_value, "ethernet") == 0) {
1002                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
1003                                 continue;
1004                         }
1005
1006                         /* ethernet_qinq */
1007                         if (strcmp(arg_value, "ethernet_qinq") == 0) {
1008                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ;
1009                                 continue;
1010                         }
1011
1012                         /* ethernet_mpls */
1013                         if (strcmp(arg_value, "ethernet_mpls") == 0) {
1014                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS;
1015                                 continue;
1016                         }
1017
1018                         /* any other */
1019                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1020                                 arg_name, arg_value);
1021                 }
1022
1023                 /* qinq_sched */
1024                 if (strcmp(arg_name, "qinq_sched") == 0) {
1025                         int status;
1026
1027                         PIPELINE_PARSE_ERR_DUPLICATE(
1028                                 qinq_sched_present == 0, params->name,
1029                                 arg_name);
1030                         qinq_sched_present = 1;
1031
1032                         status = parser_read_arg_bool(arg_value);
1033                         if (status == -EINVAL) {
1034                                 if (strcmp(arg_value, "test") == 0) {
1035                                         p->qinq_sched = 2;
1036                                         continue;
1037                                 }
1038                         } else {
1039                                 p->qinq_sched = status;
1040                                 continue;
1041                         }
1042
1043                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1044                                 arg_name, arg_value);
1045                 }
1046
1047                 /* mpls_color_mark */
1048                 if (strcmp(arg_name, "mpls_color_mark") == 0) {
1049                         int status;
1050
1051                         PIPELINE_PARSE_ERR_DUPLICATE(
1052                                 mpls_color_mark_present == 0,
1053                                 params->name, arg_name);
1054                         mpls_color_mark_present = 1;
1055
1056
1057                         status = parser_read_arg_bool(arg_value);
1058                         if (status >= 0) {
1059                                 p->mpls_color_mark = status;
1060                                 continue;
1061                         }
1062
1063                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1064                                 arg_name, arg_value);
1065                 }
1066
1067                 /* n_arp_entries */
1068                 if (strcmp(arg_name, "n_arp_entries") == 0) {
1069                         int status;
1070
1071                         PIPELINE_PARSE_ERR_DUPLICATE(
1072                                 n_arp_entries_present == 0, params->name,
1073                                 arg_name);
1074                         n_arp_entries_present = 1;
1075
1076                         status = parser_read_uint32(&p->n_arp_entries,
1077                                 arg_value);
1078                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1079                                 params->name, arg_name, arg_value);
1080                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1081                                 params->name, arg_name, arg_value);
1082
1083                         continue;
1084                 }
1085
1086                 /* ip_hdr_offset */
1087                 if (strcmp(arg_name, "ip_hdr_offset") == 0) {
1088                         int status;
1089
1090                         PIPELINE_PARSE_ERR_DUPLICATE(
1091                                 ip_hdr_offset_present == 0, params->name,
1092                                 arg_name);
1093                         ip_hdr_offset_present = 1;
1094
1095                         status = parser_read_uint32(&p->ip_hdr_offset,
1096                                 arg_value);
1097                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1098                                 params->name, arg_name, arg_value);
1099                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1100                                 params->name, arg_name, arg_value);
1101
1102                         continue;
1103                 }
1104
1105                 /* arp_key_offset */
1106                 if (strcmp(arg_name, "arp_key_offset") == 0) {
1107                         int status;
1108
1109                         PIPELINE_PARSE_ERR_DUPLICATE(
1110                                 arp_key_offset_present == 0, params->name,
1111                                 arg_name);
1112                         arp_key_offset_present = 1;
1113
1114                         status = parser_read_uint32(&p->arp_key_offset,
1115                                 arg_value);
1116                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1117                                 params->name, arg_name, arg_value);
1118                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1119                                 params->name, arg_name, arg_value);
1120
1121                         continue;
1122                 }
1123
1124                 /* color_offset */
1125                 if (strcmp(arg_name, "color_offset") == 0) {
1126                         int status;
1127
1128                         PIPELINE_PARSE_ERR_DUPLICATE(
1129                                 color_offset_present == 0, params->name,
1130                                 arg_name);
1131                         color_offset_present = 1;
1132
1133                         status = parser_read_uint32(&p->color_offset,
1134                                 arg_value);
1135                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1136                                 params->name, arg_name, arg_value);
1137                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1138                                 params->name, arg_name, arg_value);
1139
1140                         continue;
1141                 }
1142
1143                 /* debug */
1144                 if (strcmp(arg_name, "dbg_ah_disable") == 0) {
1145                         int status;
1146
1147                         PIPELINE_PARSE_ERR_DUPLICATE(
1148                                 dbg_ah_disable_present == 0, params->name,
1149                                 arg_name);
1150                         dbg_ah_disable_present = 1;
1151
1152                         status = parser_read_arg_bool(arg_value);
1153                         if (status >= 0) {
1154                                 p->dbg_ah_disable = status;
1155                                 continue;
1156                         }
1157
1158                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1159                                 arg_name, arg_value);
1160
1161                         continue;
1162                 }
1163
1164                 /* any other */
1165                 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
1166         }
1167
1168         /* Check that mandatory arguments are present */
1169         PIPELINE_PARSE_ERR_MANDATORY(ip_hdr_offset_present, params->name,
1170                 "ip_hdr_offset");
1171
1172         /* Check relations between arguments */
1173         switch (p->encap) {
1174         case PIPELINE_ROUTING_ENCAP_ETHERNET:
1175                 PIPELINE_ARG_CHECK((!p->qinq_sched), "Parse error in "
1176                         "section \"%s\": encap = ethernet, therefore "
1177                         "qinq_sched = yes/test is not allowed",
1178                         params->name);
1179                 PIPELINE_ARG_CHECK((!p->mpls_color_mark), "Parse error "
1180                         "in section \"%s\": encap = ethernet, therefore "
1181                         "mpls_color_mark = yes is not allowed",
1182                         params->name);
1183                 PIPELINE_ARG_CHECK((!color_offset_present), "Parse error "
1184                         "in section \"%s\": encap = ethernet, therefore "
1185                         "color_offset is not allowed",
1186                         params->name);
1187                 break;
1188
1189         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
1190                 PIPELINE_ARG_CHECK((!p->mpls_color_mark), "Parse error "
1191                         "in section \"%s\": encap = ethernet_qinq, "
1192                         "therefore mpls_color_mark = yes is not allowed",
1193                         params->name);
1194                 PIPELINE_ARG_CHECK((!color_offset_present), "Parse error "
1195                         "in section \"%s\": encap = ethernet_qinq, "
1196                         "therefore color_offset is not allowed",
1197                         params->name);
1198                 break;
1199
1200         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
1201                 PIPELINE_ARG_CHECK((!p->qinq_sched), "Parse error in "
1202                         "section \"%s\": encap = ethernet_mpls, therefore "
1203                         "qinq_sched  = yes/test is not allowed",
1204                         params->name);
1205                 break;
1206         }
1207
1208         PIPELINE_ARG_CHECK((!(p->n_arp_entries &&
1209                 (!arp_key_offset_present))), "Parse error in section "
1210                         "\"%s\": n_arp_entries is set while "
1211                         "arp_key_offset is not set", params->name);
1212
1213         PIPELINE_ARG_CHECK((!((p->n_arp_entries == 0) &&
1214                 arp_key_offset_present)), "Parse error in section "
1215                         "\"%s\": arp_key_offset present while "
1216                         "n_arp_entries is not set", params->name);
1217
1218         return 0;
1219 }
1220
1221 static void *
1222 pipeline_routing_init(struct pipeline_params *params,
1223         __rte_unused void *arg)
1224 {
1225         struct pipeline *p;
1226         struct pipeline_routing *p_rt;
1227         uint32_t size, i;
1228
1229         /* Check input arguments */
1230         if ((params == NULL) ||
1231                 (params->n_ports_in == 0) ||
1232                 (params->n_ports_out == 0))
1233                 return NULL;
1234
1235         /* Memory allocation */
1236         size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_routing));
1237         p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
1238         p_rt = (struct pipeline_routing *) p;
1239         if (p == NULL)
1240                 return NULL;
1241
1242         strcpy(p->name, params->name);
1243         p->log_level = params->log_level;
1244
1245         PLOG(p, HIGH, "Routing");
1246
1247         /* Parse arguments */
1248         if (pipeline_routing_parse_args(&p_rt->params, params))
1249                 return NULL;
1250
1251         /* Pipeline */
1252         {
1253                 struct rte_pipeline_params pipeline_params = {
1254                         .name = params->name,
1255                         .socket_id = params->socket_id,
1256                         .offset_port_id = 0,
1257                 };
1258
1259                 p->p = rte_pipeline_create(&pipeline_params);
1260                 if (p->p == NULL) {
1261                         rte_free(p);
1262                         return NULL;
1263                 }
1264         }
1265
1266         /* Input ports */
1267         p->n_ports_in = params->n_ports_in;
1268         for (i = 0; i < p->n_ports_in; i++) {
1269                 struct rte_pipeline_port_in_params port_params = {
1270                         .ops = pipeline_port_in_params_get_ops(
1271                                 &params->port_in[i]),
1272                         .arg_create = pipeline_port_in_params_convert(
1273                                 &params->port_in[i]),
1274                         .f_action = NULL,
1275                         .arg_ah = NULL,
1276                         .burst_size = params->port_in[i].burst_size,
1277                 };
1278
1279                 int status = rte_pipeline_port_in_create(p->p,
1280                         &port_params,
1281                         &p->port_in_id[i]);
1282
1283                 if (status) {
1284                         rte_pipeline_free(p->p);
1285                         rte_free(p);
1286                         return NULL;
1287                 }
1288         }
1289
1290         /* Output ports */
1291         p->n_ports_out = params->n_ports_out;
1292         for (i = 0; i < p->n_ports_out; i++) {
1293                 struct rte_pipeline_port_out_params port_params = {
1294                         .ops = pipeline_port_out_params_get_ops(
1295                                 &params->port_out[i]),
1296                         .arg_create = pipeline_port_out_params_convert(
1297                                 &params->port_out[i]),
1298                         .f_action = NULL,
1299                         .arg_ah = NULL,
1300                 };
1301
1302                 int status = rte_pipeline_port_out_create(p->p,
1303                         &port_params,
1304                         &p->port_out_id[i]);
1305
1306                 if (status) {
1307                         rte_pipeline_free(p->p);
1308                         rte_free(p);
1309                         return NULL;
1310                 }
1311         }
1312
1313         /* Routing table */
1314         p->n_tables = 1;
1315         {
1316                 struct rte_table_lpm_params table_lpm_params = {
1317                         .name = p->name,
1318                         .n_rules = p_rt->params.n_routes,
1319                         .number_tbl8s = PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s,
1320                         .flags = 0,
1321                         .entry_unique_size = sizeof(struct routing_table_entry),
1322                         .offset = p_rt->params.ip_hdr_offset +
1323                                 __builtin_offsetof(struct ipv4_hdr, dst_addr),
1324                 };
1325
1326                 struct rte_pipeline_table_params table_params = {
1327                                 .ops = &rte_table_lpm_ops,
1328                                 .arg_create = &table_lpm_params,
1329                                 .f_action_hit = get_routing_table_ah_hit(p_rt),
1330                                 .f_action_miss = NULL,
1331                                 .arg_ah = p_rt,
1332                                 .action_data_size =
1333                                         sizeof(struct routing_table_entry) -
1334                                         sizeof(struct rte_pipeline_table_entry),
1335                         };
1336
1337                 int status;
1338
1339                 status = rte_pipeline_table_create(p->p,
1340                         &table_params,
1341                         &p->table_id[0]);
1342
1343                 if (status) {
1344                         rte_pipeline_free(p->p);
1345                         rte_free(p);
1346                         return NULL;
1347                 }
1348         }
1349
1350         /* ARP table configuration */
1351         if (p_rt->params.n_arp_entries) {
1352                 struct rte_table_hash_key8_ext_params table_arp_params = {
1353                         .n_entries = p_rt->params.n_arp_entries,
1354                         .n_entries_ext = p_rt->params.n_arp_entries,
1355                         .f_hash = hash_default_key8,
1356                         .seed = 0,
1357                         .signature_offset = 0, /* Unused */
1358                         .key_offset = p_rt->params.arp_key_offset,
1359                 };
1360
1361                 struct rte_pipeline_table_params table_params = {
1362                         .ops = &rte_table_hash_key8_ext_dosig_ops,
1363                         .arg_create = &table_arp_params,
1364                         .f_action_hit = get_arp_table_ah_hit(p_rt),
1365                         .f_action_miss = NULL,
1366                         .arg_ah = p_rt,
1367                         .action_data_size = sizeof(struct arp_table_entry) -
1368                                 sizeof(struct rte_pipeline_table_entry),
1369                 };
1370
1371                 int status;
1372
1373                 status = rte_pipeline_table_create(p->p,
1374                         &table_params,
1375                         &p->table_id[1]);
1376
1377                 if (status) {
1378                         rte_pipeline_free(p->p);
1379                         rte_free(p);
1380                         return NULL;
1381                 }
1382
1383                 p->n_tables++;
1384         }
1385
1386         /* Connecting input ports to tables */
1387         for (i = 0; i < p->n_ports_in; i++) {
1388                 int status = rte_pipeline_port_in_connect_to_table(p->p,
1389                         p->port_in_id[i],
1390                         p->table_id[0]);
1391
1392                 if (status) {
1393                         rte_pipeline_free(p->p);
1394                         rte_free(p);
1395                         return NULL;
1396                 }
1397         }
1398
1399         /* Enable input ports */
1400         for (i = 0; i < p->n_ports_in; i++) {
1401                 int status = rte_pipeline_port_in_enable(p->p,
1402                         p->port_in_id[i]);
1403
1404                 if (status) {
1405                         rte_pipeline_free(p->p);
1406                         rte_free(p);
1407                         return NULL;
1408                 }
1409         }
1410
1411         /* Check pipeline consistency */
1412         if (rte_pipeline_check(p->p) < 0) {
1413                 rte_pipeline_free(p->p);
1414                 rte_free(p);
1415                 return NULL;
1416         }
1417
1418         /* Message queues */
1419         p->n_msgq = params->n_msgq;
1420         for (i = 0; i < p->n_msgq; i++)
1421                 p->msgq_in[i] = params->msgq_in[i];
1422         for (i = 0; i < p->n_msgq; i++)
1423                 p->msgq_out[i] = params->msgq_out[i];
1424
1425         /* Message handlers */
1426         memcpy(p->handlers, handlers, sizeof(p->handlers));
1427         memcpy(p_rt->custom_handlers,
1428                 custom_handlers,
1429                 sizeof(p_rt->custom_handlers));
1430
1431         return p;
1432 }
1433
1434 static int
1435 pipeline_routing_free(void *pipeline)
1436 {
1437         struct pipeline *p = (struct pipeline *) pipeline;
1438
1439         /* Check input arguments */
1440         if (p == NULL)
1441                 return -1;
1442
1443         /* Free resources */
1444         rte_pipeline_free(p->p);
1445         rte_free(p);
1446         return 0;
1447 }
1448
1449 static int
1450 pipeline_routing_timer(void *pipeline)
1451 {
1452         struct pipeline *p = (struct pipeline *) pipeline;
1453
1454         pipeline_msg_req_handle(p);
1455         rte_pipeline_flush(p->p);
1456
1457         return 0;
1458 }
1459
1460 void *
1461 pipeline_routing_msg_req_custom_handler(struct pipeline *p,
1462         void *msg)
1463 {
1464         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1465         struct pipeline_custom_msg_req *req = msg;
1466         pipeline_msg_req_handler f_handle;
1467
1468         f_handle = (req->subtype < PIPELINE_ROUTING_MSG_REQS) ?
1469                 p_rt->custom_handlers[req->subtype] :
1470                 pipeline_msg_req_invalid_handler;
1471
1472         if (f_handle == NULL)
1473                 f_handle = pipeline_msg_req_invalid_handler;
1474
1475         return f_handle(p, req);
1476 }
1477
1478 void *
1479 pipeline_routing_msg_req_route_add_handler(struct pipeline *p, void *msg)
1480 {
1481         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1482         struct pipeline_routing_route_add_msg_req *req = msg;
1483         struct pipeline_routing_route_add_msg_rsp *rsp = msg;
1484
1485         struct rte_table_lpm_key key = {
1486                 .ip = req->key.key.ipv4.ip,
1487                 .depth = req->key.key.ipv4.depth,
1488         };
1489
1490         struct routing_table_entry entry_arp0 = {
1491                 .head = {
1492                         .action = RTE_PIPELINE_ACTION_PORT,
1493                         {.port_id = p->port_out_id[req->data.port_id]},
1494                 },
1495
1496                 .flags = req->data.flags,
1497                 .port_id = req->data.port_id,
1498                 .ip = 0,
1499                 .data_offset = 0,
1500                 .ether_l2_length = 0,
1501                 .slab = {0},
1502                 .slab_offset = {0},
1503         };
1504
1505         struct routing_table_entry entry_arp1 = {
1506                 .head = {
1507                         .action = RTE_PIPELINE_ACTION_TABLE,
1508                         {.table_id = p->table_id[1]},
1509                 },
1510
1511                 .flags = req->data.flags,
1512                 .port_id = req->data.port_id,
1513                 .ip = rte_bswap32(req->data.ethernet.ip),
1514                 .data_offset = 0,
1515                 .ether_l2_length = 0,
1516                 .slab = {0},
1517                 .slab_offset = {0},
1518         };
1519
1520         struct rte_pipeline_table_entry *entry = (p_rt->params.n_arp_entries) ?
1521                 (struct rte_pipeline_table_entry *) &entry_arp1 :
1522                 (struct rte_pipeline_table_entry *) &entry_arp0;
1523
1524         if ((req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) ||
1525                 ((p_rt->params.n_arp_entries == 0) &&
1526                         (req->data.flags & PIPELINE_ROUTING_ROUTE_ARP)) ||
1527                 (p_rt->params.n_arp_entries &&
1528                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_ARP) == 0)) ||
1529                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1530                         (req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ)) ||
1531                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1532                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ) == 0)) ||
1533                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1534                         (req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS)) ||
1535                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1536                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS) == 0))) {
1537                 rsp->status = -1;
1538                 return rsp;
1539         }
1540
1541         /* Ether - ARP off */
1542         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1543                 (p_rt->params.n_arp_entries == 0)) {
1544                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1545                 uint64_t macaddr_dst;
1546                 uint64_t ethertype = ETHER_TYPE_IPv4;
1547
1548                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1549                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1550
1551                 entry_arp0.slab[0] =
1552                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype);
1553                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1554
1555                 entry_arp0.slab[1] = rte_bswap64(macaddr_dst);
1556                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1557
1558                 entry_arp0.data_offset = entry_arp0.slab_offset[1] + 2
1559                         - sizeof(struct rte_mbuf);
1560                 entry_arp0.ether_l2_length = 14;
1561         }
1562
1563         /* Ether - ARP on */
1564         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1565                 p_rt->params.n_arp_entries) {
1566                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1567                 uint64_t ethertype = ETHER_TYPE_IPv4;
1568
1569                 entry_arp1.slab[0] =
1570                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype);
1571                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1572
1573                 entry_arp1.data_offset = entry_arp1.slab_offset[0] - 6
1574                         - sizeof(struct rte_mbuf);
1575                 entry_arp1.ether_l2_length = 14;
1576         }
1577
1578         /* Ether QinQ - ARP off */
1579         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1580                 (p_rt->params.n_arp_entries == 0)) {
1581                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1582                 uint64_t macaddr_dst;
1583                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1584                 uint64_t ethertype_vlan = 0x8100;
1585                 uint64_t ethertype_qinq = 0x9100;
1586                 uint64_t svlan = req->data.l2.qinq.svlan;
1587                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1588
1589                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1590                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1591
1592                 entry_arp0.slab[0] = rte_bswap64((svlan << 48) |
1593                         (ethertype_vlan << 32) |
1594                         (cvlan << 16) |
1595                         ethertype_ipv4);
1596                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1597
1598                 entry_arp0.slab[1] =
1599                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_qinq);
1600                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1601
1602                 entry_arp0.slab[2] = rte_bswap64(macaddr_dst);
1603                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset - 3 * 8;
1604
1605                 entry_arp0.data_offset = entry_arp0.slab_offset[2] + 2
1606                         - sizeof(struct rte_mbuf);
1607                 entry_arp0.ether_l2_length = 22;
1608         }
1609
1610         /* Ether QinQ - ARP on */
1611         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1612                 p_rt->params.n_arp_entries) {
1613                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1614                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1615                 uint64_t ethertype_vlan = 0x8100;
1616                 uint64_t ethertype_qinq = 0x9100;
1617                 uint64_t svlan = req->data.l2.qinq.svlan;
1618                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1619
1620                 entry_arp1.slab[0] = rte_bswap64((svlan << 48) |
1621                         (ethertype_vlan << 32) |
1622                         (cvlan << 16) |
1623                         ethertype_ipv4);
1624                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1625
1626                 entry_arp1.slab[1] =
1627                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_qinq);
1628                 entry_arp1.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1629
1630                 entry_arp1.data_offset = entry_arp1.slab_offset[1] - 6
1631                         - sizeof(struct rte_mbuf);
1632                 entry_arp1.ether_l2_length = 22;
1633         }
1634
1635         /* Ether MPLS - ARP off */
1636         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1637                 (p_rt->params.n_arp_entries == 0)) {
1638                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1639                 uint64_t macaddr_dst;
1640                 uint64_t ethertype_mpls = 0x8847;
1641
1642                 uint64_t label0 = req->data.l2.mpls.labels[0];
1643                 uint64_t label1 = req->data.l2.mpls.labels[1];
1644                 uint64_t label2 = req->data.l2.mpls.labels[2];
1645                 uint64_t label3 = req->data.l2.mpls.labels[3];
1646                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1647
1648                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1649                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1650
1651                 switch (n_labels) {
1652                 case 1:
1653                         entry_arp0.slab[0] = 0;
1654                         entry_arp0.slab_offset[0] =
1655                                 p_rt->params.ip_hdr_offset - 8;
1656
1657                         entry_arp0.slab[1] = rte_bswap64(
1658                                 MPLS_LABEL(label0, 0, 1, 0));
1659                         entry_arp0.slab_offset[1] =
1660                                 p_rt->params.ip_hdr_offset - 8;
1661                         break;
1662
1663                 case 2:
1664                         entry_arp0.slab[0] = 0;
1665                         entry_arp0.slab_offset[0] =
1666                                 p_rt->params.ip_hdr_offset - 8;
1667
1668                         entry_arp0.slab[1] = rte_bswap64(
1669                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1670                                 MPLS_LABEL(label1, 0, 1, 0));
1671                         entry_arp0.slab_offset[1] =
1672                                 p_rt->params.ip_hdr_offset - 8;
1673                         break;
1674
1675                 case 3:
1676                         entry_arp0.slab[0] = rte_bswap64(
1677                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1678                                 MPLS_LABEL(label2, 0, 1, 0));
1679                         entry_arp0.slab_offset[0] =
1680                                 p_rt->params.ip_hdr_offset - 8;
1681
1682                         entry_arp0.slab[1] = rte_bswap64(
1683                                 MPLS_LABEL(label0, 0, 0, 0));
1684                         entry_arp0.slab_offset[1] =
1685                                 p_rt->params.ip_hdr_offset - 2 * 8;
1686                         break;
1687
1688                 case 4:
1689                         entry_arp0.slab[0] = rte_bswap64(
1690                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1691                                 MPLS_LABEL(label3, 0, 1, 0));
1692                         entry_arp0.slab_offset[0] =
1693                                 p_rt->params.ip_hdr_offset - 8;
1694
1695                         entry_arp0.slab[1] = rte_bswap64(
1696                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1697                                 MPLS_LABEL(label1, 0, 0, 0));
1698                         entry_arp0.slab_offset[1] =
1699                                 p_rt->params.ip_hdr_offset - 2 * 8;
1700                         break;
1701
1702                 default:
1703                         rsp->status = -1;
1704                         return rsp;
1705                 }
1706
1707                 entry_arp0.slab[2] =
1708                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_mpls);
1709                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset -
1710                         (n_labels * 4 + 8);
1711
1712                 entry_arp0.slab[3] = rte_bswap64(macaddr_dst);
1713                 entry_arp0.slab_offset[3] = p_rt->params.ip_hdr_offset -
1714                         (n_labels * 4 + 2 * 8);
1715
1716                 entry_arp0.data_offset = entry_arp0.slab_offset[3] + 2
1717                         - sizeof(struct rte_mbuf);
1718                 entry_arp0.ether_l2_length = n_labels * 4 + 14;
1719         }
1720
1721         /* Ether MPLS - ARP on */
1722         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1723                 p_rt->params.n_arp_entries) {
1724                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1725                 uint64_t ethertype_mpls = 0x8847;
1726
1727                 uint64_t label0 = req->data.l2.mpls.labels[0];
1728                 uint64_t label1 = req->data.l2.mpls.labels[1];
1729                 uint64_t label2 = req->data.l2.mpls.labels[2];
1730                 uint64_t label3 = req->data.l2.mpls.labels[3];
1731                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1732
1733                 switch (n_labels) {
1734                 case 1:
1735                         entry_arp1.slab[0] = 0;
1736                         entry_arp1.slab_offset[0] =
1737                                 p_rt->params.ip_hdr_offset - 8;
1738
1739                         entry_arp1.slab[1] = rte_bswap64(
1740                                 MPLS_LABEL(label0, 0, 1, 0));
1741                         entry_arp1.slab_offset[1] =
1742                                 p_rt->params.ip_hdr_offset - 8;
1743                         break;
1744
1745                 case 2:
1746                         entry_arp1.slab[0] = 0;
1747                         entry_arp1.slab_offset[0] =
1748                                 p_rt->params.ip_hdr_offset - 8;
1749
1750                         entry_arp1.slab[1] = rte_bswap64(
1751                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1752                                 MPLS_LABEL(label1, 0, 1, 0));
1753                         entry_arp1.slab_offset[1] =
1754                                 p_rt->params.ip_hdr_offset - 8;
1755                         break;
1756
1757                 case 3:
1758                         entry_arp1.slab[0] = rte_bswap64(
1759                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1760                                 MPLS_LABEL(label2, 0, 1, 0));
1761                         entry_arp1.slab_offset[0] =
1762                                 p_rt->params.ip_hdr_offset - 8;
1763
1764                         entry_arp1.slab[1] = rte_bswap64(
1765                                 MPLS_LABEL(label0, 0, 0, 0));
1766                         entry_arp1.slab_offset[1] =
1767                                 p_rt->params.ip_hdr_offset - 2 * 8;
1768                         break;
1769
1770                 case 4:
1771                         entry_arp1.slab[0] = rte_bswap64(
1772                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1773                                 MPLS_LABEL(label3, 0, 1, 0));
1774                         entry_arp1.slab_offset[0] =
1775                                 p_rt->params.ip_hdr_offset - 8;
1776
1777                         entry_arp1.slab[1] = rte_bswap64(
1778                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1779                                 MPLS_LABEL(label1, 0, 0, 0));
1780                         entry_arp1.slab_offset[1] =
1781                                 p_rt->params.ip_hdr_offset - 2 * 8;
1782                         break;
1783
1784                 default:
1785                         rsp->status = -1;
1786                         return rsp;
1787                 }
1788
1789                 entry_arp1.slab[2] =
1790                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_mpls);
1791                 entry_arp1.slab_offset[2] = p_rt->params.ip_hdr_offset -
1792                         (n_labels * 4 + 8);
1793
1794                 entry_arp1.data_offset = entry_arp1.slab_offset[2] - 6
1795                         - sizeof(struct rte_mbuf);
1796                 entry_arp1.ether_l2_length = n_labels * 4 + 14;
1797         }
1798
1799         rsp->status = rte_pipeline_table_entry_add(p->p,
1800                 p->table_id[0],
1801                 &key,
1802                 entry,
1803                 &rsp->key_found,
1804                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1805
1806         return rsp;
1807 }
1808
1809 void *
1810 pipeline_routing_msg_req_route_del_handler(struct pipeline *p, void *msg)
1811 {
1812         struct pipeline_routing_route_delete_msg_req *req = msg;
1813         struct pipeline_routing_route_delete_msg_rsp *rsp = msg;
1814
1815         struct rte_table_lpm_key key = {
1816                 .ip = req->key.key.ipv4.ip,
1817                 .depth = req->key.key.ipv4.depth,
1818         };
1819
1820         if (req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) {
1821                 rsp->status = -1;
1822                 return rsp;
1823         }
1824
1825         rsp->status = rte_pipeline_table_entry_delete(p->p,
1826                 p->table_id[0],
1827                 &key,
1828                 &rsp->key_found,
1829                 NULL);
1830
1831         return rsp;
1832 }
1833
1834 void *
1835 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
1836         void *msg)
1837 {
1838         struct pipeline_routing_route_add_default_msg_req *req = msg;
1839         struct pipeline_routing_route_add_default_msg_rsp *rsp = msg;
1840
1841         struct routing_table_entry default_entry = {
1842                 .head = {
1843                         .action = RTE_PIPELINE_ACTION_PORT,
1844                         {.port_id = p->port_out_id[req->port_id]},
1845                 },
1846
1847                 .flags = 0,
1848                 .port_id = 0,
1849                 .ip = 0,
1850         };
1851
1852         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1853                 p->table_id[0],
1854                 (struct rte_pipeline_table_entry *) &default_entry,
1855                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1856
1857         return rsp;
1858 }
1859
1860 void *
1861 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
1862         void *msg)
1863 {
1864         struct pipeline_routing_route_delete_default_msg_rsp *rsp = msg;
1865
1866         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1867                 p->table_id[0],
1868                 NULL);
1869
1870         return rsp;
1871 }
1872
1873 void *
1874 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p, void *msg)
1875 {
1876         struct pipeline_routing_arp_add_msg_req *req = msg;
1877         struct pipeline_routing_arp_add_msg_rsp *rsp = msg;
1878
1879         struct pipeline_routing_arp_key_ipv4 key = {
1880                 .port_id = req->key.key.ipv4.port_id,
1881                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1882         };
1883
1884         struct arp_table_entry entry = {
1885                 .head = {
1886                         .action = RTE_PIPELINE_ACTION_PORT,
1887                         {.port_id = p->port_out_id[req->key.key.ipv4.port_id]},
1888                 },
1889
1890                 .macaddr = 0, /* set below */
1891         };
1892
1893         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1894                 rsp->status = -1;
1895                 return rsp;
1896         }
1897
1898         entry.macaddr = *((uint64_t *)&(req->macaddr));
1899         entry.macaddr = entry.macaddr << 16;
1900
1901         rsp->status = rte_pipeline_table_entry_add(p->p,
1902                 p->table_id[1],
1903                 &key,
1904                 (struct rte_pipeline_table_entry *) &entry,
1905                 &rsp->key_found,
1906                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1907
1908         return rsp;
1909 }
1910
1911 void *
1912 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p, void *msg)
1913 {
1914         struct pipeline_routing_arp_delete_msg_req *req = msg;
1915         struct pipeline_routing_arp_delete_msg_rsp *rsp = msg;
1916
1917         struct pipeline_routing_arp_key_ipv4 key = {
1918                 .port_id = req->key.key.ipv4.port_id,
1919                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1920         };
1921
1922         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1923                 rsp->status = -1;
1924                 return rsp;
1925         }
1926
1927         rsp->status = rte_pipeline_table_entry_delete(p->p,
1928                 p->table_id[1],
1929                 &key,
1930                 &rsp->key_found,
1931                 NULL);
1932
1933         return rsp;
1934 }
1935
1936 void *
1937 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p, void *msg)
1938 {
1939         struct pipeline_routing_arp_add_default_msg_req *req = msg;
1940         struct pipeline_routing_arp_add_default_msg_rsp *rsp = msg;
1941
1942         struct arp_table_entry default_entry = {
1943                 .head = {
1944                         .action = RTE_PIPELINE_ACTION_PORT,
1945                         {.port_id = p->port_out_id[req->port_id]},
1946                 },
1947
1948                 .macaddr = 0,
1949         };
1950
1951         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1952                 p->table_id[1],
1953                 (struct rte_pipeline_table_entry *) &default_entry,
1954                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1955
1956         return rsp;
1957 }
1958
1959 void *
1960 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p, void *msg)
1961 {
1962         struct pipeline_routing_arp_delete_default_msg_rsp *rsp = msg;
1963
1964         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1965                 p->table_id[1],
1966                 NULL);
1967
1968         return rsp;
1969 }
1970
1971 void *
1972 pipeline_routing_msg_req_set_macaddr_handler(struct pipeline *p, void *msg)
1973 {
1974         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1975         struct pipeline_routing_set_macaddr_msg_req *req = msg;
1976         struct pipeline_routing_set_macaddr_msg_rsp *rsp = msg;
1977         uint32_t port_id;
1978
1979         for (port_id = 0; port_id < p->n_ports_out; port_id++)
1980                 p_rt->macaddr[port_id] = req->macaddr[port_id];
1981
1982         rsp->status = 0;
1983
1984         return rsp;
1985 }
1986
1987 struct pipeline_be_ops pipeline_routing_be_ops = {
1988         .f_init = pipeline_routing_init,
1989         .f_free = pipeline_routing_free,
1990         .f_run = NULL,
1991         .f_timer = pipeline_routing_timer,
1992 };