drivers/net/mlx5/mlx5_flow_tcf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 6WIND S.A.
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <assert.h>
   7 #include <errno.h>
   8 #include <libmnl/libmnl.h>
   9 #include <linux/gen_stats.h>
  10 #include <linux/if_ether.h>
  11 #include <linux/netlink.h>
  12 #include <linux/pkt_cls.h>
  13 #include <linux/pkt_sched.h>
  14 #include <linux/rtnetlink.h>
  15 #include <linux/tc_act/tc_gact.h>
  16 #include <linux/tc_act/tc_mirred.h>
  17 #include <netinet/in.h>
  18 #include <stdalign.h>
  19 #include <stdbool.h>
  20 #include <stddef.h>
  21 #include <stdint.h>
  22 #include <stdlib.h>
  23 #include <sys/socket.h>
  24
  25 #include <rte_byteorder.h>
  26 #include <rte_errno.h>
  27 #include <rte_ether.h>
  28 #include <rte_flow.h>
  29 #include <rte_malloc.h>
  30 #include <rte_common.h>
  31
  32 #include "mlx5.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_autoconf.h"
  35
  36 #ifdef HAVE_TC_ACT_VLAN
  37
  38 #include <linux/tc_act/tc_vlan.h>
  39
  40 #else /* HAVE_TC_ACT_VLAN */
  41
  42 #define TCA_VLAN_ACT_POP 1
  43 #define TCA_VLAN_ACT_PUSH 2
  44 #define TCA_VLAN_ACT_MODIFY 3
  45 #define TCA_VLAN_PARMS 2
  46 #define TCA_VLAN_PUSH_VLAN_ID 3
  47 #define TCA_VLAN_PUSH_VLAN_PROTOCOL 4
  48 #define TCA_VLAN_PAD 5
  49 #define TCA_VLAN_PUSH_VLAN_PRIORITY 6
  50
  51 struct tc_vlan {
  52         tc_gen;
  53         int v_action;
  54 };
  55
  56 #endif /* HAVE_TC_ACT_VLAN */
  57
  58 #ifdef HAVE_TC_ACT_PEDIT
  59
  60 #include <linux/tc_act/tc_pedit.h>
  61
  62 #else /* HAVE_TC_ACT_VLAN */
  63
  64 enum {
  65         TCA_PEDIT_UNSPEC,
  66         TCA_PEDIT_TM,
  67         TCA_PEDIT_PARMS,
  68         TCA_PEDIT_PAD,
  69         TCA_PEDIT_PARMS_EX,
  70         TCA_PEDIT_KEYS_EX,
  71         TCA_PEDIT_KEY_EX,
  72         __TCA_PEDIT_MAX
  73 };
  74
  75 enum {
  76         TCA_PEDIT_KEY_EX_HTYPE = 1,
  77         TCA_PEDIT_KEY_EX_CMD = 2,
  78         __TCA_PEDIT_KEY_EX_MAX
  79 };
  80
  81 enum pedit_header_type {
  82         TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK = 0,
  83         TCA_PEDIT_KEY_EX_HDR_TYPE_ETH = 1,
  84         TCA_PEDIT_KEY_EX_HDR_TYPE_IP4 = 2,
  85         TCA_PEDIT_KEY_EX_HDR_TYPE_IP6 = 3,
  86         TCA_PEDIT_KEY_EX_HDR_TYPE_TCP = 4,
  87         TCA_PEDIT_KEY_EX_HDR_TYPE_UDP = 5,
  88         __PEDIT_HDR_TYPE_MAX,
  89 };
  90
  91 enum pedit_cmd {
  92         TCA_PEDIT_KEY_EX_CMD_SET = 0,
  93         TCA_PEDIT_KEY_EX_CMD_ADD = 1,
  94         __PEDIT_CMD_MAX,
  95 };
  96
  97 struct tc_pedit_key {
  98         __u32 mask; /* AND */
  99         __u32 val; /*XOR */
 100         __u32 off; /*offset */
 101         __u32 at;
 102         __u32 offmask;
 103         __u32 shift;
 104 };
 105
 106 __extension__
 107 struct tc_pedit_sel {
 108         tc_gen;
 109         unsigned char nkeys;
 110         unsigned char flags;
 111         struct tc_pedit_key keys[0];
 112 };
 113
 114 #endif /* HAVE_TC_ACT_VLAN */
 115
 116 #ifdef HAVE_TC_ACT_TUNNEL_KEY
 117
 118 #include <linux/tc_act/tc_tunnel_key.h>
 119
 120 #ifndef HAVE_TCA_TUNNEL_KEY_ENC_DST_PORT
 121 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 122 #endif
 123
 124 #ifndef HAVE_TCA_TUNNEL_KEY_NO_CSUM
 125 #define TCA_TUNNEL_KEY_NO_CSUM 10
 126 #endif
 127
 128 #else /* HAVE_TC_ACT_TUNNEL_KEY */
 129
 130 #define TCA_ACT_TUNNEL_KEY 17
 131 #define TCA_TUNNEL_KEY_ACT_SET 1
 132 #define TCA_TUNNEL_KEY_ACT_RELEASE 2
 133 #define TCA_TUNNEL_KEY_PARMS 2
 134 #define TCA_TUNNEL_KEY_ENC_IPV4_SRC 3
 135 #define TCA_TUNNEL_KEY_ENC_IPV4_DST 4
 136 #define TCA_TUNNEL_KEY_ENC_IPV6_SRC 5
 137 #define TCA_TUNNEL_KEY_ENC_IPV6_DST 6
 138 #define TCA_TUNNEL_KEY_ENC_KEY_ID 7
 139 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 140 #define TCA_TUNNEL_KEY_NO_CSUM 10
 141
 142 struct tc_tunnel_key {
 143         tc_gen;
 144         int t_action;
 145 };
 146
 147 #endif /* HAVE_TC_ACT_TUNNEL_KEY */
 148
 149 /* Normally found in linux/netlink.h. */
 150 #ifndef NETLINK_CAP_ACK
 151 #define NETLINK_CAP_ACK 10
 152 #endif
 153
 154 /* Normally found in linux/pkt_sched.h. */
 155 #ifndef TC_H_MIN_INGRESS
 156 #define TC_H_MIN_INGRESS 0xfff2u
 157 #endif
 158
 159 /* Normally found in linux/pkt_cls.h. */
 160 #ifndef TCA_CLS_FLAGS_SKIP_SW
 161 #define TCA_CLS_FLAGS_SKIP_SW (1 << 1)
 162 #endif
 163 #ifndef TCA_CLS_FLAGS_IN_HW
 164 #define TCA_CLS_FLAGS_IN_HW (1 << 2)
 165 #endif
 166 #ifndef HAVE_TCA_CHAIN
 167 #define TCA_CHAIN 11
 168 #endif
 169 #ifndef HAVE_TCA_FLOWER_ACT
 170 #define TCA_FLOWER_ACT 3
 171 #endif
 172 #ifndef HAVE_TCA_FLOWER_FLAGS
 173 #define TCA_FLOWER_FLAGS 22
 174 #endif
 175 #ifndef HAVE_TCA_FLOWER_KEY_ETH_TYPE
 176 #define TCA_FLOWER_KEY_ETH_TYPE 8
 177 #endif
 178 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST
 179 #define TCA_FLOWER_KEY_ETH_DST 4
 180 #endif
 181 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST_MASK
 182 #define TCA_FLOWER_KEY_ETH_DST_MASK 5
 183 #endif
 184 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC
 185 #define TCA_FLOWER_KEY_ETH_SRC 6
 186 #endif
 187 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK
 188 #define TCA_FLOWER_KEY_ETH_SRC_MASK 7
 189 #endif
 190 #ifndef HAVE_TCA_FLOWER_KEY_IP_PROTO
 191 #define TCA_FLOWER_KEY_IP_PROTO 9
 192 #endif
 193 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC
 194 #define TCA_FLOWER_KEY_IPV4_SRC 10
 195 #endif
 196 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK
 197 #define TCA_FLOWER_KEY_IPV4_SRC_MASK 11
 198 #endif
 199 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST
 200 #define TCA_FLOWER_KEY_IPV4_DST 12
 201 #endif
 202 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK
 203 #define TCA_FLOWER_KEY_IPV4_DST_MASK 13
 204 #endif
 205 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC
 206 #define TCA_FLOWER_KEY_IPV6_SRC 14
 207 #endif
 208 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK
 209 #define TCA_FLOWER_KEY_IPV6_SRC_MASK 15
 210 #endif
 211 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST
 212 #define TCA_FLOWER_KEY_IPV6_DST 16
 213 #endif
 214 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK
 215 #define TCA_FLOWER_KEY_IPV6_DST_MASK 17
 216 #endif
 217 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC
 218 #define TCA_FLOWER_KEY_TCP_SRC 18
 219 #endif
 220 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK
 221 #define TCA_FLOWER_KEY_TCP_SRC_MASK 35
 222 #endif
 223 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST
 224 #define TCA_FLOWER_KEY_TCP_DST 19
 225 #endif
 226 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST_MASK
 227 #define TCA_FLOWER_KEY_TCP_DST_MASK 36
 228 #endif
 229 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC
 230 #define TCA_FLOWER_KEY_UDP_SRC 20
 231 #endif
 232 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK
 233 #define TCA_FLOWER_KEY_UDP_SRC_MASK 37
 234 #endif
 235 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST
 236 #define TCA_FLOWER_KEY_UDP_DST 21
 237 #endif
 238 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST_MASK
 239 #define TCA_FLOWER_KEY_UDP_DST_MASK 38
 240 #endif
 241 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ID
 242 #define TCA_FLOWER_KEY_VLAN_ID 23
 243 #endif
 244 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_PRIO
 245 #define TCA_FLOWER_KEY_VLAN_PRIO 24
 246 #endif
 247 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE
 248 #define TCA_FLOWER_KEY_VLAN_ETH_TYPE 25
 249 #endif
 250 #ifndef HAVE_TCA_FLOWER_KEY_ENC_KEY_ID
 251 #define TCA_FLOWER_KEY_ENC_KEY_ID 26
 252 #endif
 253 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC
 254 #define TCA_FLOWER_KEY_ENC_IPV4_SRC 27
 255 #endif
 256 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK
 257 #define TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK 28
 258 #endif
 259 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST
 260 #define TCA_FLOWER_KEY_ENC_IPV4_DST 29
 261 #endif
 262 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST_MASK
 263 #define TCA_FLOWER_KEY_ENC_IPV4_DST_MASK 30
 264 #endif
 265 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC
 266 #define TCA_FLOWER_KEY_ENC_IPV6_SRC 31
 267 #endif
 268 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK
 269 #define TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK 32
 270 #endif
 271 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST
 272 #define TCA_FLOWER_KEY_ENC_IPV6_DST 33
 273 #endif
 274 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST_MASK
 275 #define TCA_FLOWER_KEY_ENC_IPV6_DST_MASK 34
 276 #endif
 277 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT
 278 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT 43
 279 #endif
 280 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK
 281 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK 44
 282 #endif
 283 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT
 284 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT 45
 285 #endif
 286 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK
 287 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK 46
 288 #endif
 289 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS
 290 #define TCA_FLOWER_KEY_TCP_FLAGS 71
 291 #endif
 292 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK
 293 #define TCA_FLOWER_KEY_TCP_FLAGS_MASK 72
 294 #endif
 295 #ifndef HAVE_TC_ACT_GOTO_CHAIN
 296 #define TC_ACT_GOTO_CHAIN 0x20000000
 297 #endif
 298
 299 #ifndef IPV6_ADDR_LEN
 300 #define IPV6_ADDR_LEN 16
 301 #endif
 302
 303 #ifndef IPV4_ADDR_LEN
 304 #define IPV4_ADDR_LEN 4
 305 #endif
 306
 307 #ifndef TP_PORT_LEN
 308 #define TP_PORT_LEN 2 /* Transport Port (UDP/TCP) Length */
 309 #endif
 310
 311 #ifndef TTL_LEN
 312 #define TTL_LEN 1
 313 #endif
 314
 315 #ifndef TCA_ACT_MAX_PRIO
 316 #define TCA_ACT_MAX_PRIO 32
 317 #endif
 318
 319 /** UDP port range of VXLAN devices created by driver. */
 320 #define MLX5_VXLAN_PORT_MIN 30000
 321 #define MLX5_VXLAN_PORT_MAX 60000
 322 #define MLX5_VXLAN_DEVICE_PFX "vmlx_"
 323
 324 /** Tunnel action type, used for @p type in header structure. */
 325 enum flow_tcf_tunact_type {
 326         FLOW_TCF_TUNACT_VXLAN_DECAP,
 327         FLOW_TCF_TUNACT_VXLAN_ENCAP,
 328 };
 329
 330 /** Flags used for @p mask in tunnel action encap descriptors. */
 331 #define FLOW_TCF_ENCAP_ETH_SRC (1u << 0)
 332 #define FLOW_TCF_ENCAP_ETH_DST (1u << 1)
 333 #define FLOW_TCF_ENCAP_IPV4_SRC (1u << 2)
 334 #define FLOW_TCF_ENCAP_IPV4_DST (1u << 3)
 335 #define FLOW_TCF_ENCAP_IPV6_SRC (1u << 4)
 336 #define FLOW_TCF_ENCAP_IPV6_DST (1u << 5)
 337 #define FLOW_TCF_ENCAP_UDP_SRC (1u << 6)
 338 #define FLOW_TCF_ENCAP_UDP_DST (1u << 7)
 339 #define FLOW_TCF_ENCAP_VXLAN_VNI (1u << 8)
 340
 341 /**
 342  * Structure for holding netlink context.
 343  * Note the size of the message buffer which is MNL_SOCKET_BUFFER_SIZE.
 344  * Using this (8KB) buffer size ensures that netlink messages will never be
 345  * truncated.
 346  */
 347 struct mlx5_flow_tcf_context {
 348         struct mnl_socket *nl; /* NETLINK_ROUTE libmnl socket. */
 349         uint32_t seq; /* Message sequence number. */
 350         uint32_t buf_size; /* Message buffer size. */
 351         uint8_t *buf; /* Message buffer. */
 352 };
 353
 354 /**
 355  * Neigh rule structure. The neigh rule is applied via Netlink to
 356  * outer tunnel iface in order to provide destination MAC address
 357  * for the VXLAN encapsultion. The neigh rule is implicitly related
 358  * to the Flow itself and can be shared by multiple Flows.
 359  */
 360 struct tcf_neigh_rule {
 361         LIST_ENTRY(tcf_neigh_rule) next;
 362         uint32_t refcnt;
 363         struct ether_addr eth;
 364         uint16_t mask;
 365         union {
 366                 struct {
 367                         rte_be32_t dst;
 368                 } ipv4;
 369                 struct {
 370                         uint8_t dst[IPV6_ADDR_LEN];
 371                 } ipv6;
 372         };
 373 };
 374
 375 /**
 376  * Local rule structure. The local rule is applied via Netlink to
 377  * outer tunnel iface in order to provide local and peer IP addresses
 378  * of the VXLAN tunnel for encapsulation. The local rule is implicitly
 379  * related to the Flow itself and can be shared by multiple Flows.
 380  */
 381 struct tcf_local_rule {
 382         LIST_ENTRY(tcf_local_rule) next;
 383         uint32_t refcnt;
 384         uint16_t mask;
 385         union {
 386                 struct {
 387                         rte_be32_t dst;
 388                         rte_be32_t src;
 389                 } ipv4;
 390                 struct {
 391                         uint8_t dst[IPV6_ADDR_LEN];
 392                         uint8_t src[IPV6_ADDR_LEN];
 393                 } ipv6;
 394         };
 395 };
 396
 397 /** VXLAN virtual netdev. */
 398 struct tcf_vtep {
 399         LIST_ENTRY(tcf_vtep) next;
 400         LIST_HEAD(, tcf_neigh_rule) neigh;
 401         LIST_HEAD(, tcf_local_rule) local;
 402         uint32_t refcnt;
 403         unsigned int ifindex; /**< Own interface index. */
 404         unsigned int ifouter; /**< Index of device attached to. */
 405         uint16_t port;
 406         uint8_t created;
 407 };
 408
 409 /** Tunnel descriptor header, common for all tunnel types. */
 410 struct flow_tcf_tunnel_hdr {
 411         uint32_t type; /**< Tunnel action type. */
 412         struct tcf_vtep *vtep; /**< Virtual tunnel endpoint device. */
 413         unsigned int ifindex_org; /**< Original dst/src interface */
 414         unsigned int *ifindex_ptr; /**< Interface ptr in message. */
 415 };
 416
 417 struct flow_tcf_vxlan_decap {
 418         struct flow_tcf_tunnel_hdr hdr;
 419         uint16_t udp_port;
 420 };
 421
 422 struct flow_tcf_vxlan_encap {
 423         struct flow_tcf_tunnel_hdr hdr;
 424         uint32_t mask;
 425         struct {
 426                 struct ether_addr dst;
 427                 struct ether_addr src;
 428         } eth;
 429         union {
 430                 struct {
 431                         rte_be32_t dst;
 432                         rte_be32_t src;
 433                 } ipv4;
 434                 struct {
 435                         uint8_t dst[IPV6_ADDR_LEN];
 436                         uint8_t src[IPV6_ADDR_LEN];
 437                 } ipv6;
 438         };
 439 struct {
 440                 rte_be16_t src;
 441                 rte_be16_t dst;
 442         } udp;
 443         struct {
 444                 uint8_t vni[3];
 445         } vxlan;
 446 };
 447
 448 /** Structure used when extracting the values of a flow counters
 449  * from a netlink message.
 450  */
 451 struct flow_tcf_stats_basic {
 452         bool valid;
 453         struct gnet_stats_basic counters;
 454 };
 455
 456 /** Empty masks for known item types. */
 457 static const union {
 458         struct rte_flow_item_port_id port_id;
 459         struct rte_flow_item_eth eth;
 460         struct rte_flow_item_vlan vlan;
 461         struct rte_flow_item_ipv4 ipv4;
 462         struct rte_flow_item_ipv6 ipv6;
 463         struct rte_flow_item_tcp tcp;
 464         struct rte_flow_item_udp udp;
 465         struct rte_flow_item_vxlan vxlan;
 466 } flow_tcf_mask_empty;
 467
 468 /** Supported masks for known item types. */
 469 static const struct {
 470         struct rte_flow_item_port_id port_id;
 471         struct rte_flow_item_eth eth;
 472         struct rte_flow_item_vlan vlan;
 473         struct rte_flow_item_ipv4 ipv4;
 474         struct rte_flow_item_ipv6 ipv6;
 475         struct rte_flow_item_tcp tcp;
 476         struct rte_flow_item_udp udp;
 477         struct rte_flow_item_vxlan vxlan;
 478 } flow_tcf_mask_supported = {
 479         .port_id = {
 480                 .id = 0xffffffff,
 481         },
 482         .eth = {
 483                 .type = RTE_BE16(0xffff),
 484                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 485                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 486         },
 487         .vlan = {
 488                 /* PCP and VID only, no DEI. */
 489                 .tci = RTE_BE16(0xefff),
 490                 .inner_type = RTE_BE16(0xffff),
 491         },
 492         .ipv4.hdr = {
 493                 .next_proto_id = 0xff,
 494                 .src_addr = RTE_BE32(0xffffffff),
 495                 .dst_addr = RTE_BE32(0xffffffff),
 496         },
 497         .ipv6.hdr = {
 498                 .proto = 0xff,
 499                 .src_addr =
 500                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 501                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 502                 .dst_addr =
 503                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 504                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 505         },
 506         .tcp.hdr = {
 507                 .src_port = RTE_BE16(0xffff),
 508                 .dst_port = RTE_BE16(0xffff),
 509                 .tcp_flags = 0xff,
 510         },
 511         .udp.hdr = {
 512                 .src_port = RTE_BE16(0xffff),
 513                 .dst_port = RTE_BE16(0xffff),
 514         },
 515         .vxlan = {
 516                .vni = "\xff\xff\xff",
 517         },
 518 };
 519
 520 #define SZ_NLATTR_HDR MNL_ALIGN(sizeof(struct nlattr))
 521 #define SZ_NLATTR_NEST SZ_NLATTR_HDR
 522 #define SZ_NLATTR_DATA_OF(len) MNL_ALIGN(SZ_NLATTR_HDR + (len))
 523 #define SZ_NLATTR_TYPE_OF(typ) SZ_NLATTR_DATA_OF(sizeof(typ))
 524 #define SZ_NLATTR_STRZ_OF(str) SZ_NLATTR_DATA_OF(strlen(str) + 1)
 525
 526 #define PTOI_TABLE_SZ_MAX(dev) (mlx5_dev_to_port_id((dev)->device, NULL, 0) + 2)
 527
 528 /** DPDK port to network interface index (ifindex) conversion. */
 529 struct flow_tcf_ptoi {
 530         uint16_t port_id; /**< DPDK port ID. */
 531         unsigned int ifindex; /**< Network interface index. */
 532 };
 533
 534 /* Due to a limitation on driver/FW. */
 535 #define MLX5_TCF_GROUP_ID_MAX 3
 536
 537 /*
 538  * Due to a limitation on driver/FW, priority ranges from 1 to 16 in kernel.
 539  * Priority in rte_flow attribute starts from 0 and is added by 1 in
 540  * translation. This is subject to be changed to determine the max priority
 541  * based on trial-and-error like Verbs driver once the restriction is lifted or
 542  * the range is extended.
 543  */
 544 #define MLX5_TCF_GROUP_PRIORITY_MAX 15
 545
 546 #define MLX5_TCF_FATE_ACTIONS \
 547         (MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_PORT_ID | \
 548          MLX5_FLOW_ACTION_JUMP)
 549
 550 #define MLX5_TCF_VLAN_ACTIONS \
 551         (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN | \
 552          MLX5_FLOW_ACTION_OF_SET_VLAN_VID | MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
 553
 554 #define MLX5_TCF_VXLAN_ACTIONS \
 555         (MLX5_FLOW_ACTION_VXLAN_ENCAP | MLX5_FLOW_ACTION_VXLAN_DECAP)
 556
 557 #define MLX5_TCF_PEDIT_ACTIONS \
 558         (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST | \
 559          MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST | \
 560          MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST | \
 561          MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL | \
 562          MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)
 563
 564 #define MLX5_TCF_CONFIG_ACTIONS \
 565         (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_JUMP | \
 566          MLX5_FLOW_ACTION_OF_PUSH_VLAN | MLX5_FLOW_ACTION_OF_SET_VLAN_VID | \
 567          MLX5_FLOW_ACTION_OF_SET_VLAN_PCP | \
 568          (MLX5_TCF_PEDIT_ACTIONS & ~MLX5_FLOW_ACTION_DEC_TTL))
 569
 570 #define MAX_PEDIT_KEYS 128
 571 #define SZ_PEDIT_KEY_VAL 4
 572
 573 #define NUM_OF_PEDIT_KEYS(sz) \
 574         (((sz) / SZ_PEDIT_KEY_VAL) + (((sz) % SZ_PEDIT_KEY_VAL) ? 1 : 0))
 575
 576 struct pedit_key_ex {
 577         enum pedit_header_type htype;
 578         enum pedit_cmd cmd;
 579 };
 580
 581 struct pedit_parser {
 582         struct tc_pedit_sel sel;
 583         struct tc_pedit_key keys[MAX_PEDIT_KEYS];
 584         struct pedit_key_ex keys_ex[MAX_PEDIT_KEYS];
 585 };
 586
 587 /**
 588  * Create space for using the implicitly created TC flow counter.
 589  *
 590  * @param[in] dev
 591  *   Pointer to the Ethernet device structure.
 592  *
 593  * @return
 594  *   A pointer to the counter data structure, NULL otherwise and
 595  *   rte_errno is set.
 596  */
 597 static struct mlx5_flow_counter *
 598 flow_tcf_counter_new(void)
 599 {
 600         struct mlx5_flow_counter *cnt;
 601
 602         /*
 603          * eswitch counter cannot be shared and its id is unknown.
 604          * currently returning all with id 0.
 605          * in the future maybe better to switch to unique numbers.
 606          */
 607         struct mlx5_flow_counter tmpl = {
 608                 .ref_cnt = 1,
 609         };
 610         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 611         if (!cnt) {
 612                 rte_errno = ENOMEM;
 613                 return NULL;
 614         }
 615         *cnt = tmpl;
 616         /* Implicit counter, do not add to list. */
 617         return cnt;
 618 }
 619
 620 /**
 621  * Set pedit key of MAC address
 622  *
 623  * @param[in] actions
 624  *   pointer to action specification
 625  * @param[in,out] p_parser
 626  *   pointer to pedit_parser
 627  */
 628 static void
 629 flow_tcf_pedit_key_set_mac(const struct rte_flow_action *actions,
 630                            struct pedit_parser *p_parser)
 631 {
 632         int idx = p_parser->sel.nkeys;
 633         uint32_t off = actions->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
 634                                         offsetof(struct ether_hdr, s_addr) :
 635                                         offsetof(struct ether_hdr, d_addr);
 636         const struct rte_flow_action_set_mac *conf =
 637                 (const struct rte_flow_action_set_mac *)actions->conf;
 638
 639         p_parser->keys[idx].off = off;
 640         p_parser->keys[idx].mask = ~UINT32_MAX;
 641         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 642         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 643         memcpy(&p_parser->keys[idx].val,
 644                 conf->mac_addr, SZ_PEDIT_KEY_VAL);
 645         idx++;
 646         p_parser->keys[idx].off = off + SZ_PEDIT_KEY_VAL;
 647         p_parser->keys[idx].mask = 0xFFFF0000;
 648         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 649         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 650         memcpy(&p_parser->keys[idx].val,
 651                 conf->mac_addr + SZ_PEDIT_KEY_VAL,
 652                 ETHER_ADDR_LEN - SZ_PEDIT_KEY_VAL);
 653         p_parser->sel.nkeys = (++idx);
 654 }
 655
 656 /**
 657  * Set pedit key of decrease/set ttl
 658  *
 659  * @param[in] actions
 660  *   pointer to action specification
 661  * @param[in,out] p_parser
 662  *   pointer to pedit_parser
 663  * @param[in] item_flags
 664  *   flags of all items presented
 665  */
 666 static void
 667 flow_tcf_pedit_key_set_dec_ttl(const struct rte_flow_action *actions,
 668                                 struct pedit_parser *p_parser,
 669                                 uint64_t item_flags)
 670 {
 671         int idx = p_parser->sel.nkeys;
 672
 673         p_parser->keys[idx].mask = 0xFFFFFF00;
 674         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) {
 675                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 676                 p_parser->keys[idx].off =
 677                         offsetof(struct ipv4_hdr, time_to_live);
 678         }
 679         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6) {
 680                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 681                 p_parser->keys[idx].off =
 682                         offsetof(struct ipv6_hdr, hop_limits);
 683         }
 684         if (actions->type == RTE_FLOW_ACTION_TYPE_DEC_TTL) {
 685                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_ADD;
 686                 p_parser->keys[idx].val = 0x000000FF;
 687         } else {
 688                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 689                 p_parser->keys[idx].val =
 690                         (__u32)((const struct rte_flow_action_set_ttl *)
 691                          actions->conf)->ttl_value;
 692         }
 693         p_parser->sel.nkeys = (++idx);
 694 }
 695
 696 /**
 697  * Set pedit key of transport (TCP/UDP) port value
 698  *
 699  * @param[in] actions
 700  *   pointer to action specification
 701  * @param[in,out] p_parser
 702  *   pointer to pedit_parser
 703  * @param[in] item_flags
 704  *   flags of all items presented
 705  */
 706 static void
 707 flow_tcf_pedit_key_set_tp_port(const struct rte_flow_action *actions,
 708                                 struct pedit_parser *p_parser,
 709                                 uint64_t item_flags)
 710 {
 711         int idx = p_parser->sel.nkeys;
 712
 713         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 714                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_UDP;
 715         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 716                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_TCP;
 717         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 718         /* offset of src/dst port is same for TCP and UDP */
 719         p_parser->keys[idx].off =
 720                 actions->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
 721                 offsetof(struct tcp_hdr, src_port) :
 722                 offsetof(struct tcp_hdr, dst_port);
 723         p_parser->keys[idx].mask = 0xFFFF0000;
 724         p_parser->keys[idx].val =
 725                 (__u32)((const struct rte_flow_action_set_tp *)
 726                                 actions->conf)->port;
 727         p_parser->sel.nkeys = (++idx);
 728 }
 729
 730 /**
 731  * Set pedit key of ipv6 address
 732  *
 733  * @param[in] actions
 734  *   pointer to action specification
 735  * @param[in,out] p_parser
 736  *   pointer to pedit_parser
 737  */
 738 static void
 739 flow_tcf_pedit_key_set_ipv6_addr(const struct rte_flow_action *actions,
 740                                  struct pedit_parser *p_parser)
 741 {
 742         int idx = p_parser->sel.nkeys;
 743         int keys = NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 744         int off_base =
 745                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
 746                 offsetof(struct ipv6_hdr, src_addr) :
 747                 offsetof(struct ipv6_hdr, dst_addr);
 748         const struct rte_flow_action_set_ipv6 *conf =
 749                 (const struct rte_flow_action_set_ipv6 *)actions->conf;
 750
 751         for (int i = 0; i < keys; i++, idx++) {
 752                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 753                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 754                 p_parser->keys[idx].off = off_base + i * SZ_PEDIT_KEY_VAL;
 755                 p_parser->keys[idx].mask = ~UINT32_MAX;
 756                 memcpy(&p_parser->keys[idx].val,
 757                         conf->ipv6_addr + i *  SZ_PEDIT_KEY_VAL,
 758                         SZ_PEDIT_KEY_VAL);
 759         }
 760         p_parser->sel.nkeys += keys;
 761 }
 762
 763 /**
 764  * Set pedit key of ipv4 address
 765  *
 766  * @param[in] actions
 767  *   pointer to action specification
 768  * @param[in,out] p_parser
 769  *   pointer to pedit_parser
 770  */
 771 static void
 772 flow_tcf_pedit_key_set_ipv4_addr(const struct rte_flow_action *actions,
 773                                  struct pedit_parser *p_parser)
 774 {
 775         int idx = p_parser->sel.nkeys;
 776
 777         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 778         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 779         p_parser->keys[idx].off =
 780                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
 781                 offsetof(struct ipv4_hdr, src_addr) :
 782                 offsetof(struct ipv4_hdr, dst_addr);
 783         p_parser->keys[idx].mask = ~UINT32_MAX;
 784         p_parser->keys[idx].val =
 785                 ((const struct rte_flow_action_set_ipv4 *)
 786                  actions->conf)->ipv4_addr;
 787         p_parser->sel.nkeys = (++idx);
 788 }
 789
 790 /**
 791  * Create the pedit's na attribute in netlink message
 792  * on pre-allocate message buffer
 793  *
 794  * @param[in,out] nl
 795  *   pointer to pre-allocated netlink message buffer
 796  * @param[in,out] actions
 797  *   pointer to pointer of actions specification.
 798  * @param[in,out] action_flags
 799  *   pointer to actions flags
 800  * @param[in] item_flags
 801  *   flags of all item presented
 802  */
 803 static void
 804 flow_tcf_create_pedit_mnl_msg(struct nlmsghdr *nl,
 805                               const struct rte_flow_action **actions,
 806                               uint64_t item_flags)
 807 {
 808         struct pedit_parser p_parser;
 809         struct nlattr *na_act_options;
 810         struct nlattr *na_pedit_keys;
 811
 812         memset(&p_parser, 0, sizeof(p_parser));
 813         mnl_attr_put_strz(nl, TCA_ACT_KIND, "pedit");
 814         na_act_options = mnl_attr_nest_start(nl, TCA_ACT_OPTIONS);
 815         /* all modify header actions should be in one tc-pedit action */
 816         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 817                 switch ((*actions)->type) {
 818                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 819                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 820                         flow_tcf_pedit_key_set_ipv4_addr(*actions, &p_parser);
 821                         break;
 822                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 823                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 824                         flow_tcf_pedit_key_set_ipv6_addr(*actions, &p_parser);
 825                         break;
 826                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 827                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 828                         flow_tcf_pedit_key_set_tp_port(*actions,
 829                                                         &p_parser, item_flags);
 830                         break;
 831                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 832                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 833                         flow_tcf_pedit_key_set_dec_ttl(*actions,
 834                                                         &p_parser, item_flags);
 835                         break;
 836                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 837                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 838                         flow_tcf_pedit_key_set_mac(*actions, &p_parser);
 839                         break;
 840                 default:
 841                         goto pedit_mnl_msg_done;
 842                 }
 843         }
 844 pedit_mnl_msg_done:
 845         p_parser.sel.action = TC_ACT_PIPE;
 846         mnl_attr_put(nl, TCA_PEDIT_PARMS_EX,
 847                      sizeof(p_parser.sel) +
 848                      p_parser.sel.nkeys * sizeof(struct tc_pedit_key),
 849                      &p_parser);
 850         na_pedit_keys =
 851                 mnl_attr_nest_start(nl, TCA_PEDIT_KEYS_EX | NLA_F_NESTED);
 852         for (int i = 0; i < p_parser.sel.nkeys; i++) {
 853                 struct nlattr *na_pedit_key =
 854                         mnl_attr_nest_start(nl,
 855                                             TCA_PEDIT_KEY_EX | NLA_F_NESTED);
 856                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_HTYPE,
 857                                  p_parser.keys_ex[i].htype);
 858                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_CMD,
 859                                  p_parser.keys_ex[i].cmd);
 860                 mnl_attr_nest_end(nl, na_pedit_key);
 861         }
 862         mnl_attr_nest_end(nl, na_pedit_keys);
 863         mnl_attr_nest_end(nl, na_act_options);
 864         (*actions)--;
 865 }
 866
 867 /**
 868  * Calculate max memory size of one TC-pedit actions.
 869  * One TC-pedit action can contain set of keys each defining
 870  * a rewrite element (rte_flow action)
 871  *
 872  * @param[in,out] actions
 873  *   actions specification.
 874  * @param[in,out] action_flags
 875  *   actions flags
 876  * @param[in,out] size
 877  *   accumulated size
 878  * @return
 879  *   Max memory size of one TC-pedit action
 880  */
 881 static int
 882 flow_tcf_get_pedit_actions_size(const struct rte_flow_action **actions,
 883                                 uint64_t *action_flags)
 884 {
 885         int pedit_size = 0;
 886         int keys = 0;
 887         uint64_t flags = 0;
 888
 889         pedit_size += SZ_NLATTR_NEST + /* na_act_index. */
 890                       SZ_NLATTR_STRZ_OF("pedit") +
 891                       SZ_NLATTR_NEST; /* TCA_ACT_OPTIONS. */
 892         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 893                 switch ((*actions)->type) {
 894                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 895                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 896                         flags |= MLX5_FLOW_ACTION_SET_IPV4_SRC;
 897                         break;
 898                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 899                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 900                         flags |= MLX5_FLOW_ACTION_SET_IPV4_DST;
 901                         break;
 902                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 903                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 904                         flags |= MLX5_FLOW_ACTION_SET_IPV6_SRC;
 905                         break;
 906                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 907                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 908                         flags |= MLX5_FLOW_ACTION_SET_IPV6_DST;
 909                         break;
 910                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 911                         /* TCP is as same as UDP */
 912                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 913                         flags |= MLX5_FLOW_ACTION_SET_TP_SRC;
 914                         break;
 915                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 916                         /* TCP is as same as UDP */
 917                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 918                         flags |= MLX5_FLOW_ACTION_SET_TP_DST;
 919                         break;
 920                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 921                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 922                         flags |= MLX5_FLOW_ACTION_SET_TTL;
 923                         break;
 924                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 925                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 926                         flags |= MLX5_FLOW_ACTION_DEC_TTL;
 927                         break;
 928                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 929                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 930                         flags |= MLX5_FLOW_ACTION_SET_MAC_SRC;
 931                         break;
 932                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 933                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 934                         flags |= MLX5_FLOW_ACTION_SET_MAC_DST;
 935                         break;
 936                 default:
 937                         goto get_pedit_action_size_done;
 938                 }
 939         }
 940 get_pedit_action_size_done:
 941         /* TCA_PEDIT_PARAMS_EX */
 942         pedit_size +=
 943                 SZ_NLATTR_DATA_OF(sizeof(struct tc_pedit_sel) +
 944                                   keys * sizeof(struct tc_pedit_key));
 945         pedit_size += SZ_NLATTR_NEST; /* TCA_PEDIT_KEYS */
 946         pedit_size += keys *
 947                       /* TCA_PEDIT_KEY_EX + HTYPE + CMD */
 948                       (SZ_NLATTR_NEST + SZ_NLATTR_DATA_OF(2) +
 949                        SZ_NLATTR_DATA_OF(2));
 950         (*action_flags) |= flags;
 951         (*actions)--;
 952         return pedit_size;
 953 }
 954
 955 /**
 956  * Retrieve mask for pattern item.
 957  *
 958  * This function does basic sanity checks on a pattern item in order to
 959  * return the most appropriate mask for it.
 960  *
 961  * @param[in] item
 962  *   Item specification.
 963  * @param[in] mask_default
 964  *   Default mask for pattern item as specified by the flow API.
 965  * @param[in] mask_supported
 966  *   Mask fields supported by the implementation.
 967  * @param[in] mask_empty
 968  *   Empty mask to return when there is no specification.
 969  * @param[out] error
 970  *   Perform verbose error reporting if not NULL.
 971  *
 972  * @return
 973  *   Either @p item->mask or one of the mask parameters on success, NULL
 974  *   otherwise and rte_errno is set.
 975  */
 976 static const void *
 977 flow_tcf_item_mask(const struct rte_flow_item *item, const void *mask_default,
 978                    const void *mask_supported, const void *mask_empty,
 979                    size_t mask_size, struct rte_flow_error *error)
 980 {
 981         const uint8_t *mask;
 982         size_t i;
 983
 984         /* item->last and item->mask cannot exist without item->spec. */
 985         if (!item->spec && (item->mask || item->last)) {
 986                 rte_flow_error_set(error, EINVAL,
 987                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 988                                    "\"mask\" or \"last\" field provided without"
 989                                    " a corresponding \"spec\"");
 990                 return NULL;
 991         }
 992         /* No spec, no mask, no problem. */
 993         if (!item->spec)
 994                 return mask_empty;
 995         mask = item->mask ? item->mask : mask_default;
 996         assert(mask);
 997         /*
 998          * Single-pass check to make sure that:
 999          * - Mask is supported, no bits are set outside mask_supported.
1000          * - Both item->spec and item->last are included in mask.
1001          */
1002         for (i = 0; i != mask_size; ++i) {
1003                 if (!mask[i])
1004                         continue;
1005                 if ((mask[i] | ((const uint8_t *)mask_supported)[i]) !=
1006                     ((const uint8_t *)mask_supported)[i]) {
1007                         rte_flow_error_set(error, ENOTSUP,
1008                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1009                                            "unsupported field found"
1010                                            " in \"mask\"");
1011                         return NULL;
1012                 }
1013                 if (item->last &&
1014                     (((const uint8_t *)item->spec)[i] & mask[i]) !=
1015                     (((const uint8_t *)item->last)[i] & mask[i])) {
1016                         rte_flow_error_set(error, EINVAL,
1017                                            RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1018                                            item->last,
1019                                            "range between \"spec\" and \"last\""
1020                                            " not comprised in \"mask\"");
1021                         return NULL;
1022                 }
1023         }
1024         return mask;
1025 }
1026
1027 /**
1028  * Build a conversion table between port ID and ifindex.
1029  *
1030  * @param[in] dev
1031  *   Pointer to Ethernet device.
1032  * @param[out] ptoi
1033  *   Pointer to ptoi table.
1034  * @param[in] len
1035  *   Size of ptoi table provided.
1036  *
1037  * @return
1038  *   Size of ptoi table filled.
1039  */
1040 static unsigned int
1041 flow_tcf_build_ptoi_table(struct rte_eth_dev *dev, struct flow_tcf_ptoi *ptoi,
1042                           unsigned int len)
1043 {
1044         unsigned int n = mlx5_dev_to_port_id(dev->device, NULL, 0);
1045         uint16_t port_id[n + 1];
1046         unsigned int i;
1047         unsigned int own = 0;
1048
1049         /* At least one port is needed when no switch domain is present. */
1050         if (!n) {
1051                 n = 1;
1052                 port_id[0] = dev->data->port_id;
1053         } else {
1054                 n = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, n), n);
1055         }
1056         if (n > len)
1057                 return 0;
1058         for (i = 0; i != n; ++i) {
1059                 struct rte_eth_dev_info dev_info;
1060
1061                 rte_eth_dev_info_get(port_id[i], &dev_info);
1062                 if (port_id[i] == dev->data->port_id)
1063                         own = i;
1064                 ptoi[i].port_id = port_id[i];
1065                 ptoi[i].ifindex = dev_info.if_index;
1066         }
1067         /* Ensure first entry of ptoi[] is the current device. */
1068         if (own) {
1069                 ptoi[n] = ptoi[0];
1070                 ptoi[0] = ptoi[own];
1071                 ptoi[own] = ptoi[n];
1072         }
1073         /* An entry with zero ifindex terminates ptoi[]. */
1074         ptoi[n].port_id = 0;
1075         ptoi[n].ifindex = 0;
1076         return n;
1077 }
1078
1079 /**
1080  * Verify the @p attr will be correctly understood by the E-switch.
1081  *
1082  * @param[in] attr
1083  *   Pointer to flow attributes
1084  * @param[out] error
1085  *   Pointer to error structure.
1086  *
1087  * @return
1088  *   0 on success, a negative errno value otherwise and rte_errno is set.
1089  */
1090 static int
1091 flow_tcf_validate_attributes(const struct rte_flow_attr *attr,
1092                              struct rte_flow_error *error)
1093 {
1094         /*
1095          * Supported attributes: groups, some priorities and ingress only.
1096          * group is supported only if kernel supports chain. Don't care about
1097          * transfer as it is the caller's problem.
1098          */
1099         if (attr->group > MLX5_TCF_GROUP_ID_MAX)
1100                 return rte_flow_error_set(error, ENOTSUP,
1101                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
1102                                           "group ID larger than "
1103                                           RTE_STR(MLX5_TCF_GROUP_ID_MAX)
1104                                           " isn't supported");
1105         else if (attr->priority > MLX5_TCF_GROUP_PRIORITY_MAX)
1106                 return rte_flow_error_set(error, ENOTSUP,
1107                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1108                                           attr,
1109                                           "priority more than "
1110                                           RTE_STR(MLX5_TCF_GROUP_PRIORITY_MAX)
1111                                           " is not supported");
1112         if (!attr->ingress)
1113                 return rte_flow_error_set(error, EINVAL,
1114                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1115                                           attr, "only ingress is supported");
1116         if (attr->egress)
1117                 return rte_flow_error_set(error, ENOTSUP,
1118                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1119                                           attr, "egress is not supported");
1120         return 0;
1121 }
1122
1123 /**
1124  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_ETH item for E-Switch.
1125  * The routine checks the L2 fields to be used in encapsulation header.
1126  *
1127  * @param[in] item
1128  *   Pointer to the item structure.
1129  * @param[out] error
1130  *   Pointer to the error structure.
1131  *
1132  * @return
1133  *   0 on success, a negative errno value otherwise and rte_errno is set.
1134  **/
1135 static int
1136 flow_tcf_validate_vxlan_encap_eth(const struct rte_flow_item *item,
1137                                   struct rte_flow_error *error)
1138 {
1139         const struct rte_flow_item_eth *spec = item->spec;
1140         const struct rte_flow_item_eth *mask = item->mask;
1141
1142         if (!spec) {
1143                 /*
1144                  * Specification for L2 addresses can be empty
1145                  * because these ones are optional and not
1146                  * required directly by tc rule. Kernel tries
1147                  * to resolve these ones on its own
1148                  */
1149                 return 0;
1150         }
1151         if (!mask) {
1152                 /* If mask is not specified use the default one. */
1153                 mask = &rte_flow_item_eth_mask;
1154         }
1155         if (memcmp(&mask->dst,
1156                    &flow_tcf_mask_empty.eth.dst,
1157                    sizeof(flow_tcf_mask_empty.eth.dst))) {
1158                 if (memcmp(&mask->dst,
1159                            &rte_flow_item_eth_mask.dst,
1160                            sizeof(rte_flow_item_eth_mask.dst)))
1161                         return rte_flow_error_set
1162                                 (error, ENOTSUP,
1163                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1164                                  "no support for partial mask on"
1165                                  " \"eth.dst\" field");
1166         }
1167         if (memcmp(&mask->src,
1168                    &flow_tcf_mask_empty.eth.src,
1169                    sizeof(flow_tcf_mask_empty.eth.src))) {
1170                 if (memcmp(&mask->src,
1171                            &rte_flow_item_eth_mask.src,
1172                            sizeof(rte_flow_item_eth_mask.src)))
1173                         return rte_flow_error_set
1174                                 (error, ENOTSUP,
1175                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1176                                  "no support for partial mask on"
1177                                  " \"eth.src\" field");
1178         }
1179         if (mask->type != RTE_BE16(0x0000)) {
1180                 if (mask->type != RTE_BE16(0xffff))
1181                         return rte_flow_error_set
1182                                 (error, ENOTSUP,
1183                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1184                                  "no support for partial mask on"
1185                                  " \"eth.type\" field");
1186                 DRV_LOG(WARNING,
1187                         "outer ethernet type field"
1188                         " cannot be forced for vxlan"
1189                         " encapsulation, parameter ignored");
1190         }
1191         return 0;
1192 }
1193
1194 /**
1195  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV4 item for E-Switch.
1196  * The routine checks the IPv4 fields to be used in encapsulation header.
1197  *
1198  * @param[in] item
1199  *   Pointer to the item structure.
1200  * @param[out] error
1201  *   Pointer to the error structure.
1202  *
1203  * @return
1204  *   0 on success, a negative errno value otherwise and rte_errno is set.
1205  **/
1206 static int
1207 flow_tcf_validate_vxlan_encap_ipv4(const struct rte_flow_item *item,
1208                                    struct rte_flow_error *error)
1209 {
1210         const struct rte_flow_item_ipv4 *spec = item->spec;
1211         const struct rte_flow_item_ipv4 *mask = item->mask;
1212
1213         if (!spec) {
1214                 /*
1215                  * Specification for IP addresses cannot be empty
1216                  * because it is required by tunnel_key parameter.
1217                  */
1218                 return rte_flow_error_set(error, EINVAL,
1219                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1220                                           "NULL outer ipv4 address"
1221                                           " specification for vxlan"
1222                                           " encapsulation");
1223         }
1224         if (!mask)
1225                 mask = &rte_flow_item_ipv4_mask;
1226         if (mask->hdr.dst_addr != RTE_BE32(0x00000000)) {
1227                 if (mask->hdr.dst_addr != RTE_BE32(0xffffffff))
1228                         return rte_flow_error_set
1229                                 (error, ENOTSUP,
1230                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1231                                  "no support for partial mask on"
1232                                  " \"ipv4.hdr.dst_addr\" field"
1233                                  " for vxlan encapsulation");
1234                 /* More IPv4 address validations can be put here. */
1235         } else {
1236                 /*
1237                  * Kernel uses the destination IP address to determine
1238                  * the routing path and obtain the MAC destination
1239                  * address, so IP destination address must be
1240                  * specified in the tc rule.
1241                  */
1242                 return rte_flow_error_set(error, EINVAL,
1243                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1244                                           "outer ipv4 destination address"
1245                                           " must be specified for"
1246                                           " vxlan encapsulation");
1247         }
1248         if (mask->hdr.src_addr != RTE_BE32(0x00000000)) {
1249                 if (mask->hdr.src_addr != RTE_BE32(0xffffffff))
1250                         return rte_flow_error_set
1251                                 (error, ENOTSUP,
1252                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1253                                  "no support for partial mask on"
1254                                  " \"ipv4.hdr.src_addr\" field"
1255                                  " for vxlan encapsulation");
1256                 /* More IPv4 address validations can be put here. */
1257         } else {
1258                 /*
1259                  * Kernel uses the source IP address to select the
1260                  * interface for egress encapsulated traffic, so
1261                  * it must be specified in the tc rule.
1262                  */
1263                 return rte_flow_error_set(error, EINVAL,
1264                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1265                                           "outer ipv4 source address"
1266                                           " must be specified for"
1267                                           " vxlan encapsulation");
1268         }
1269         return 0;
1270 }
1271
1272 /**
1273  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV6 item for E-Switch.
1274  * The routine checks the IPv6 fields to be used in encapsulation header.
1275  *
1276  * @param[in] item
1277  *   Pointer to the item structure.
1278  * @param[out] error
1279  *   Pointer to the error structure.
1280  *
1281  * @return
1282  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1283  **/
1284 static int
1285 flow_tcf_validate_vxlan_encap_ipv6(const struct rte_flow_item *item,
1286                                    struct rte_flow_error *error)
1287 {
1288         const struct rte_flow_item_ipv6 *spec = item->spec;
1289         const struct rte_flow_item_ipv6 *mask = item->mask;
1290
1291         if (!spec) {
1292                 /*
1293                  * Specification for IP addresses cannot be empty
1294                  * because it is required by tunnel_key parameter.
1295                  */
1296                 return rte_flow_error_set(error, EINVAL,
1297                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1298                                           "NULL outer ipv6 address"
1299                                           " specification for"
1300                                           " vxlan encapsulation");
1301         }
1302         if (!mask)
1303                 mask = &rte_flow_item_ipv6_mask;
1304         if (memcmp(&mask->hdr.dst_addr,
1305                    &flow_tcf_mask_empty.ipv6.hdr.dst_addr,
1306                    IPV6_ADDR_LEN)) {
1307                 if (memcmp(&mask->hdr.dst_addr,
1308                            &rte_flow_item_ipv6_mask.hdr.dst_addr,
1309                            IPV6_ADDR_LEN))
1310                         return rte_flow_error_set
1311                                         (error, ENOTSUP,
1312                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1313                                          "no support for partial mask on"
1314                                          " \"ipv6.hdr.dst_addr\" field"
1315                                          " for vxlan encapsulation");
1316                 /* More IPv6 address validations can be put here. */
1317         } else {
1318                 /*
1319                  * Kernel uses the destination IP address to determine
1320                  * the routing path and obtain the MAC destination
1321                  * address (heigh or gate), so IP destination address
1322                  * must be specified within the tc rule.
1323                  */
1324                 return rte_flow_error_set(error, EINVAL,
1325                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1326                                           "outer ipv6 destination address"
1327                                           " must be specified for"
1328                                           " vxlan encapsulation");
1329         }
1330         if (memcmp(&mask->hdr.src_addr,
1331                    &flow_tcf_mask_empty.ipv6.hdr.src_addr,
1332                    IPV6_ADDR_LEN)) {
1333                 if (memcmp(&mask->hdr.src_addr,
1334                            &rte_flow_item_ipv6_mask.hdr.src_addr,
1335                            IPV6_ADDR_LEN))
1336                         return rte_flow_error_set
1337                                         (error, ENOTSUP,
1338                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1339                                          "no support for partial mask on"
1340                                          " \"ipv6.hdr.src_addr\" field"
1341                                          " for vxlan encapsulation");
1342                 /* More L3 address validation can be put here. */
1343         } else {
1344                 /*
1345                  * Kernel uses the source IP address to select the
1346                  * interface for egress encapsulated traffic, so
1347                  * it must be specified in the tc rule.
1348                  */
1349                 return rte_flow_error_set(error, EINVAL,
1350                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1351                                           "outer L3 source address"
1352                                           " must be specified for"
1353                                           " vxlan encapsulation");
1354         }
1355         return 0;
1356 }
1357
1358 /**
1359  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_UDP item for E-Switch.
1360  * The routine checks the UDP fields to be used in encapsulation header.
1361  *
1362  * @param[in] item
1363  *   Pointer to the item structure.
1364  * @param[out] error
1365  *   Pointer to the error structure.
1366  *
1367  * @return
1368  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1369  **/
1370 static int
1371 flow_tcf_validate_vxlan_encap_udp(const struct rte_flow_item *item,
1372                                   struct rte_flow_error *error)
1373 {
1374         const struct rte_flow_item_udp *spec = item->spec;
1375         const struct rte_flow_item_udp *mask = item->mask;
1376
1377         if (!spec) {
1378                 /*
1379                  * Specification for UDP ports cannot be empty
1380                  * because it is required by tunnel_key parameter.
1381                  */
1382                 return rte_flow_error_set(error, EINVAL,
1383                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1384                                           "NULL UDP port specification "
1385                                           " for vxlan encapsulation");
1386         }
1387         if (!mask)
1388                 mask = &rte_flow_item_udp_mask;
1389         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1390                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1391                         return rte_flow_error_set
1392                                         (error, ENOTSUP,
1393                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1394                                          "no support for partial mask on"
1395                                          " \"udp.hdr.dst_port\" field"
1396                                          " for vxlan encapsulation");
1397                 if (!spec->hdr.dst_port)
1398                         return rte_flow_error_set
1399                                         (error, EINVAL,
1400                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
1401                                          "outer UDP remote port cannot be"
1402                                          " 0 for vxlan encapsulation");
1403         } else {
1404                 return rte_flow_error_set(error, EINVAL,
1405                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1406                                           "outer UDP remote port"
1407                                           " must be specified for"
1408                                           " vxlan encapsulation");
1409         }
1410         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1411                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1412                         return rte_flow_error_set
1413                                         (error, ENOTSUP,
1414                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1415                                          "no support for partial mask on"
1416                                          " \"udp.hdr.src_port\" field"
1417                                          " for vxlan encapsulation");
1418                 DRV_LOG(WARNING,
1419                         "outer UDP source port cannot be"
1420                         " forced for vxlan encapsulation,"
1421                         " parameter ignored");
1422         }
1423         return 0;
1424 }
1425
1426 /**
1427  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_VXLAN item for E-Switch.
1428  * The routine checks the VNIP fields to be used in encapsulation header.
1429  *
1430  * @param[in] item
1431  *   Pointer to the item structure.
1432  * @param[out] error
1433  *   Pointer to the error structure.
1434  *
1435  * @return
1436  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1437  **/
1438 static int
1439 flow_tcf_validate_vxlan_encap_vni(const struct rte_flow_item *item,
1440                                   struct rte_flow_error *error)
1441 {
1442         const struct rte_flow_item_vxlan *spec = item->spec;
1443         const struct rte_flow_item_vxlan *mask = item->mask;
1444
1445         if (!spec) {
1446                 /* Outer VNI is required by tunnel_key parameter. */
1447                 return rte_flow_error_set(error, EINVAL,
1448                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1449                                           "NULL VNI specification"
1450                                           " for vxlan encapsulation");
1451         }
1452         if (!mask)
1453                 mask = &rte_flow_item_vxlan_mask;
1454         if (!mask->vni[0] && !mask->vni[1] && !mask->vni[2])
1455                 return rte_flow_error_set(error, EINVAL,
1456                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1457                                           "outer VNI must be specified "
1458                                           "for vxlan encapsulation");
1459         if (mask->vni[0] != 0xff ||
1460             mask->vni[1] != 0xff ||
1461             mask->vni[2] != 0xff)
1462                 return rte_flow_error_set(error, ENOTSUP,
1463                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1464                                           "no support for partial mask on"
1465                                           " \"vxlan.vni\" field");
1466
1467         if (!spec->vni[0] && !spec->vni[1] && !spec->vni[2])
1468                 return rte_flow_error_set(error, EINVAL,
1469                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1470                                           "vxlan vni cannot be 0");
1471         return 0;
1472 }
1473
1474 /**
1475  * Validate VXLAN_ENCAP action item list for E-Switch.
1476  * The routine checks items to be used in encapsulation header.
1477  *
1478  * @param[in] action
1479  *   Pointer to the VXLAN_ENCAP action structure.
1480  * @param[out] error
1481  *   Pointer to the error structure.
1482  *
1483  * @return
1484  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1485  **/
1486 static int
1487 flow_tcf_validate_vxlan_encap(const struct rte_flow_action *action,
1488                               struct rte_flow_error *error)
1489 {
1490         const struct rte_flow_item *items;
1491         int ret;
1492         uint32_t item_flags = 0;
1493
1494         if (!action->conf)
1495                 return rte_flow_error_set(error, EINVAL,
1496                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1497                                           "Missing vxlan tunnel"
1498                                           " action configuration");
1499         items = ((const struct rte_flow_action_vxlan_encap *)
1500                                         action->conf)->definition;
1501         if (!items)
1502                 return rte_flow_error_set(error, EINVAL,
1503                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1504                                           "Missing vxlan tunnel"
1505                                           " encapsulation parameters");
1506         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1507                 switch (items->type) {
1508                 case RTE_FLOW_ITEM_TYPE_VOID:
1509                         break;
1510                 case RTE_FLOW_ITEM_TYPE_ETH:
1511                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1512                                                           error);
1513                         if (ret < 0)
1514                                 return ret;
1515                         ret = flow_tcf_validate_vxlan_encap_eth(items, error);
1516                         if (ret < 0)
1517                                 return ret;
1518                         item_flags |= MLX5_FLOW_LAYER_OUTER_L2;
1519                         break;
1520                 break;
1521                 case RTE_FLOW_ITEM_TYPE_IPV4:
1522                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1523                                                            error);
1524                         if (ret < 0)
1525                                 return ret;
1526                         ret = flow_tcf_validate_vxlan_encap_ipv4(items, error);
1527                         if (ret < 0)
1528                                 return ret;
1529                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1530                         break;
1531                 case RTE_FLOW_ITEM_TYPE_IPV6:
1532                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1533                                                            error);
1534                         if (ret < 0)
1535                                 return ret;
1536                         ret = flow_tcf_validate_vxlan_encap_ipv6(items, error);
1537                         if (ret < 0)
1538                                 return ret;
1539                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1540                         break;
1541                 case RTE_FLOW_ITEM_TYPE_UDP:
1542                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1543                                                            0xFF, error);
1544                         if (ret < 0)
1545                                 return ret;
1546                         ret = flow_tcf_validate_vxlan_encap_udp(items, error);
1547                         if (ret < 0)
1548                                 return ret;
1549                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
1550                         break;
1551                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1552                         ret = mlx5_flow_validate_item_vxlan(items,
1553                                                             item_flags, error);
1554                         if (ret < 0)
1555                                 return ret;
1556                         ret = flow_tcf_validate_vxlan_encap_vni(items, error);
1557                         if (ret < 0)
1558                                 return ret;
1559                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1560                         break;
1561                 default:
1562                         return rte_flow_error_set
1563                                         (error, ENOTSUP,
1564                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1565                                          "vxlan encap item not supported");
1566                 }
1567         }
1568         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
1569                 return rte_flow_error_set(error, EINVAL,
1570                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1571                                           "no outer IP layer found"
1572                                           " for vxlan encapsulation");
1573         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1574                 return rte_flow_error_set(error, EINVAL,
1575                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1576                                           "no outer UDP layer found"
1577                                           " for vxlan encapsulation");
1578         if (!(item_flags & MLX5_FLOW_LAYER_VXLAN))
1579                 return rte_flow_error_set(error, EINVAL,
1580                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1581                                           "no VXLAN VNI found"
1582                                           " for vxlan encapsulation");
1583         return 0;
1584 }
1585
1586 /**
1587  * Validate RTE_FLOW_ITEM_TYPE_IPV4 item if VXLAN_DECAP action
1588  * is present in actions list.
1589  *
1590  * @param[in] ipv4
1591  *   Outer IPv4 address item (if any, NULL otherwise).
1592  * @param[out] error
1593  *   Pointer to the error structure.
1594  *
1595  * @return
1596  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1597  **/
1598 static int
1599 flow_tcf_validate_vxlan_decap_ipv4(const struct rte_flow_item *ipv4,
1600                                    struct rte_flow_error *error)
1601 {
1602         const struct rte_flow_item_ipv4 *spec = ipv4->spec;
1603         const struct rte_flow_item_ipv4 *mask = ipv4->mask;
1604
1605         if (!spec) {
1606                 /*
1607                  * Specification for IP addresses cannot be empty
1608                  * because it is required as decap parameter.
1609                  */
1610                 return rte_flow_error_set(error, EINVAL,
1611                                           RTE_FLOW_ERROR_TYPE_ITEM, ipv4,
1612                                           "NULL outer ipv4 address"
1613                                           " specification for vxlan"
1614                                           " for vxlan decapsulation");
1615         }
1616         if (!mask)
1617                 mask = &rte_flow_item_ipv4_mask;
1618         if (mask->hdr.dst_addr != RTE_BE32(0x00000000)) {
1619                 if (mask->hdr.dst_addr != RTE_BE32(0xffffffff))
1620                         return rte_flow_error_set
1621                                         (error, ENOTSUP,
1622                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1623                                          "no support for partial mask on"
1624                                          " \"ipv4.hdr.dst_addr\" field");
1625                 /* More IP address validations can be put here. */
1626         } else {
1627                 /*
1628                  * Kernel uses the destination IP address
1629                  * to determine the ingress network interface
1630                  * for traffic being decapsulated.
1631                  */
1632                 return rte_flow_error_set(error, EINVAL,
1633                                           RTE_FLOW_ERROR_TYPE_ITEM, ipv4,
1634                                           "outer ipv4 destination address"
1635                                           " must be specified for"
1636                                           " vxlan decapsulation");
1637         }
1638         /* Source IP address is optional for decap. */
1639         if (mask->hdr.src_addr != RTE_BE32(0x00000000) &&
1640             mask->hdr.src_addr != RTE_BE32(0xffffffff))
1641                 return rte_flow_error_set(error, ENOTSUP,
1642                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1643                                           "no support for partial mask on"
1644                                           " \"ipv4.hdr.src_addr\" field");
1645         return 0;
1646 }
1647
1648 /**
1649  * Validate RTE_FLOW_ITEM_TYPE_IPV6 item if VXLAN_DECAP action
1650  * is present in actions list.
1651  *
1652  * @param[in] ipv6
1653  *   Outer IPv6 address item (if any, NULL otherwise).
1654  * @param[out] error
1655  *   Pointer to the error structure.
1656  *
1657  * @return
1658  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1659  **/
1660 static int
1661 flow_tcf_validate_vxlan_decap_ipv6(const struct rte_flow_item *ipv6,
1662                                    struct rte_flow_error *error)
1663 {
1664         const struct rte_flow_item_ipv6 *spec = ipv6->spec;
1665         const struct rte_flow_item_ipv6 *mask = ipv6->mask;
1666
1667         if (!spec) {
1668                 /*
1669                  * Specification for IP addresses cannot be empty
1670                  * because it is required as decap parameter.
1671                  */
1672                 return rte_flow_error_set(error, EINVAL,
1673                                           RTE_FLOW_ERROR_TYPE_ITEM, ipv6,
1674                                           "NULL outer ipv6 address"
1675                                           " specification for vxlan"
1676                                           " decapsulation");
1677         }
1678         if (!mask)
1679                 mask = &rte_flow_item_ipv6_mask;
1680         if (memcmp(&mask->hdr.dst_addr,
1681                    &flow_tcf_mask_empty.ipv6.hdr.dst_addr,
1682                    IPV6_ADDR_LEN)) {
1683                 if (memcmp(&mask->hdr.dst_addr,
1684                         &rte_flow_item_ipv6_mask.hdr.dst_addr,
1685                         IPV6_ADDR_LEN))
1686                         return rte_flow_error_set
1687                                         (error, ENOTSUP,
1688                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1689                                          "no support for partial mask on"
1690                                          " \"ipv6.hdr.dst_addr\" field");
1691                 /* More IP address validations can be put here. */
1692         } else {
1693                 /*
1694                  * Kernel uses the destination IP address
1695                  * to determine the ingress network interface
1696                  * for traffic being decapsulated.
1697                  */
1698                 return rte_flow_error_set(error, EINVAL,
1699                                           RTE_FLOW_ERROR_TYPE_ITEM, ipv6,
1700                                           "outer ipv6 destination address must be "
1701                                           "specified for vxlan decapsulation");
1702         }
1703         /* Source IP address is optional for decap. */
1704         if (memcmp(&mask->hdr.src_addr,
1705                    &flow_tcf_mask_empty.ipv6.hdr.src_addr,
1706                    IPV6_ADDR_LEN)) {
1707                 if (memcmp(&mask->hdr.src_addr,
1708                            &rte_flow_item_ipv6_mask.hdr.src_addr,
1709                            IPV6_ADDR_LEN))
1710                         return rte_flow_error_set
1711                                         (error, ENOTSUP,
1712                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1713                                          "no support for partial mask on"
1714                                          " \"ipv6.hdr.src_addr\" field");
1715         }
1716         return 0;
1717 }
1718
1719 /**
1720  * Validate RTE_FLOW_ITEM_TYPE_UDP item if VXLAN_DECAP action
1721  * is present in actions list.
1722  *
1723  * @param[in] udp
1724  *   Outer UDP layer item (if any, NULL otherwise).
1725  * @param[out] error
1726  *   Pointer to the error structure.
1727  *
1728  * @return
1729  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1730  **/
1731 static int
1732 flow_tcf_validate_vxlan_decap_udp(const struct rte_flow_item *udp,
1733                                   struct rte_flow_error *error)
1734 {
1735         const struct rte_flow_item_udp *spec = udp->spec;
1736         const struct rte_flow_item_udp *mask = udp->mask;
1737
1738         if (!spec)
1739                 /*
1740                  * Specification for UDP ports cannot be empty
1741                  * because it is required as decap parameter.
1742                  */
1743                 return rte_flow_error_set(error, EINVAL,
1744                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1745                                           "NULL UDP port specification"
1746                                           " for VXLAN decapsulation");
1747         if (!mask)
1748                 mask = &rte_flow_item_udp_mask;
1749         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1750                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1751                         return rte_flow_error_set
1752                                         (error, ENOTSUP,
1753                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1754                                          "no support for partial mask on"
1755                                          " \"udp.hdr.dst_port\" field");
1756                 if (!spec->hdr.dst_port)
1757                         return rte_flow_error_set
1758                                         (error, EINVAL,
1759                                          RTE_FLOW_ERROR_TYPE_ITEM, udp,
1760                                          "zero decap local UDP port");
1761         } else {
1762                 return rte_flow_error_set(error, EINVAL,
1763                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1764                                           "outer UDP destination port must be "
1765                                           "specified for vxlan decapsulation");
1766         }
1767         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1768                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1769                         return rte_flow_error_set
1770                                         (error, ENOTSUP,
1771                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1772                                          "no support for partial mask on"
1773                                          " \"udp.hdr.src_port\" field");
1774                 DRV_LOG(WARNING,
1775                         "outer UDP local port cannot be "
1776                         "forced for VXLAN encapsulation, "
1777                         "parameter ignored");
1778         }
1779         return 0;
1780 }
1781
1782 /**
1783  * Validate flow for E-Switch.
1784  *
1785  * @param[in] priv
1786  *   Pointer to the priv structure.
1787  * @param[in] attr
1788  *   Pointer to the flow attributes.
1789  * @param[in] items
1790  *   Pointer to the list of items.
1791  * @param[in] actions
1792  *   Pointer to the list of actions.
1793  * @param[out] error
1794  *   Pointer to the error structure.
1795  *
1796  * @return
1797  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1798  */
1799 static int
1800 flow_tcf_validate(struct rte_eth_dev *dev,
1801                   const struct rte_flow_attr *attr,
1802                   const struct rte_flow_item items[],
1803                   const struct rte_flow_action actions[],
1804                   struct rte_flow_error *error)
1805 {
1806         union {
1807                 const struct rte_flow_item_port_id *port_id;
1808                 const struct rte_flow_item_eth *eth;
1809                 const struct rte_flow_item_vlan *vlan;
1810                 const struct rte_flow_item_ipv4 *ipv4;
1811                 const struct rte_flow_item_ipv6 *ipv6;
1812                 const struct rte_flow_item_tcp *tcp;
1813                 const struct rte_flow_item_udp *udp;
1814                 const struct rte_flow_item_vxlan *vxlan;
1815         } spec, mask;
1816         union {
1817                 const struct rte_flow_action_port_id *port_id;
1818                 const struct rte_flow_action_jump *jump;
1819                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
1820                 const struct rte_flow_action_of_set_vlan_vid *
1821                         of_set_vlan_vid;
1822                 const struct rte_flow_action_of_set_vlan_pcp *
1823                         of_set_vlan_pcp;
1824                 const struct rte_flow_action_vxlan_encap *vxlan_encap;
1825                 const struct rte_flow_action_set_ipv4 *set_ipv4;
1826                 const struct rte_flow_action_set_ipv6 *set_ipv6;
1827         } conf;
1828         uint64_t item_flags = 0;
1829         uint64_t action_flags = 0;
1830         uint8_t next_protocol = -1;
1831         unsigned int tcm_ifindex = 0;
1832         uint8_t pedit_validated = 0;
1833         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
1834         struct rte_eth_dev *port_id_dev = NULL;
1835         bool in_port_id_set;
1836         int ret;
1837
1838         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
1839                                                 PTOI_TABLE_SZ_MAX(dev)));
1840         ret = flow_tcf_validate_attributes(attr, error);
1841         if (ret < 0)
1842                 return ret;
1843         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1844                 unsigned int i;
1845                 uint64_t current_action_flag = 0;
1846
1847                 switch (actions->type) {
1848                 case RTE_FLOW_ACTION_TYPE_VOID:
1849                         break;
1850                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
1851                         current_action_flag = MLX5_FLOW_ACTION_PORT_ID;
1852                         if (!actions->conf)
1853                                 break;
1854                         conf.port_id = actions->conf;
1855                         if (conf.port_id->original)
1856                                 i = 0;
1857                         else
1858                                 for (i = 0; ptoi[i].ifindex; ++i)
1859                                         if (ptoi[i].port_id == conf.port_id->id)
1860                                                 break;
1861                         if (!ptoi[i].ifindex)
1862                                 return rte_flow_error_set
1863                                         (error, ENODEV,
1864                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1865                                          conf.port_id,
1866                                          "missing data to convert port ID to"
1867                                          " ifindex");
1868                         port_id_dev = &rte_eth_devices[conf.port_id->id];
1869                         break;
1870                 case RTE_FLOW_ACTION_TYPE_JUMP:
1871                         current_action_flag = MLX5_FLOW_ACTION_JUMP;
1872                         if (!actions->conf)
1873                                 break;
1874                         conf.jump = actions->conf;
1875                         if (attr->group >= conf.jump->group)
1876                                 return rte_flow_error_set
1877                                         (error, ENOTSUP,
1878                                          RTE_FLOW_ERROR_TYPE_ACTION,
1879                                          actions,
1880                                          "can jump only to a group forward");
1881                         break;
1882                 case RTE_FLOW_ACTION_TYPE_DROP:
1883                         current_action_flag = MLX5_FLOW_ACTION_DROP;
1884                         break;
1885                 case RTE_FLOW_ACTION_TYPE_COUNT:
1886                         break;
1887                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
1888                         current_action_flag = MLX5_FLOW_ACTION_OF_POP_VLAN;
1889                         break;
1890                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
1891                         current_action_flag = MLX5_FLOW_ACTION_OF_PUSH_VLAN;
1892                         break;
1893                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
1894                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1895                                 return rte_flow_error_set
1896                                         (error, ENOTSUP,
1897                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1898                                          "vlan modify is not supported,"
1899                                          " set action must follow push action");
1900                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
1901                         break;
1902                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
1903                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1904                                 return rte_flow_error_set
1905                                         (error, ENOTSUP,
1906                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1907                                          "vlan modify is not supported,"
1908                                          " set action must follow push action");
1909                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
1910                         break;
1911                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1912                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_DECAP;
1913                         break;
1914                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1915                         ret = flow_tcf_validate_vxlan_encap(actions, error);
1916                         if (ret < 0)
1917                                 return ret;
1918                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_ENCAP;
1919                         break;
1920                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
1921                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_SRC;
1922                         break;
1923                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
1924                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_DST;
1925                         break;
1926                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
1927                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_SRC;
1928                         break;
1929                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
1930                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_DST;
1931                         break;
1932                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
1933                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_SRC;
1934                         break;
1935                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
1936                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_DST;
1937                         break;
1938                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
1939                         current_action_flag = MLX5_FLOW_ACTION_SET_TTL;
1940                         break;
1941                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
1942                         current_action_flag = MLX5_FLOW_ACTION_DEC_TTL;
1943                         break;
1944                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
1945                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_SRC;
1946                         break;
1947                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
1948                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_DST;
1949                         break;
1950                 default:
1951                         return rte_flow_error_set(error, ENOTSUP,
1952                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1953                                                   actions,
1954                                                   "action not supported");
1955                 }
1956                 if (current_action_flag & MLX5_TCF_CONFIG_ACTIONS) {
1957                         if (!actions->conf)
1958                                 return rte_flow_error_set
1959                                         (error, EINVAL,
1960                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1961                                          actions,
1962                                          "action configuration not set");
1963                 }
1964                 if ((current_action_flag & MLX5_TCF_PEDIT_ACTIONS) &&
1965                     pedit_validated)
1966                         return rte_flow_error_set(error, ENOTSUP,
1967                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1968                                                   actions,
1969                                                   "set actions should be "
1970                                                   "listed successively");
1971                 if ((current_action_flag & ~MLX5_TCF_PEDIT_ACTIONS) &&
1972                     (action_flags & MLX5_TCF_PEDIT_ACTIONS))
1973                         pedit_validated = 1;
1974                 if ((current_action_flag & MLX5_TCF_FATE_ACTIONS) &&
1975                     (action_flags & MLX5_TCF_FATE_ACTIONS))
1976                         return rte_flow_error_set(error, EINVAL,
1977                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1978                                                   actions,
1979                                                   "can't have multiple fate"
1980                                                   " actions");
1981                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1982                     (action_flags & MLX5_TCF_VXLAN_ACTIONS))
1983                         return rte_flow_error_set(error, EINVAL,
1984                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1985                                                   actions,
1986                                                   "can't have multiple vxlan"
1987                                                   " actions");
1988                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1989                     (action_flags & MLX5_TCF_VLAN_ACTIONS))
1990                         return rte_flow_error_set(error, ENOTSUP,
1991                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1992                                                   actions,
1993                                                   "can't have vxlan and vlan"
1994                                                   " actions in the same rule");
1995                 action_flags |= current_action_flag;
1996         }
1997         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1998                 unsigned int i;
1999
2000                 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2001                     items->type != RTE_FLOW_ITEM_TYPE_ETH)
2002                         return rte_flow_error_set(error, ENOTSUP,
2003                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2004                                                   items,
2005                                                   "only L2 inner item"
2006                                                   " is supported");
2007                 switch (items->type) {
2008                 case RTE_FLOW_ITEM_TYPE_VOID:
2009                         break;
2010                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2011                         mask.port_id = flow_tcf_item_mask
2012                                 (items, &rte_flow_item_port_id_mask,
2013                                  &flow_tcf_mask_supported.port_id,
2014                                  &flow_tcf_mask_empty.port_id,
2015                                  sizeof(flow_tcf_mask_supported.port_id),
2016                                  error);
2017                         if (!mask.port_id)
2018                                 return -rte_errno;
2019                         if (mask.port_id == &flow_tcf_mask_empty.port_id) {
2020                                 in_port_id_set = 1;
2021                                 break;
2022                         }
2023                         spec.port_id = items->spec;
2024                         if (mask.port_id->id && mask.port_id->id != 0xffffffff)
2025                                 return rte_flow_error_set
2026                                         (error, ENOTSUP,
2027                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2028                                          mask.port_id,
2029                                          "no support for partial mask on"
2030                                          " \"id\" field");
2031                         if (!mask.port_id->id)
2032                                 i = 0;
2033                         else
2034                                 for (i = 0; ptoi[i].ifindex; ++i)
2035                                         if (ptoi[i].port_id == spec.port_id->id)
2036                                                 break;
2037                         if (!ptoi[i].ifindex)
2038                                 return rte_flow_error_set
2039                                         (error, ENODEV,
2040                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2041                                          spec.port_id,
2042                                          "missing data to convert port ID to"
2043                                          " ifindex");
2044                         if (in_port_id_set && ptoi[i].ifindex != tcm_ifindex)
2045                                 return rte_flow_error_set
2046                                         (error, ENOTSUP,
2047                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2048                                          spec.port_id,
2049                                          "cannot match traffic for"
2050                                          " several port IDs through"
2051                                          " a single flow rule");
2052                         tcm_ifindex = ptoi[i].ifindex;
2053                         in_port_id_set = 1;
2054                         break;
2055                 case RTE_FLOW_ITEM_TYPE_ETH:
2056                         ret = mlx5_flow_validate_item_eth(items, item_flags,
2057                                                           error);
2058                         if (ret < 0)
2059                                 return ret;
2060                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2061                                         MLX5_FLOW_LAYER_INNER_L2 :
2062                                         MLX5_FLOW_LAYER_OUTER_L2;
2063                         /* TODO:
2064                          * Redundant check due to different supported mask.
2065                          * Same for the rest of items.
2066                          */
2067                         mask.eth = flow_tcf_item_mask
2068                                 (items, &rte_flow_item_eth_mask,
2069                                  &flow_tcf_mask_supported.eth,
2070                                  &flow_tcf_mask_empty.eth,
2071                                  sizeof(flow_tcf_mask_supported.eth),
2072                                  error);
2073                         if (!mask.eth)
2074                                 return -rte_errno;
2075                         if (mask.eth->type && mask.eth->type !=
2076                             RTE_BE16(0xffff))
2077                                 return rte_flow_error_set
2078                                         (error, ENOTSUP,
2079                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2080                                          mask.eth,
2081                                          "no support for partial mask on"
2082                                          " \"type\" field");
2083                         break;
2084                 case RTE_FLOW_ITEM_TYPE_VLAN:
2085                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
2086                                                            error);
2087                         if (ret < 0)
2088                                 return ret;
2089                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
2090                         mask.vlan = flow_tcf_item_mask
2091                                 (items, &rte_flow_item_vlan_mask,
2092                                  &flow_tcf_mask_supported.vlan,
2093                                  &flow_tcf_mask_empty.vlan,
2094                                  sizeof(flow_tcf_mask_supported.vlan),
2095                                  error);
2096                         if (!mask.vlan)
2097                                 return -rte_errno;
2098                         if ((mask.vlan->tci & RTE_BE16(0xe000) &&
2099                              (mask.vlan->tci & RTE_BE16(0xe000)) !=
2100                               RTE_BE16(0xe000)) ||
2101                             (mask.vlan->tci & RTE_BE16(0x0fff) &&
2102                              (mask.vlan->tci & RTE_BE16(0x0fff)) !=
2103                               RTE_BE16(0x0fff)) ||
2104                             (mask.vlan->inner_type &&
2105                              mask.vlan->inner_type != RTE_BE16(0xffff)))
2106                                 return rte_flow_error_set
2107                                         (error, ENOTSUP,
2108                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2109                                          mask.vlan,
2110                                          "no support for partial masks on"
2111                                          " \"tci\" (PCP and VID parts) and"
2112                                          " \"inner_type\" fields");
2113                         break;
2114                 case RTE_FLOW_ITEM_TYPE_IPV4:
2115                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
2116                                                            error);
2117                         if (ret < 0)
2118                                 return ret;
2119                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2120                         mask.ipv4 = flow_tcf_item_mask
2121                                 (items, &rte_flow_item_ipv4_mask,
2122                                  &flow_tcf_mask_supported.ipv4,
2123                                  &flow_tcf_mask_empty.ipv4,
2124                                  sizeof(flow_tcf_mask_supported.ipv4),
2125                                  error);
2126                         if (!mask.ipv4)
2127                                 return -rte_errno;
2128                         if (mask.ipv4->hdr.next_proto_id &&
2129                             mask.ipv4->hdr.next_proto_id != 0xff)
2130                                 return rte_flow_error_set
2131                                         (error, ENOTSUP,
2132                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2133                                          mask.ipv4,
2134                                          "no support for partial mask on"
2135                                          " \"hdr.next_proto_id\" field");
2136                         else if (mask.ipv4->hdr.next_proto_id)
2137                                 next_protocol =
2138                                         ((const struct rte_flow_item_ipv4 *)
2139                                          (items->spec))->hdr.next_proto_id;
2140                         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2141                                 ret = flow_tcf_validate_vxlan_decap_ipv4
2142                                                                 (items, error);
2143                                 if (ret < 0)
2144                                         return ret;
2145                         }
2146                         break;
2147                 case RTE_FLOW_ITEM_TYPE_IPV6:
2148                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
2149                                                            error);
2150                         if (ret < 0)
2151                                 return ret;
2152                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2153                         mask.ipv6 = flow_tcf_item_mask
2154                                 (items, &rte_flow_item_ipv6_mask,
2155                                  &flow_tcf_mask_supported.ipv6,
2156                                  &flow_tcf_mask_empty.ipv6,
2157                                  sizeof(flow_tcf_mask_supported.ipv6),
2158                                  error);
2159                         if (!mask.ipv6)
2160                                 return -rte_errno;
2161                         if (mask.ipv6->hdr.proto &&
2162                             mask.ipv6->hdr.proto != 0xff)
2163                                 return rte_flow_error_set
2164                                         (error, ENOTSUP,
2165                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2166                                          mask.ipv6,
2167                                          "no support for partial mask on"
2168                                          " \"hdr.proto\" field");
2169                         else if (mask.ipv6->hdr.proto)
2170                                 next_protocol =
2171                                         ((const struct rte_flow_item_ipv6 *)
2172                                          (items->spec))->hdr.proto;
2173                         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2174                                 ret = flow_tcf_validate_vxlan_decap_ipv6
2175                                                                 (items, error);
2176                                 if (ret < 0)
2177                                         return ret;
2178                         }
2179                         break;
2180                 case RTE_FLOW_ITEM_TYPE_UDP:
2181                         ret = mlx5_flow_validate_item_udp(items, item_flags,
2182                                                           next_protocol, error);
2183                         if (ret < 0)
2184                                 return ret;
2185                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
2186                         mask.udp = flow_tcf_item_mask
2187                                 (items, &rte_flow_item_udp_mask,
2188                                  &flow_tcf_mask_supported.udp,
2189                                  &flow_tcf_mask_empty.udp,
2190                                  sizeof(flow_tcf_mask_supported.udp),
2191                                  error);
2192                         if (!mask.udp)
2193                                 return -rte_errno;
2194                         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2195                                 ret = flow_tcf_validate_vxlan_decap_udp
2196                                                                 (items, error);
2197                                 if (ret < 0)
2198                                         return ret;
2199                         }
2200                         break;
2201                 case RTE_FLOW_ITEM_TYPE_TCP:
2202                         ret = mlx5_flow_validate_item_tcp
2203                                              (items, item_flags,
2204                                               next_protocol,
2205                                               &flow_tcf_mask_supported.tcp,
2206                                               error);
2207                         if (ret < 0)
2208                                 return ret;
2209                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
2210                         mask.tcp = flow_tcf_item_mask
2211                                 (items, &rte_flow_item_tcp_mask,
2212                                  &flow_tcf_mask_supported.tcp,
2213                                  &flow_tcf_mask_empty.tcp,
2214                                  sizeof(flow_tcf_mask_supported.tcp),
2215                                  error);
2216                         if (!mask.tcp)
2217                                 return -rte_errno;
2218                         break;
2219                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2220                         if (!(action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP))
2221                                 return rte_flow_error_set
2222                                         (error, ENOTSUP,
2223                                          RTE_FLOW_ERROR_TYPE_ITEM,
2224                                          items,
2225                                          "vni pattern should be followed by"
2226                                          " vxlan decapsulation action");
2227                         ret = mlx5_flow_validate_item_vxlan(items,
2228                                                             item_flags, error);
2229                         if (ret < 0)
2230                                 return ret;
2231                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
2232                         mask.vxlan = flow_tcf_item_mask
2233                                 (items, &rte_flow_item_vxlan_mask,
2234                                  &flow_tcf_mask_supported.vxlan,
2235                                  &flow_tcf_mask_empty.vxlan,
2236                                  sizeof(flow_tcf_mask_supported.vxlan), error);
2237                         if (!mask.vxlan)
2238                                 return -rte_errno;
2239                         if (mask.vxlan->vni[0] != 0xff ||
2240                             mask.vxlan->vni[1] != 0xff ||
2241                             mask.vxlan->vni[2] != 0xff)
2242                                 return rte_flow_error_set
2243                                         (error, ENOTSUP,
2244                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2245                                          mask.vxlan,
2246                                          "no support for partial or "
2247                                          "empty mask on \"vxlan.vni\" field");
2248                         break;
2249                 default:
2250                         return rte_flow_error_set(error, ENOTSUP,
2251                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2252                                                   items, "item not supported");
2253                 }
2254         }
2255         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2256             (action_flags & MLX5_FLOW_ACTION_DROP))
2257                 return rte_flow_error_set(error, ENOTSUP,
2258                                           RTE_FLOW_ERROR_TYPE_ACTION,
2259                                           actions,
2260                                           "set action is not compatible with "
2261                                           "drop action");
2262         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2263             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2264                 return rte_flow_error_set(error, ENOTSUP,
2265                                           RTE_FLOW_ERROR_TYPE_ACTION,
2266                                           actions,
2267                                           "set action must be followed by "
2268                                           "port_id action");
2269         if (action_flags &
2270            (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST)) {
2271                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4))
2272                         return rte_flow_error_set(error, EINVAL,
2273                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2274                                                   actions,
2275                                                   "no ipv4 item found in"
2276                                                   " pattern");
2277         }
2278         if (action_flags &
2279            (MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST)) {
2280                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6))
2281                         return rte_flow_error_set(error, EINVAL,
2282                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2283                                                   actions,
2284                                                   "no ipv6 item found in"
2285                                                   " pattern");
2286         }
2287         if (action_flags &
2288            (MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST)) {
2289                 if (!(item_flags &
2290                      (MLX5_FLOW_LAYER_OUTER_L4_UDP |
2291                       MLX5_FLOW_LAYER_OUTER_L4_TCP)))
2292                         return rte_flow_error_set(error, EINVAL,
2293                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2294                                                   actions,
2295                                                   "no TCP/UDP item found in"
2296                                                   " pattern");
2297         }
2298         /*
2299          * FW syndrome (0xA9C090):
2300          *     set_flow_table_entry: push vlan action fte in fdb can ONLY be
2301          *     forward to the uplink.
2302          */
2303         if ((action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2304             (action_flags & MLX5_FLOW_ACTION_PORT_ID) &&
2305             ((struct priv *)port_id_dev->data->dev_private)->representor)
2306                 return rte_flow_error_set(error, ENOTSUP,
2307                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2308                                           "vlan push can only be applied"
2309                                           " when forwarding to uplink port");
2310         /*
2311          * FW syndrome (0x294609):
2312          *     set_flow_table_entry: modify/pop/push actions in fdb flow table
2313          *     are supported only while forwarding to vport.
2314          */
2315         if ((action_flags & MLX5_TCF_VLAN_ACTIONS) &&
2316             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2317                 return rte_flow_error_set(error, ENOTSUP,
2318                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2319                                           "vlan actions are supported"
2320                                           " only with port_id action");
2321         if ((action_flags & MLX5_TCF_VXLAN_ACTIONS) &&
2322             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2323                 return rte_flow_error_set(error, ENOTSUP,
2324                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2325                                           "vxlan actions are supported"
2326                                           " only with port_id action");
2327         if (!(action_flags & MLX5_TCF_FATE_ACTIONS))
2328                 return rte_flow_error_set(error, EINVAL,
2329                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2330                                           "no fate action is found");
2331         if (action_flags &
2332            (MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL)) {
2333                 if (!(item_flags &
2334                      (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2335                       MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2336                         return rte_flow_error_set(error, EINVAL,
2337                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2338                                                   actions,
2339                                                   "no IP found in pattern");
2340         }
2341         if (action_flags &
2342             (MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)) {
2343                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L2))
2344                         return rte_flow_error_set(error, ENOTSUP,
2345                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2346                                                   actions,
2347                                                   "no ethernet found in"
2348                                                   " pattern");
2349         }
2350         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2351                 if (!(item_flags &
2352                      (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2353                       MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2354                         return rte_flow_error_set(error, EINVAL,
2355                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2356                                                   NULL,
2357                                                   "no outer IP pattern found"
2358                                                   " for vxlan decap action");
2359                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2360                         return rte_flow_error_set(error, EINVAL,
2361                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2362                                                   NULL,
2363                                                   "no outer UDP pattern found"
2364                                                   " for vxlan decap action");
2365                 if (!(item_flags & MLX5_FLOW_LAYER_VXLAN))
2366                         return rte_flow_error_set(error, EINVAL,
2367                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2368                                                   NULL,
2369                                                   "no VNI pattern found"
2370                                                   " for vxlan decap action");
2371         }
2372         return 0;
2373 }
2374
2375 /**
2376  * Calculate maximum size of memory for flow items of Linux TC flower.
2377  *
2378  * @param[in] attr
2379  *   Pointer to the flow attributes.
2380  * @param[in] items
2381  *   Pointer to the list of items.
2382  *
2383  * @return
2384  *   Maximum size of memory for items.
2385  */
2386 static int
2387 flow_tcf_get_items_size(const struct rte_flow_attr *attr,
2388                         const struct rte_flow_item items[])
2389 {
2390         int size = 0;
2391
2392         size += SZ_NLATTR_STRZ_OF("flower") +
2393                 SZ_NLATTR_NEST + /* TCA_OPTIONS. */
2394                 SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CLS_FLAGS_SKIP_SW. */
2395         if (attr->group > 0)
2396                 size += SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CHAIN. */
2397         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2398                 switch (items->type) {
2399                 case RTE_FLOW_ITEM_TYPE_VOID:
2400                         break;
2401                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2402                         break;
2403                 case RTE_FLOW_ITEM_TYPE_ETH:
2404                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2405                                 SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) * 4;
2406                                 /* dst/src MAC addr and mask. */
2407                         break;
2408                 case RTE_FLOW_ITEM_TYPE_VLAN:
2409                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2410                                 SZ_NLATTR_TYPE_OF(uint16_t) +
2411                                 /* VLAN Ether type. */
2412                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* VLAN prio. */
2413                                 SZ_NLATTR_TYPE_OF(uint16_t); /* VLAN ID. */
2414                         break;
2415                 case RTE_FLOW_ITEM_TYPE_IPV4:
2416                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2417                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2418                                 SZ_NLATTR_TYPE_OF(uint32_t) * 4;
2419                                 /* dst/src IP addr and mask. */
2420                         break;
2421                 case RTE_FLOW_ITEM_TYPE_IPV6:
2422                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2423                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2424                                 SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 4;
2425                                 /* dst/src IP addr and mask. */
2426                         break;
2427                 case RTE_FLOW_ITEM_TYPE_UDP:
2428                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2429                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2430                                 /* dst/src port and mask. */
2431                         break;
2432                 case RTE_FLOW_ITEM_TYPE_TCP:
2433                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2434                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2435                                 /* dst/src port and mask. */
2436                         break;
2437                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2438                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2439                         break;
2440                 default:
2441                         DRV_LOG(WARNING,
2442                                 "unsupported item %p type %d,"
2443                                 " items must be validated before flow creation",
2444                                 (const void *)items, items->type);
2445                         break;
2446                 }
2447         }
2448         return size;
2449 }
2450
2451 /**
2452  * Calculate size of memory to store the VXLAN encapsultion
2453  * related items in the Netlink message buffer. Items list
2454  * is specified by RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action.
2455  * The item list should be validated.
2456  *
2457  * @param[in] action
2458  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
2459  *   List of pattern items to scan data from.
2460  *
2461  * @return
2462  *   The size the part of Netlink message buffer to store the
2463  *   VXLAN encapsulation item attributes.
2464  */
2465 static int
2466 flow_tcf_vxlan_encap_size(const struct rte_flow_action *action)
2467 {
2468         const struct rte_flow_item *items;
2469         int size = 0;
2470
2471         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
2472         assert(action->conf);
2473
2474         items = ((const struct rte_flow_action_vxlan_encap *)
2475                                         action->conf)->definition;
2476         assert(items);
2477         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2478                 switch (items->type) {
2479                 case RTE_FLOW_ITEM_TYPE_VOID:
2480                         break;
2481                 case RTE_FLOW_ITEM_TYPE_ETH:
2482                         /* This item does not require message buffer. */
2483                         break;
2484                 case RTE_FLOW_ITEM_TYPE_IPV4:
2485                         size += SZ_NLATTR_DATA_OF(IPV4_ADDR_LEN) * 2;
2486                         break;
2487                 case RTE_FLOW_ITEM_TYPE_IPV6:
2488                         size += SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 2;
2489                         break;
2490                 case RTE_FLOW_ITEM_TYPE_UDP: {
2491                         const struct rte_flow_item_udp *udp = items->mask;
2492
2493                         size += SZ_NLATTR_TYPE_OF(uint16_t);
2494                         if (!udp || udp->hdr.src_port != RTE_BE16(0x0000))
2495                                 size += SZ_NLATTR_TYPE_OF(uint16_t);
2496                         break;
2497                 }
2498                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2499                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2500                         break;
2501                 default:
2502                         assert(false);
2503                         DRV_LOG(WARNING,
2504                                 "unsupported item %p type %d,"
2505                                 " items must be validated"
2506                                 " before flow creation",
2507                                 (const void *)items, items->type);
2508                         return 0;
2509                 }
2510         }
2511         return size;
2512 }
2513
2514 /**
2515  * Calculate maximum size of memory for flow actions of Linux TC flower and
2516  * extract specified actions.
2517  *
2518  * @param[in] actions
2519  *   Pointer to the list of actions.
2520  * @param[out] action_flags
2521  *   Pointer to the detected actions.
2522  *
2523  * @return
2524  *   Maximum size of memory for actions.
2525  */
2526 static int
2527 flow_tcf_get_actions_and_size(const struct rte_flow_action actions[],
2528                               uint64_t *action_flags)
2529 {
2530         int size = 0;
2531         uint64_t flags = 0;
2532
2533         size += SZ_NLATTR_NEST; /* TCA_FLOWER_ACT. */
2534         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2535                 switch (actions->type) {
2536                 case RTE_FLOW_ACTION_TYPE_VOID:
2537                         break;
2538                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
2539                         size += SZ_NLATTR_NEST + /* na_act_index. */
2540                                 SZ_NLATTR_STRZ_OF("mirred") +
2541                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2542                                 SZ_NLATTR_TYPE_OF(struct tc_mirred);
2543                         flags |= MLX5_FLOW_ACTION_PORT_ID;
2544                         break;
2545                 case RTE_FLOW_ACTION_TYPE_JUMP:
2546                         size += SZ_NLATTR_NEST + /* na_act_index. */
2547                                 SZ_NLATTR_STRZ_OF("gact") +
2548                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2549                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2550                         flags |= MLX5_FLOW_ACTION_JUMP;
2551                         break;
2552                 case RTE_FLOW_ACTION_TYPE_DROP:
2553                         size += SZ_NLATTR_NEST + /* na_act_index. */
2554                                 SZ_NLATTR_STRZ_OF("gact") +
2555                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2556                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2557                         flags |= MLX5_FLOW_ACTION_DROP;
2558                         break;
2559                 case RTE_FLOW_ACTION_TYPE_COUNT:
2560                         break;
2561                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
2562                         flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
2563                         goto action_of_vlan;
2564                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
2565                         flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
2566                         goto action_of_vlan;
2567                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
2568                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
2569                         goto action_of_vlan;
2570                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
2571                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
2572                         goto action_of_vlan;
2573 action_of_vlan:
2574                         size += SZ_NLATTR_NEST + /* na_act_index. */
2575                                 SZ_NLATTR_STRZ_OF("vlan") +
2576                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2577                                 SZ_NLATTR_TYPE_OF(struct tc_vlan) +
2578                                 SZ_NLATTR_TYPE_OF(uint16_t) +
2579                                 /* VLAN protocol. */
2580                                 SZ_NLATTR_TYPE_OF(uint16_t) + /* VLAN ID. */
2581                                 SZ_NLATTR_TYPE_OF(uint8_t); /* VLAN prio. */
2582                         break;
2583                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2584                         size += SZ_NLATTR_NEST + /* na_act_index. */
2585                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2586                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2587                                 SZ_NLATTR_TYPE_OF(uint8_t);
2588                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2589                         size += flow_tcf_vxlan_encap_size(actions) +
2590                                 RTE_ALIGN_CEIL /* preceding encap params. */
2591                                 (sizeof(struct flow_tcf_vxlan_encap),
2592                                 MNL_ALIGNTO);
2593                         flags |= MLX5_FLOW_ACTION_VXLAN_ENCAP;
2594                         break;
2595                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
2596                         size += SZ_NLATTR_NEST + /* na_act_index. */
2597                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2598                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2599                                 SZ_NLATTR_TYPE_OF(uint8_t);
2600                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2601                         size += RTE_ALIGN_CEIL /* preceding decap params. */
2602                                 (sizeof(struct flow_tcf_vxlan_decap),
2603                                 MNL_ALIGNTO);
2604                         flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
2605                         break;
2606                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
2607                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
2608                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
2609                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
2610                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
2611                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
2612                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
2613                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
2614                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
2615                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
2616                         size += flow_tcf_get_pedit_actions_size(&actions,
2617                                                                 &flags);
2618                         break;
2619                 default:
2620                         DRV_LOG(WARNING,
2621                                 "unsupported action %p type %d,"
2622                                 " items must be validated before flow creation",
2623                                 (const void *)actions, actions->type);
2624                         break;
2625                 }
2626         }
2627         *action_flags = flags;
2628         return size;
2629 }
2630
2631 /**
2632  * Brand rtnetlink buffer with unique handle.
2633  *
2634  * This handle should be unique for a given network interface to avoid
2635  * collisions.
2636  *
2637  * @param nlh
2638  *   Pointer to Netlink message.
2639  * @param handle
2640  *   Unique 32-bit handle to use.
2641  */
2642 static void
2643 flow_tcf_nl_brand(struct nlmsghdr *nlh, uint32_t handle)
2644 {
2645         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
2646
2647         tcm->tcm_handle = handle;
2648         DRV_LOG(DEBUG, "Netlink msg %p is branded with handle %x",
2649                 (void *)nlh, handle);
2650 }
2651
2652 /**
2653  * Prepare a flow object for Linux TC flower. It calculates the maximum size of
2654  * memory required, allocates the memory, initializes Netlink message headers
2655  * and set unique TC message handle.
2656  *
2657  * @param[in] attr
2658  *   Pointer to the flow attributes.
2659  * @param[in] items
2660  *   Pointer to the list of items.
2661  * @param[in] actions
2662  *   Pointer to the list of actions.
2663  * @param[out] error
2664  *   Pointer to the error structure.
2665  *
2666  * @return
2667  *   Pointer to mlx5_flow object on success,
2668  *   otherwise NULL and rte_ernno is set.
2669  */
2670 static struct mlx5_flow *
2671 flow_tcf_prepare(const struct rte_flow_attr *attr,
2672                  const struct rte_flow_item items[],
2673                  const struct rte_flow_action actions[],
2674                  struct rte_flow_error *error)
2675 {
2676         size_t size = RTE_ALIGN_CEIL
2677                         (sizeof(struct mlx5_flow),
2678                          alignof(struct flow_tcf_tunnel_hdr)) +
2679                       MNL_ALIGN(sizeof(struct nlmsghdr)) +
2680                       MNL_ALIGN(sizeof(struct tcmsg));
2681         struct mlx5_flow *dev_flow;
2682         uint64_t action_flags = 0;
2683         struct nlmsghdr *nlh;
2684         struct tcmsg *tcm;
2685         uint8_t *sp, *tun = NULL;
2686
2687         size += flow_tcf_get_items_size(attr, items);
2688         size += flow_tcf_get_actions_and_size(actions, &action_flags);
2689         dev_flow = rte_zmalloc(__func__, size, MNL_ALIGNTO);
2690         if (!dev_flow) {
2691                 rte_flow_error_set(error, ENOMEM,
2692                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2693                                    "not enough memory to create E-Switch flow");
2694                 return NULL;
2695         }
2696         sp = (uint8_t *)(dev_flow + 1);
2697         if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP) {
2698                 sp = RTE_PTR_ALIGN
2699                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2700                 tun = sp;
2701                 sp += RTE_ALIGN_CEIL
2702                         (sizeof(struct flow_tcf_vxlan_encap),
2703                         MNL_ALIGNTO);
2704 #ifndef NDEBUG
2705                 size -= RTE_ALIGN_CEIL
2706                         (sizeof(struct flow_tcf_vxlan_encap),
2707                         MNL_ALIGNTO);
2708 #endif
2709         } else if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2710                 sp = RTE_PTR_ALIGN
2711                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2712                 tun = sp;
2713                 sp += RTE_ALIGN_CEIL
2714                         (sizeof(struct flow_tcf_vxlan_decap),
2715                         MNL_ALIGNTO);
2716 #ifndef NDEBUG
2717                 size -= RTE_ALIGN_CEIL
2718                         (sizeof(struct flow_tcf_vxlan_decap),
2719                         MNL_ALIGNTO);
2720 #endif
2721         } else {
2722                 sp = RTE_PTR_ALIGN(sp, MNL_ALIGNTO);
2723         }
2724         nlh = mnl_nlmsg_put_header(sp);
2725         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
2726         *dev_flow = (struct mlx5_flow){
2727                 .tcf = (struct mlx5_flow_tcf){
2728 #ifndef NDEBUG
2729                         .nlsize = size - RTE_ALIGN_CEIL
2730                                 (sizeof(struct mlx5_flow),
2731                                  alignof(struct flow_tcf_tunnel_hdr)),
2732 #endif
2733                         .tunnel = (struct flow_tcf_tunnel_hdr *)tun,
2734                         .nlh = nlh,
2735                         .tcm = tcm,
2736                 },
2737         };
2738         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP)
2739                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_DECAP;
2740         else if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP)
2741                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_ENCAP;
2742         /*
2743          * Generate a reasonably unique handle based on the address of the
2744          * target buffer.
2745          *
2746          * This is straightforward on 32-bit systems where the flow pointer can
2747          * be used directly. Otherwise, its least significant part is taken
2748          * after shifting it by the previous power of two of the pointed buffer
2749          * size.
2750          */
2751         if (sizeof(dev_flow) <= 4)
2752                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow);
2753         else
2754                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow >>
2755                                        rte_log2_u32(rte_align32prevpow2(size)));
2756         return dev_flow;
2757 }
2758
2759 /**
2760  * Make adjustments for supporting count actions.
2761  *
2762  * @param[in] dev
2763  *   Pointer to the Ethernet device structure.
2764  * @param[in] dev_flow
2765  *   Pointer to mlx5_flow.
2766  * @param[out] error
2767  *   Pointer to error structure.
2768  *
2769  * @return
2770  *   0 On success else a negative errno value is returned and rte_errno is set.
2771  */
2772 static int
2773 flow_tcf_translate_action_count(struct rte_eth_dev *dev __rte_unused,
2774                                   struct mlx5_flow *dev_flow,
2775                                   struct rte_flow_error *error)
2776 {
2777         struct rte_flow *flow = dev_flow->flow;
2778
2779         if (!flow->counter) {
2780                 flow->counter = flow_tcf_counter_new();
2781                 if (!flow->counter)
2782                         return rte_flow_error_set(error, rte_errno,
2783                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2784                                                   NULL,
2785                                                   "cannot get counter"
2786                                                   " context.");
2787         }
2788         return 0;
2789 }
2790
2791 /**
2792  * Convert VXLAN VNI to 32-bit integer.
2793  *
2794  * @param[in] vni
2795  *   VXLAN VNI in 24-bit wire format.
2796  *
2797  * @return
2798  *   VXLAN VNI as a 32-bit integer value in network endian.
2799  */
2800 static inline rte_be32_t
2801 vxlan_vni_as_be32(const uint8_t vni[3])
2802 {
2803         union {
2804                 uint8_t vni[4];
2805                 rte_be32_t dword;
2806         } ret = {
2807                 .vni = { 0, vni[0], vni[1], vni[2] },
2808         };
2809         return ret.dword;
2810 }
2811
2812 /**
2813  * Helper function to process RTE_FLOW_ITEM_TYPE_ETH entry in configuration
2814  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the MAC address fields
2815  * in the encapsulation parameters structure. The item must be prevalidated,
2816  * no any validation checks performed by function.
2817  *
2818  * @param[in] spec
2819  *   RTE_FLOW_ITEM_TYPE_ETH entry specification.
2820  * @param[in] mask
2821  *   RTE_FLOW_ITEM_TYPE_ETH entry mask.
2822  * @param[out] encap
2823  *   Structure to fill the gathered MAC address data.
2824  */
2825 static void
2826 flow_tcf_parse_vxlan_encap_eth(const struct rte_flow_item_eth *spec,
2827                                const struct rte_flow_item_eth *mask,
2828                                struct flow_tcf_vxlan_encap *encap)
2829 {
2830         /* Item must be validated before. No redundant checks. */
2831         assert(spec);
2832         if (!mask || !memcmp(&mask->dst,
2833                              &rte_flow_item_eth_mask.dst,
2834                              sizeof(rte_flow_item_eth_mask.dst))) {
2835                 /*
2836                  * Ethernet addresses are not supported by
2837                  * tc as tunnel_key parameters. Destination
2838                  * address is needed to form encap packet
2839                  * header and retrieved by kernel from
2840                  * implicit sources (ARP table, etc),
2841                  * address masks are not supported at all.
2842                  */
2843                 encap->eth.dst = spec->dst;
2844                 encap->mask |= FLOW_TCF_ENCAP_ETH_DST;
2845         }
2846         if (!mask || !memcmp(&mask->src,
2847                              &rte_flow_item_eth_mask.src,
2848                              sizeof(rte_flow_item_eth_mask.src))) {
2849                 /*
2850                  * Ethernet addresses are not supported by
2851                  * tc as tunnel_key parameters. Source ethernet
2852                  * address is ignored anyway.
2853                  */
2854                 encap->eth.src = spec->src;
2855                 encap->mask |= FLOW_TCF_ENCAP_ETH_SRC;
2856         }
2857 }
2858
2859 /**
2860  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV4 entry in configuration
2861  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV4 address fields
2862  * in the encapsulation parameters structure. The item must be prevalidated,
2863  * no any validation checks performed by function.
2864  *
2865  * @param[in] spec
2866  *   RTE_FLOW_ITEM_TYPE_IPV4 entry specification.
2867  * @param[out] encap
2868  *   Structure to fill the gathered IPV4 address data.
2869  */
2870 static void
2871 flow_tcf_parse_vxlan_encap_ipv4(const struct rte_flow_item_ipv4 *spec,
2872                                 struct flow_tcf_vxlan_encap *encap)
2873 {
2874         /* Item must be validated before. No redundant checks. */
2875         assert(spec);
2876         encap->ipv4.dst = spec->hdr.dst_addr;
2877         encap->ipv4.src = spec->hdr.src_addr;
2878         encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC |
2879                        FLOW_TCF_ENCAP_IPV4_DST;
2880 }
2881
2882 /**
2883  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV6 entry in configuration
2884  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV6 address fields
2885  * in the encapsulation parameters structure. The item must be prevalidated,
2886  * no any validation checks performed by function.
2887  *
2888  * @param[in] spec
2889  *   RTE_FLOW_ITEM_TYPE_IPV6 entry specification.
2890  * @param[out] encap
2891  *   Structure to fill the gathered IPV6 address data.
2892  */
2893 static void
2894 flow_tcf_parse_vxlan_encap_ipv6(const struct rte_flow_item_ipv6 *spec,
2895                                 struct flow_tcf_vxlan_encap *encap)
2896 {
2897         /* Item must be validated before. No redundant checks. */
2898         assert(spec);
2899         memcpy(encap->ipv6.dst, spec->hdr.dst_addr, IPV6_ADDR_LEN);
2900         memcpy(encap->ipv6.src, spec->hdr.src_addr, IPV6_ADDR_LEN);
2901         encap->mask |= FLOW_TCF_ENCAP_IPV6_SRC |
2902                        FLOW_TCF_ENCAP_IPV6_DST;
2903 }
2904
2905 /**
2906  * Helper function to process RTE_FLOW_ITEM_TYPE_UDP entry in configuration
2907  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the UDP port fields
2908  * in the encapsulation parameters structure. The item must be prevalidated,
2909  * no any validation checks performed by function.
2910  *
2911  * @param[in] spec
2912  *   RTE_FLOW_ITEM_TYPE_UDP entry specification.
2913  * @param[in] mask
2914  *   RTE_FLOW_ITEM_TYPE_UDP entry mask.
2915  * @param[out] encap
2916  *   Structure to fill the gathered UDP port data.
2917  */
2918 static void
2919 flow_tcf_parse_vxlan_encap_udp(const struct rte_flow_item_udp *spec,
2920                                const struct rte_flow_item_udp *mask,
2921                                struct flow_tcf_vxlan_encap *encap)
2922 {
2923         assert(spec);
2924         encap->udp.dst = spec->hdr.dst_port;
2925         encap->mask |= FLOW_TCF_ENCAP_UDP_DST;
2926         if (!mask || mask->hdr.src_port != RTE_BE16(0x0000)) {
2927                 encap->udp.src = spec->hdr.src_port;
2928                 encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC;
2929         }
2930 }
2931
2932 /**
2933  * Helper function to process RTE_FLOW_ITEM_TYPE_VXLAN entry in configuration
2934  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the VNI fields
2935  * in the encapsulation parameters structure. The item must be prevalidated,
2936  * no any validation checks performed by function.
2937  *
2938  * @param[in] spec
2939  *   RTE_FLOW_ITEM_TYPE_VXLAN entry specification.
2940  * @param[out] encap
2941  *   Structure to fill the gathered VNI address data.
2942  */
2943 static void
2944 flow_tcf_parse_vxlan_encap_vni(const struct rte_flow_item_vxlan *spec,
2945                                struct flow_tcf_vxlan_encap *encap)
2946 {
2947         /* Item must be validated before. Do not redundant checks. */
2948         assert(spec);
2949         memcpy(encap->vxlan.vni, spec->vni, sizeof(encap->vxlan.vni));
2950         encap->mask |= FLOW_TCF_ENCAP_VXLAN_VNI;
2951 }
2952
2953 /**
2954  * Populate consolidated encapsulation object from list of pattern items.
2955  *
2956  * Helper function to process configuration of action such as
2957  * RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. The item list should be
2958  * validated, there is no way to return an meaningful error.
2959  *
2960  * @param[in] action
2961  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
2962  *   List of pattern items to gather data from.
2963  * @param[out] src
2964  *   Structure to fill gathered data.
2965  */
2966 static void
2967 flow_tcf_vxlan_encap_parse(const struct rte_flow_action *action,
2968                            struct flow_tcf_vxlan_encap *encap)
2969 {
2970         union {
2971                 const struct rte_flow_item_eth *eth;
2972                 const struct rte_flow_item_ipv4 *ipv4;
2973                 const struct rte_flow_item_ipv6 *ipv6;
2974                 const struct rte_flow_item_udp *udp;
2975                 const struct rte_flow_item_vxlan *vxlan;
2976         } spec, mask;
2977         const struct rte_flow_item *items;
2978
2979         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
2980         assert(action->conf);
2981
2982         items = ((const struct rte_flow_action_vxlan_encap *)
2983                                         action->conf)->definition;
2984         assert(items);
2985         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2986                 switch (items->type) {
2987                 case RTE_FLOW_ITEM_TYPE_VOID:
2988                         break;
2989                 case RTE_FLOW_ITEM_TYPE_ETH:
2990                         mask.eth = items->mask;
2991                         spec.eth = items->spec;
2992                         flow_tcf_parse_vxlan_encap_eth(spec.eth, mask.eth,
2993                                                        encap);
2994                         break;
2995                 case RTE_FLOW_ITEM_TYPE_IPV4:
2996                         spec.ipv4 = items->spec;
2997                         flow_tcf_parse_vxlan_encap_ipv4(spec.ipv4, encap);
2998                         break;
2999                 case RTE_FLOW_ITEM_TYPE_IPV6:
3000                         spec.ipv6 = items->spec;
3001                         flow_tcf_parse_vxlan_encap_ipv6(spec.ipv6, encap);
3002                         break;
3003                 case RTE_FLOW_ITEM_TYPE_UDP:
3004                         mask.udp = items->mask;
3005                         spec.udp = items->spec;
3006                         flow_tcf_parse_vxlan_encap_udp(spec.udp, mask.udp,
3007                                                        encap);
3008                         break;
3009                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3010                         spec.vxlan = items->spec;
3011                         flow_tcf_parse_vxlan_encap_vni(spec.vxlan, encap);
3012                         break;
3013                 default:
3014                         assert(false);
3015                         DRV_LOG(WARNING,
3016                                 "unsupported item %p type %d,"
3017                                 " items must be validated"
3018                                 " before flow creation",
3019                                 (const void *)items, items->type);
3020                         encap->mask = 0;
3021                         return;
3022                 }
3023         }
3024 }
3025
3026 /**
3027  * Translate flow for Linux TC flower and construct Netlink message.
3028  *
3029  * @param[in] priv
3030  *   Pointer to the priv structure.
3031  * @param[in, out] flow
3032  *   Pointer to the sub flow.
3033  * @param[in] attr
3034  *   Pointer to the flow attributes.
3035  * @param[in] items
3036  *   Pointer to the list of items.
3037  * @param[in] actions
3038  *   Pointer to the list of actions.
3039  * @param[out] error
3040  *   Pointer to the error structure.
3041  *
3042  * @return
3043  *   0 on success, a negative errno value otherwise and rte_ernno is set.
3044  */
3045 static int
3046 flow_tcf_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3047                    const struct rte_flow_attr *attr,
3048                    const struct rte_flow_item items[],
3049                    const struct rte_flow_action actions[],
3050                    struct rte_flow_error *error)
3051 {
3052         union {
3053                 const struct rte_flow_item_port_id *port_id;
3054                 const struct rte_flow_item_eth *eth;
3055                 const struct rte_flow_item_vlan *vlan;
3056                 const struct rte_flow_item_ipv4 *ipv4;
3057                 const struct rte_flow_item_ipv6 *ipv6;
3058                 const struct rte_flow_item_tcp *tcp;
3059                 const struct rte_flow_item_udp *udp;
3060                 const struct rte_flow_item_vxlan *vxlan;
3061         } spec, mask;
3062         union {
3063                 const struct rte_flow_action_port_id *port_id;
3064                 const struct rte_flow_action_jump *jump;
3065                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
3066                 const struct rte_flow_action_of_set_vlan_vid *
3067                         of_set_vlan_vid;
3068                 const struct rte_flow_action_of_set_vlan_pcp *
3069                         of_set_vlan_pcp;
3070         } conf;
3071         union {
3072                 struct flow_tcf_tunnel_hdr *hdr;
3073                 struct flow_tcf_vxlan_decap *vxlan;
3074         } decap = {
3075                 .hdr = NULL,
3076         };
3077         union {
3078                 struct flow_tcf_tunnel_hdr *hdr;
3079                 struct flow_tcf_vxlan_encap *vxlan;
3080         } encap = {
3081                 .hdr = NULL,
3082         };
3083         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
3084         struct nlmsghdr *nlh = dev_flow->tcf.nlh;
3085         struct tcmsg *tcm = dev_flow->tcf.tcm;
3086         uint32_t na_act_index_cur;
3087         bool eth_type_set = 0;
3088         bool vlan_present = 0;
3089         bool vlan_eth_type_set = 0;
3090         bool ip_proto_set = 0;
3091         struct nlattr *na_flower;
3092         struct nlattr *na_flower_act;
3093         struct nlattr *na_vlan_id = NULL;
3094         struct nlattr *na_vlan_priority = NULL;
3095         uint64_t item_flags = 0;
3096         int ret;
3097
3098         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
3099                                                 PTOI_TABLE_SZ_MAX(dev)));
3100         if (dev_flow->tcf.tunnel) {
3101                 switch (dev_flow->tcf.tunnel->type) {
3102                 case FLOW_TCF_TUNACT_VXLAN_DECAP:
3103                         decap.vxlan = dev_flow->tcf.vxlan_decap;
3104                         break;
3105                 case FLOW_TCF_TUNACT_VXLAN_ENCAP:
3106                         encap.vxlan = dev_flow->tcf.vxlan_encap;
3107                         break;
3108                 /* New tunnel actions can be added here. */
3109                 default:
3110                         assert(false);
3111                         break;
3112                 }
3113         }
3114         nlh = dev_flow->tcf.nlh;
3115         tcm = dev_flow->tcf.tcm;
3116         /* Prepare API must have been called beforehand. */
3117         assert(nlh != NULL && tcm != NULL);
3118         tcm->tcm_family = AF_UNSPEC;
3119         tcm->tcm_ifindex = ptoi[0].ifindex;
3120         tcm->tcm_parent = TC_H_MAKE(TC_H_INGRESS, TC_H_MIN_INGRESS);
3121         /*
3122          * Priority cannot be zero to prevent the kernel from picking one
3123          * automatically.
3124          */
3125         tcm->tcm_info = TC_H_MAKE((attr->priority + 1) << 16,
3126                                   RTE_BE16(ETH_P_ALL));
3127         if (attr->group > 0)
3128                 mnl_attr_put_u32(nlh, TCA_CHAIN, attr->group);
3129         mnl_attr_put_strz(nlh, TCA_KIND, "flower");
3130         na_flower = mnl_attr_nest_start(nlh, TCA_OPTIONS);
3131         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3132                 unsigned int i;
3133
3134                 switch (items->type) {
3135                 case RTE_FLOW_ITEM_TYPE_VOID:
3136                         break;
3137                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3138                         mask.port_id = flow_tcf_item_mask
3139                                 (items, &rte_flow_item_port_id_mask,
3140                                  &flow_tcf_mask_supported.port_id,
3141                                  &flow_tcf_mask_empty.port_id,
3142                                  sizeof(flow_tcf_mask_supported.port_id),
3143                                  error);
3144                         assert(mask.port_id);
3145                         if (mask.port_id == &flow_tcf_mask_empty.port_id)
3146                                 break;
3147                         spec.port_id = items->spec;
3148                         if (!mask.port_id->id)
3149                                 i = 0;
3150                         else
3151                                 for (i = 0; ptoi[i].ifindex; ++i)
3152                                         if (ptoi[i].port_id == spec.port_id->id)
3153                                                 break;
3154                         assert(ptoi[i].ifindex);
3155                         tcm->tcm_ifindex = ptoi[i].ifindex;
3156                         break;
3157                 case RTE_FLOW_ITEM_TYPE_ETH:
3158                         item_flags |= (item_flags & MLX5_FLOW_LAYER_VXLAN) ?
3159                                       MLX5_FLOW_LAYER_INNER_L2 :
3160                                       MLX5_FLOW_LAYER_OUTER_L2;
3161                         mask.eth = flow_tcf_item_mask
3162                                 (items, &rte_flow_item_eth_mask,
3163                                  &flow_tcf_mask_supported.eth,
3164                                  &flow_tcf_mask_empty.eth,
3165                                  sizeof(flow_tcf_mask_supported.eth),
3166                                  error);
3167                         assert(mask.eth);
3168                         if (mask.eth == &flow_tcf_mask_empty.eth)
3169                                 break;
3170                         spec.eth = items->spec;
3171                         if (decap.vxlan &&
3172                             !(item_flags & MLX5_FLOW_LAYER_VXLAN)) {
3173                                 DRV_LOG(WARNING,
3174                                         "outer L2 addresses cannot be forced"
3175                                         " for vxlan decapsulation, parameter"
3176                                         " ignored");
3177                                 break;
3178                         }
3179                         if (mask.eth->type) {
3180                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
3181                                                  spec.eth->type);
3182                                 eth_type_set = 1;
3183                         }
3184                         if (!is_zero_ether_addr(&mask.eth->dst)) {
3185                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST,
3186                                              ETHER_ADDR_LEN,
3187                                              spec.eth->dst.addr_bytes);
3188                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST_MASK,
3189                                              ETHER_ADDR_LEN,
3190                                              mask.eth->dst.addr_bytes);
3191                         }
3192                         if (!is_zero_ether_addr(&mask.eth->src)) {
3193                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC,
3194                                              ETHER_ADDR_LEN,
3195                                              spec.eth->src.addr_bytes);
3196                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC_MASK,
3197                                              ETHER_ADDR_LEN,
3198                                              mask.eth->src.addr_bytes);
3199                         }
3200                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3201                         break;
3202                 case RTE_FLOW_ITEM_TYPE_VLAN:
3203                         assert(!encap.hdr);
3204                         assert(!decap.hdr);
3205                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
3206                         mask.vlan = flow_tcf_item_mask
3207                                 (items, &rte_flow_item_vlan_mask,
3208                                  &flow_tcf_mask_supported.vlan,
3209                                  &flow_tcf_mask_empty.vlan,
3210                                  sizeof(flow_tcf_mask_supported.vlan),
3211                                  error);
3212                         assert(mask.vlan);
3213                         if (!eth_type_set)
3214                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
3215                                                  RTE_BE16(ETH_P_8021Q));
3216                         eth_type_set = 1;
3217                         vlan_present = 1;
3218                         if (mask.vlan == &flow_tcf_mask_empty.vlan)
3219                                 break;
3220                         spec.vlan = items->spec;
3221                         if (mask.vlan->inner_type) {
3222                                 mnl_attr_put_u16(nlh,
3223                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE,
3224                                                  spec.vlan->inner_type);
3225                                 vlan_eth_type_set = 1;
3226                         }
3227                         if (mask.vlan->tci & RTE_BE16(0xe000))
3228                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_VLAN_PRIO,
3229                                                 (rte_be_to_cpu_16
3230                                                  (spec.vlan->tci) >> 13) & 0x7);
3231                         if (mask.vlan->tci & RTE_BE16(0x0fff))
3232                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_VLAN_ID,
3233                                                  rte_be_to_cpu_16
3234                                                  (spec.vlan->tci &
3235                                                   RTE_BE16(0x0fff)));
3236                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3237                         break;
3238                 case RTE_FLOW_ITEM_TYPE_IPV4:
3239                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3240                         mask.ipv4 = flow_tcf_item_mask
3241                                 (items, &rte_flow_item_ipv4_mask,
3242                                  &flow_tcf_mask_supported.ipv4,
3243                                  &flow_tcf_mask_empty.ipv4,
3244                                  sizeof(flow_tcf_mask_supported.ipv4),
3245                                  error);
3246                         assert(mask.ipv4);
3247                         spec.ipv4 = items->spec;
3248                         if (!decap.vxlan) {
3249                                 if (!eth_type_set && !vlan_eth_type_set)
3250                                         mnl_attr_put_u16
3251                                                 (nlh,
3252                                                  vlan_present ?
3253                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE :
3254                                                  TCA_FLOWER_KEY_ETH_TYPE,
3255                                                  RTE_BE16(ETH_P_IP));
3256                                 eth_type_set = 1;
3257                                 vlan_eth_type_set = 1;
3258                                 if (mask.ipv4 == &flow_tcf_mask_empty.ipv4)
3259                                         break;
3260                                 if (mask.ipv4->hdr.next_proto_id) {
3261                                         mnl_attr_put_u8
3262                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3263                                                  spec.ipv4->hdr.next_proto_id);
3264                                         ip_proto_set = 1;
3265                                 }
3266                         } else {
3267                                 assert(mask.ipv4 != &flow_tcf_mask_empty.ipv4);
3268                         }
3269                         if (mask.ipv4->hdr.src_addr) {
3270                                 mnl_attr_put_u32
3271                                         (nlh, decap.vxlan ?
3272                                          TCA_FLOWER_KEY_ENC_IPV4_SRC :
3273                                          TCA_FLOWER_KEY_IPV4_SRC,
3274                                          spec.ipv4->hdr.src_addr);
3275                                 mnl_attr_put_u32
3276                                         (nlh, decap.vxlan ?
3277                                          TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK :
3278                                          TCA_FLOWER_KEY_IPV4_SRC_MASK,
3279                                          mask.ipv4->hdr.src_addr);
3280                         }
3281                         if (mask.ipv4->hdr.dst_addr) {
3282                                 mnl_attr_put_u32
3283                                         (nlh, decap.vxlan ?
3284                                          TCA_FLOWER_KEY_ENC_IPV4_DST :
3285                                          TCA_FLOWER_KEY_IPV4_DST,
3286                                          spec.ipv4->hdr.dst_addr);
3287                                 mnl_attr_put_u32
3288                                         (nlh, decap.vxlan ?
3289                                          TCA_FLOWER_KEY_ENC_IPV4_DST_MASK :
3290                                          TCA_FLOWER_KEY_IPV4_DST_MASK,
3291                                          mask.ipv4->hdr.dst_addr);
3292                         }
3293                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3294                         break;
3295                 case RTE_FLOW_ITEM_TYPE_IPV6:
3296                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3297                         mask.ipv6 = flow_tcf_item_mask
3298                                 (items, &rte_flow_item_ipv6_mask,
3299                                  &flow_tcf_mask_supported.ipv6,
3300                                  &flow_tcf_mask_empty.ipv6,
3301                                  sizeof(flow_tcf_mask_supported.ipv6),
3302                                  error);
3303                         assert(mask.ipv6);
3304                         spec.ipv6 = items->spec;
3305                         if (!decap.vxlan) {
3306                                 if (!eth_type_set || !vlan_eth_type_set) {
3307                                         mnl_attr_put_u16
3308                                                 (nlh,
3309                                                  vlan_present ?
3310                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE :
3311                                                  TCA_FLOWER_KEY_ETH_TYPE,
3312                                                  RTE_BE16(ETH_P_IPV6));
3313                                 }
3314                                 eth_type_set = 1;
3315                                 vlan_eth_type_set = 1;
3316                                 if (mask.ipv6 == &flow_tcf_mask_empty.ipv6)
3317                                         break;
3318                                 if (mask.ipv6->hdr.proto) {
3319                                         mnl_attr_put_u8
3320                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3321                                                  spec.ipv6->hdr.proto);
3322                                         ip_proto_set = 1;
3323                                 }
3324                         } else {
3325                                 assert(mask.ipv6 != &flow_tcf_mask_empty.ipv6);
3326                         }
3327                         if (!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.src_addr)) {
3328                                 mnl_attr_put(nlh, decap.vxlan ?
3329                                              TCA_FLOWER_KEY_ENC_IPV6_SRC :
3330                                              TCA_FLOWER_KEY_IPV6_SRC,
3331                                              IPV6_ADDR_LEN,
3332                                              spec.ipv6->hdr.src_addr);
3333                                 mnl_attr_put(nlh, decap.vxlan ?
3334                                              TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK :
3335                                              TCA_FLOWER_KEY_IPV6_SRC_MASK,
3336                                              IPV6_ADDR_LEN,
3337                                              mask.ipv6->hdr.src_addr);
3338                         }
3339                         if (!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.dst_addr)) {
3340                                 mnl_attr_put(nlh, decap.vxlan ?
3341                                              TCA_FLOWER_KEY_ENC_IPV6_DST :
3342                                              TCA_FLOWER_KEY_IPV6_DST,
3343                                              IPV6_ADDR_LEN,
3344                                              spec.ipv6->hdr.dst_addr);
3345                                 mnl_attr_put(nlh, decap.vxlan ?
3346                                              TCA_FLOWER_KEY_ENC_IPV6_DST_MASK :
3347                                              TCA_FLOWER_KEY_IPV6_DST_MASK,
3348                                              IPV6_ADDR_LEN,
3349                                              mask.ipv6->hdr.dst_addr);
3350                         }
3351                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3352                         break;
3353                 case RTE_FLOW_ITEM_TYPE_UDP:
3354                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3355                         mask.udp = flow_tcf_item_mask
3356                                 (items, &rte_flow_item_udp_mask,
3357                                  &flow_tcf_mask_supported.udp,
3358                                  &flow_tcf_mask_empty.udp,
3359                                  sizeof(flow_tcf_mask_supported.udp),
3360                                  error);
3361                         assert(mask.udp);
3362                         spec.udp = items->spec;
3363                         if (!decap.vxlan) {
3364                                 if (!ip_proto_set)
3365                                         mnl_attr_put_u8
3366                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3367                                                 IPPROTO_UDP);
3368                                 if (mask.udp == &flow_tcf_mask_empty.udp)
3369                                         break;
3370                         } else {
3371                                 assert(mask.udp != &flow_tcf_mask_empty.udp);
3372                                 decap.vxlan->udp_port =
3373                                         rte_be_to_cpu_16
3374                                                 (spec.udp->hdr.dst_port);
3375                         }
3376                         if (mask.udp->hdr.src_port) {
3377                                 mnl_attr_put_u16
3378                                         (nlh, decap.vxlan ?
3379                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT :
3380                                          TCA_FLOWER_KEY_UDP_SRC,
3381                                          spec.udp->hdr.src_port);
3382                                 mnl_attr_put_u16
3383                                         (nlh, decap.vxlan ?
3384                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK :
3385                                          TCA_FLOWER_KEY_UDP_SRC_MASK,
3386                                          mask.udp->hdr.src_port);
3387                         }
3388                         if (mask.udp->hdr.dst_port) {
3389                                 mnl_attr_put_u16
3390                                         (nlh, decap.vxlan ?
3391                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT :
3392                                          TCA_FLOWER_KEY_UDP_DST,
3393                                          spec.udp->hdr.dst_port);
3394                                 mnl_attr_put_u16
3395                                         (nlh, decap.vxlan ?
3396                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK :
3397                                          TCA_FLOWER_KEY_UDP_DST_MASK,
3398                                          mask.udp->hdr.dst_port);
3399                         }
3400                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3401                         break;
3402                 case RTE_FLOW_ITEM_TYPE_TCP:
3403                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3404                         mask.tcp = flow_tcf_item_mask
3405                                 (items, &rte_flow_item_tcp_mask,
3406                                  &flow_tcf_mask_supported.tcp,
3407                                  &flow_tcf_mask_empty.tcp,
3408                                  sizeof(flow_tcf_mask_supported.tcp),
3409                                  error);
3410                         assert(mask.tcp);
3411                         if (!ip_proto_set)
3412                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3413                                                 IPPROTO_TCP);
3414                         if (mask.tcp == &flow_tcf_mask_empty.tcp)
3415                                 break;
3416                         spec.tcp = items->spec;
3417                         if (mask.tcp->hdr.src_port) {
3418                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_SRC,
3419                                                  spec.tcp->hdr.src_port);
3420                                 mnl_attr_put_u16(nlh,
3421                                                  TCA_FLOWER_KEY_TCP_SRC_MASK,
3422                                                  mask.tcp->hdr.src_port);
3423                         }
3424                         if (mask.tcp->hdr.dst_port) {
3425                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_DST,
3426                                                  spec.tcp->hdr.dst_port);
3427                                 mnl_attr_put_u16(nlh,
3428                                                  TCA_FLOWER_KEY_TCP_DST_MASK,
3429                                                  mask.tcp->hdr.dst_port);
3430                         }
3431                         if (mask.tcp->hdr.tcp_flags) {
3432                                 mnl_attr_put_u16
3433                                         (nlh,
3434                                          TCA_FLOWER_KEY_TCP_FLAGS,
3435                                          rte_cpu_to_be_16
3436                                                 (spec.tcp->hdr.tcp_flags));
3437                                 mnl_attr_put_u16
3438                                         (nlh,
3439                                          TCA_FLOWER_KEY_TCP_FLAGS_MASK,
3440                                          rte_cpu_to_be_16
3441                                                 (mask.tcp->hdr.tcp_flags));
3442                         }
3443                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3444                         break;
3445                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3446                         assert(decap.vxlan);
3447                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
3448                         spec.vxlan = items->spec;
3449                         mnl_attr_put_u32(nlh,
3450                                          TCA_FLOWER_KEY_ENC_KEY_ID,
3451                                          vxlan_vni_as_be32(spec.vxlan->vni));
3452                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3453                         break;
3454                 default:
3455                         return rte_flow_error_set(error, ENOTSUP,
3456                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3457                                                   NULL, "item not supported");
3458                 }
3459         }
3460         na_flower_act = mnl_attr_nest_start(nlh, TCA_FLOWER_ACT);
3461         na_act_index_cur = 1;
3462         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3463                 struct nlattr *na_act_index;
3464                 struct nlattr *na_act;
3465                 unsigned int vlan_act;
3466                 unsigned int i;
3467
3468                 switch (actions->type) {
3469                 case RTE_FLOW_ACTION_TYPE_VOID:
3470                         break;
3471                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3472                         conf.port_id = actions->conf;
3473                         if (conf.port_id->original)
3474                                 i = 0;
3475                         else
3476                                 for (i = 0; ptoi[i].ifindex; ++i)
3477                                         if (ptoi[i].port_id == conf.port_id->id)
3478                                                 break;
3479                         assert(ptoi[i].ifindex);
3480                         na_act_index =
3481                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3482                         assert(na_act_index);
3483                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "mirred");
3484                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3485                         assert(na_act);
3486                         if (encap.hdr) {
3487                                 assert(dev_flow->tcf.tunnel);
3488                                 dev_flow->tcf.tunnel->ifindex_ptr =
3489                                         &((struct tc_mirred *)
3490                                         mnl_attr_get_payload
3491                                         (mnl_nlmsg_get_payload_tail
3492                                                 (nlh)))->ifindex;
3493                         }
3494                         mnl_attr_put(nlh, TCA_MIRRED_PARMS,
3495                                      sizeof(struct tc_mirred),
3496                                      &(struct tc_mirred){
3497                                         .action = TC_ACT_STOLEN,
3498                                         .eaction = TCA_EGRESS_REDIR,
3499                                         .ifindex = ptoi[i].ifindex,
3500                                      });
3501                         mnl_attr_nest_end(nlh, na_act);
3502                         mnl_attr_nest_end(nlh, na_act_index);
3503                         break;
3504                 case RTE_FLOW_ACTION_TYPE_JUMP:
3505                         conf.jump = actions->conf;
3506                         na_act_index =
3507                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3508                         assert(na_act_index);
3509                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3510                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3511                         assert(na_act);
3512                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3513                                      sizeof(struct tc_gact),
3514                                      &(struct tc_gact){
3515                                         .action = TC_ACT_GOTO_CHAIN |
3516                                                   conf.jump->group,
3517                                      });
3518                         mnl_attr_nest_end(nlh, na_act);
3519                         mnl_attr_nest_end(nlh, na_act_index);
3520                         break;
3521                 case RTE_FLOW_ACTION_TYPE_DROP:
3522                         na_act_index =
3523                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3524                         assert(na_act_index);
3525                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3526                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3527                         assert(na_act);
3528                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3529                                      sizeof(struct tc_gact),
3530                                      &(struct tc_gact){
3531                                         .action = TC_ACT_SHOT,
3532                                      });
3533                         mnl_attr_nest_end(nlh, na_act);
3534                         mnl_attr_nest_end(nlh, na_act_index);
3535                         break;
3536                 case RTE_FLOW_ACTION_TYPE_COUNT:
3537                         /*
3538                          * Driver adds the count action implicitly for
3539                          * each rule it creates.
3540                          */
3541                         ret = flow_tcf_translate_action_count(dev,
3542                                                               dev_flow, error);
3543                         if (ret < 0)
3544                                 return ret;
3545                         break;
3546                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3547                         conf.of_push_vlan = NULL;
3548                         vlan_act = TCA_VLAN_ACT_POP;
3549                         goto action_of_vlan;
3550                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3551                         conf.of_push_vlan = actions->conf;
3552                         vlan_act = TCA_VLAN_ACT_PUSH;
3553                         goto action_of_vlan;
3554                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3555                         conf.of_set_vlan_vid = actions->conf;
3556                         if (na_vlan_id)
3557                                 goto override_na_vlan_id;
3558                         vlan_act = TCA_VLAN_ACT_MODIFY;
3559                         goto action_of_vlan;
3560                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3561                         conf.of_set_vlan_pcp = actions->conf;
3562                         if (na_vlan_priority)
3563                                 goto override_na_vlan_priority;
3564                         vlan_act = TCA_VLAN_ACT_MODIFY;
3565                         goto action_of_vlan;
3566 action_of_vlan:
3567                         na_act_index =
3568                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3569                         assert(na_act_index);
3570                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "vlan");
3571                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3572                         assert(na_act);
3573                         mnl_attr_put(nlh, TCA_VLAN_PARMS,
3574                                      sizeof(struct tc_vlan),
3575                                      &(struct tc_vlan){
3576                                         .action = TC_ACT_PIPE,
3577                                         .v_action = vlan_act,
3578                                      });
3579                         if (vlan_act == TCA_VLAN_ACT_POP) {
3580                                 mnl_attr_nest_end(nlh, na_act);
3581                                 mnl_attr_nest_end(nlh, na_act_index);
3582                                 break;
3583                         }
3584                         if (vlan_act == TCA_VLAN_ACT_PUSH)
3585                                 mnl_attr_put_u16(nlh,
3586                                                  TCA_VLAN_PUSH_VLAN_PROTOCOL,
3587                                                  conf.of_push_vlan->ethertype);
3588                         na_vlan_id = mnl_nlmsg_get_payload_tail(nlh);
3589                         mnl_attr_put_u16(nlh, TCA_VLAN_PAD, 0);
3590                         na_vlan_priority = mnl_nlmsg_get_payload_tail(nlh);
3591                         mnl_attr_put_u8(nlh, TCA_VLAN_PAD, 0);
3592                         mnl_attr_nest_end(nlh, na_act);
3593                         mnl_attr_nest_end(nlh, na_act_index);
3594                         if (actions->type ==
3595                             RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
3596 override_na_vlan_id:
3597                                 na_vlan_id->nla_type = TCA_VLAN_PUSH_VLAN_ID;
3598                                 *(uint16_t *)mnl_attr_get_payload(na_vlan_id) =
3599                                         rte_be_to_cpu_16
3600                                         (conf.of_set_vlan_vid->vlan_vid);
3601                         } else if (actions->type ==
3602                                    RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
3603 override_na_vlan_priority:
3604                                 na_vlan_priority->nla_type =
3605                                         TCA_VLAN_PUSH_VLAN_PRIORITY;
3606                                 *(uint8_t *)mnl_attr_get_payload
3607                                         (na_vlan_priority) =
3608                                         conf.of_set_vlan_pcp->vlan_pcp;
3609                         }
3610                         break;
3611                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3612                         assert(decap.vxlan);
3613                         assert(dev_flow->tcf.tunnel);
3614                         dev_flow->tcf.tunnel->ifindex_ptr =
3615                                 (unsigned int *)&tcm->tcm_ifindex;
3616                         na_act_index =
3617                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3618                         assert(na_act_index);
3619                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3620                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3621                         assert(na_act);
3622                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3623                                 sizeof(struct tc_tunnel_key),
3624                                 &(struct tc_tunnel_key){
3625                                         .action = TC_ACT_PIPE,
3626                                         .t_action = TCA_TUNNEL_KEY_ACT_RELEASE,
3627                                         });
3628                         mnl_attr_nest_end(nlh, na_act);
3629                         mnl_attr_nest_end(nlh, na_act_index);
3630                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3631                         break;
3632                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3633                         assert(encap.vxlan);
3634                         flow_tcf_vxlan_encap_parse(actions, encap.vxlan);
3635                         na_act_index =
3636                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3637                         assert(na_act_index);
3638                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3639                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3640                         assert(na_act);
3641                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3642                                 sizeof(struct tc_tunnel_key),
3643                                 &(struct tc_tunnel_key){
3644                                         .action = TC_ACT_PIPE,
3645                                         .t_action = TCA_TUNNEL_KEY_ACT_SET,
3646                                         });
3647                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_UDP_DST)
3648                                 mnl_attr_put_u16(nlh,
3649                                          TCA_TUNNEL_KEY_ENC_DST_PORT,
3650                                          encap.vxlan->udp.dst);
3651                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_SRC)
3652                                 mnl_attr_put_u32(nlh,
3653                                          TCA_TUNNEL_KEY_ENC_IPV4_SRC,
3654                                          encap.vxlan->ipv4.src);
3655                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_DST)
3656                                 mnl_attr_put_u32(nlh,
3657                                          TCA_TUNNEL_KEY_ENC_IPV4_DST,
3658                                          encap.vxlan->ipv4.dst);
3659                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_SRC)
3660                                 mnl_attr_put(nlh,
3661                                          TCA_TUNNEL_KEY_ENC_IPV6_SRC,
3662                                          sizeof(encap.vxlan->ipv6.src),
3663                                          &encap.vxlan->ipv6.src);
3664                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_DST)
3665                                 mnl_attr_put(nlh,
3666                                          TCA_TUNNEL_KEY_ENC_IPV6_DST,
3667                                          sizeof(encap.vxlan->ipv6.dst),
3668                                          &encap.vxlan->ipv6.dst);
3669                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_VXLAN_VNI)
3670                                 mnl_attr_put_u32(nlh,
3671                                          TCA_TUNNEL_KEY_ENC_KEY_ID,
3672                                          vxlan_vni_as_be32
3673                                                 (encap.vxlan->vxlan.vni));
3674                         mnl_attr_put_u8(nlh, TCA_TUNNEL_KEY_NO_CSUM, 0);
3675                         mnl_attr_nest_end(nlh, na_act);
3676                         mnl_attr_nest_end(nlh, na_act_index);
3677                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3678                         break;
3679                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3680                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3681                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3682                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3683                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3684                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3685                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3686                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3687                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3688                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3689                         na_act_index =
3690                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3691                         flow_tcf_create_pedit_mnl_msg(nlh,
3692                                                       &actions, item_flags);
3693                         mnl_attr_nest_end(nlh, na_act_index);
3694                         break;
3695                 default:
3696                         return rte_flow_error_set(error, ENOTSUP,
3697                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3698                                                   actions,
3699                                                   "action not supported");
3700                 }
3701         }
3702         assert(na_flower);
3703         assert(na_flower_act);
3704         mnl_attr_nest_end(nlh, na_flower_act);
3705         dev_flow->tcf.ptc_flags = mnl_attr_get_payload
3706                                         (mnl_nlmsg_get_payload_tail(nlh));
3707         mnl_attr_put_u32(nlh, TCA_FLOWER_FLAGS, decap.vxlan ?
3708                                                 0 : TCA_CLS_FLAGS_SKIP_SW);
3709         mnl_attr_nest_end(nlh, na_flower);
3710         if (dev_flow->tcf.tunnel && dev_flow->tcf.tunnel->ifindex_ptr)
3711                 dev_flow->tcf.tunnel->ifindex_org =
3712                         *dev_flow->tcf.tunnel->ifindex_ptr;
3713         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3714         return 0;
3715 }
3716
3717 /**
3718  * Send Netlink message with acknowledgment.
3719  *
3720  * @param tcf
3721  *   Flow context to use.
3722  * @param nlh
3723  *   Message to send. This function always raises the NLM_F_ACK flag before
3724  *   sending.
3725  * @param[in] cb
3726  *   Callback handler for received message.
3727  * @param[in] arg
3728  *   Context pointer for callback handler.
3729  *
3730  * @return
3731  *   0 on success, a negative errno value otherwise and rte_errno is set.
3732  */
3733 static int
3734 flow_tcf_nl_ack(struct mlx5_flow_tcf_context *tcf,
3735                 struct nlmsghdr *nlh,
3736                 mnl_cb_t cb, void *arg)
3737 {
3738         unsigned int portid = mnl_socket_get_portid(tcf->nl);
3739         uint32_t seq = tcf->seq++;
3740         int ret, err = 0;
3741
3742         assert(tcf->nl);
3743         assert(tcf->buf);
3744         if (!seq) {
3745                 /* seq 0 is reserved for kernel event-driven notifications. */
3746                 seq = tcf->seq++;
3747         }
3748         nlh->nlmsg_seq = seq;
3749         nlh->nlmsg_flags |= NLM_F_ACK;
3750         ret = mnl_socket_sendto(tcf->nl, nlh, nlh->nlmsg_len);
3751         if (ret <= 0) {
3752                 /* Message send error occurres. */
3753                 rte_errno = errno;
3754                 return -rte_errno;
3755         }
3756         nlh = (struct nlmsghdr *)(tcf->buf);
3757         /*
3758          * The following loop postpones non-fatal errors until multipart
3759          * messages are complete.
3760          */
3761         while (true) {
3762                 ret = mnl_socket_recvfrom(tcf->nl, tcf->buf, tcf->buf_size);
3763                 if (ret < 0) {
3764                         err = errno;
3765                         /*
3766                          * In case of overflow Will receive till
3767                          * end of multipart message. We may lost part
3768                          * of reply messages but mark and return an error.
3769                          */
3770                         if (err != ENOSPC ||
3771                             !(nlh->nlmsg_flags & NLM_F_MULTI) ||
3772                             nlh->nlmsg_type == NLMSG_DONE)
3773                                 break;
3774                 } else {
3775                         ret = mnl_cb_run(nlh, ret, seq, portid, cb, arg);
3776                         if (!ret) {
3777                                 /*
3778                                  * libmnl returns 0 if DONE or
3779                                  * success ACK message found.
3780                                  */
3781                                 break;
3782                         }
3783                         if (ret < 0) {
3784                                 /*
3785                                  * ACK message with error found
3786                                  * or some error occurred.
3787                                  */
3788                                 err = errno;
3789                                 break;
3790                         }
3791                         /* We should continue receiving. */
3792                 }
3793         }
3794         if (!err)
3795                 return 0;
3796         rte_errno = err;
3797         return -err;
3798 }
3799
3800 #define MNL_BUF_EXTRA_SPACE 16
3801 #define MNL_REQUEST_SIZE_MIN 256
3802 #define MNL_REQUEST_SIZE_MAX 2048
3803 #define MNL_REQUEST_SIZE RTE_MIN(RTE_MAX(sysconf(_SC_PAGESIZE), \
3804                                  MNL_REQUEST_SIZE_MIN), MNL_REQUEST_SIZE_MAX)
3805
3806 /* Data structures used by flow_tcf_xxx_cb() routines. */
3807 struct tcf_nlcb_buf {
3808         LIST_ENTRY(tcf_nlcb_buf) next;
3809         uint32_t size;
3810         alignas(struct nlmsghdr)
3811         uint8_t msg[]; /**< Netlink message data. */
3812 };
3813
3814 struct tcf_nlcb_context {
3815         unsigned int ifindex; /**< Base interface index. */
3816         uint32_t bufsize;
3817         LIST_HEAD(, tcf_nlcb_buf) nlbuf;
3818 };
3819
3820 /**
3821  * Allocate space for netlink command in buffer list
3822  *
3823  * @param[in, out] ctx
3824  *   Pointer to callback context with command buffers list.
3825  * @param[in] size
3826  *   Required size of data buffer to be allocated.
3827  *
3828  * @return
3829  *   Pointer to allocated memory, aligned as message header.
3830  *   NULL if some error occurred.
3831  */
3832 static struct nlmsghdr *
3833 flow_tcf_alloc_nlcmd(struct tcf_nlcb_context *ctx, uint32_t size)
3834 {
3835         struct tcf_nlcb_buf *buf;
3836         struct nlmsghdr *nlh;
3837
3838         size = NLMSG_ALIGN(size);
3839         buf = LIST_FIRST(&ctx->nlbuf);
3840         if (buf && (buf->size + size) <= ctx->bufsize) {
3841                 nlh = (struct nlmsghdr *)&buf->msg[buf->size];
3842                 buf->size += size;
3843                 return nlh;
3844         }
3845         if (size > ctx->bufsize) {
3846                 DRV_LOG(WARNING, "netlink: too long command buffer requested");
3847                 return NULL;
3848         }
3849         buf = rte_malloc(__func__,
3850                         ctx->bufsize + sizeof(struct tcf_nlcb_buf),
3851                         alignof(struct tcf_nlcb_buf));
3852         if (!buf) {
3853                 DRV_LOG(WARNING, "netlink: no memory for command buffer");
3854                 return NULL;
3855         }
3856         LIST_INSERT_HEAD(&ctx->nlbuf, buf, next);
3857         buf->size = size;
3858         nlh = (struct nlmsghdr *)&buf->msg[0];
3859         return nlh;
3860 }
3861
3862 /**
3863  * Send the buffers with prepared netlink commands. Scans the list and
3864  * sends all found buffers. Buffers are sent and freed anyway in order
3865  * to prevent memory leakage if some every message in received packet.
3866  *
3867  * @param[in] tcf
3868  *   Context object initialized by mlx5_flow_tcf_context_create().
3869  * @param[in, out] ctx
3870  *   Pointer to callback context with command buffers list.
3871  *
3872  * @return
3873  *   Zero value on success, negative errno value otherwise
3874  *   and rte_errno is set.
3875  */
3876 static int
3877 flow_tcf_send_nlcmd(struct mlx5_flow_tcf_context *tcf,
3878                     struct tcf_nlcb_context *ctx)
3879 {
3880         struct tcf_nlcb_buf *bc = LIST_FIRST(&ctx->nlbuf);
3881         int ret = 0;
3882
3883         while (bc) {
3884                 struct tcf_nlcb_buf *bn = LIST_NEXT(bc, next);
3885                 struct nlmsghdr *nlh;
3886                 uint32_t msg = 0;
3887                 int rc;
3888
3889                 while (msg < bc->size) {
3890                         /*
3891                          * Send Netlink commands from buffer in one by one
3892                          * fashion. If we send multiple rule deletion commands
3893                          * in one Netlink message and some error occurs it may
3894                          * cause multiple ACK error messages and break sequence
3895                          * numbers of Netlink communication, because we expect
3896                          * the only one ACK reply.
3897                          */
3898                         assert((bc->size - msg) >= sizeof(struct nlmsghdr));
3899                         nlh = (struct nlmsghdr *)&bc->msg[msg];
3900                         assert((bc->size - msg) >= nlh->nlmsg_len);
3901                         msg += nlh->nlmsg_len;
3902                         rc = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
3903                         if (rc) {
3904                                 DRV_LOG(WARNING,
3905                                         "netlink: cleanup error %d", rc);
3906                                 if (!ret)
3907                                         ret = rc;
3908                         }
3909                 }
3910                 rte_free(bc);
3911                 bc = bn;
3912         }
3913         LIST_INIT(&ctx->nlbuf);
3914         return ret;
3915 }
3916
3917 /**
3918  * Collect local IP address rules with scope link attribute  on specified
3919  * network device. This is callback routine called by libmnl mnl_cb_run()
3920  * in loop for every message in received packet.
3921  *
3922  * @param[in] nlh
3923  *   Pointer to reply header.
3924  * @param[in, out] arg
3925  *   Opaque data pointer for this callback.
3926  *
3927  * @return
3928  *   A positive, nonzero value on success, negative errno value otherwise
3929  *   and rte_errno is set.
3930  */
3931 static int
3932 flow_tcf_collect_local_cb(const struct nlmsghdr *nlh, void *arg)
3933 {
3934         struct tcf_nlcb_context *ctx = arg;
3935         struct nlmsghdr *cmd;
3936         struct ifaddrmsg *ifa;
3937         struct nlattr *na;
3938         struct nlattr *na_local = NULL;
3939         struct nlattr *na_peer = NULL;
3940         unsigned char family;
3941         uint32_t size;
3942
3943         if (nlh->nlmsg_type != RTM_NEWADDR) {
3944                 rte_errno = EINVAL;
3945                 return -rte_errno;
3946         }
3947         ifa = mnl_nlmsg_get_payload(nlh);
3948         family = ifa->ifa_family;
3949         if (ifa->ifa_index != ctx->ifindex ||
3950             ifa->ifa_scope != RT_SCOPE_LINK ||
3951             !(ifa->ifa_flags & IFA_F_PERMANENT) ||
3952             (family != AF_INET && family != AF_INET6))
3953                 return 1;
3954         mnl_attr_for_each(na, nlh, sizeof(*ifa)) {
3955                 switch (mnl_attr_get_type(na)) {
3956                 case IFA_LOCAL:
3957                         na_local = na;
3958                         break;
3959                 case IFA_ADDRESS:
3960                         na_peer = na;
3961                         break;
3962                 }
3963                 if (na_local && na_peer)
3964                         break;
3965         }
3966         if (!na_local || !na_peer)
3967                 return 1;
3968         /* Local rule found with scope link, permanent and assigned peer. */
3969         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
3970                MNL_ALIGN(sizeof(struct ifaddrmsg)) +
3971                (family == AF_INET6 ? 2 * SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
3972                                    : 2 * SZ_NLATTR_TYPE_OF(uint32_t));
3973         cmd = flow_tcf_alloc_nlcmd(ctx, size);
3974         if (!cmd) {
3975                 rte_errno = ENOMEM;
3976                 return -rte_errno;
3977         }
3978         cmd = mnl_nlmsg_put_header(cmd);
3979         cmd->nlmsg_type = RTM_DELADDR;
3980         cmd->nlmsg_flags = NLM_F_REQUEST;
3981         ifa = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifa));
3982         ifa->ifa_flags = IFA_F_PERMANENT;
3983         ifa->ifa_scope = RT_SCOPE_LINK;
3984         ifa->ifa_index = ctx->ifindex;
3985         if (family == AF_INET) {
3986                 ifa->ifa_family = AF_INET;
3987                 ifa->ifa_prefixlen = 32;
3988                 mnl_attr_put_u32(cmd, IFA_LOCAL, mnl_attr_get_u32(na_local));
3989                 mnl_attr_put_u32(cmd, IFA_ADDRESS, mnl_attr_get_u32(na_peer));
3990         } else {
3991                 ifa->ifa_family = AF_INET6;
3992                 ifa->ifa_prefixlen = 128;
3993                 mnl_attr_put(cmd, IFA_LOCAL, IPV6_ADDR_LEN,
3994                         mnl_attr_get_payload(na_local));
3995                 mnl_attr_put(cmd, IFA_ADDRESS, IPV6_ADDR_LEN,
3996                         mnl_attr_get_payload(na_peer));
3997         }
3998         assert(size == cmd->nlmsg_len);
3999         return 1;
4000 }
4001
4002 /**
4003  * Cleanup the local IP addresses on outer interface.
4004  *
4005  * @param[in] tcf
4006  *   Context object initialized by mlx5_flow_tcf_context_create().
4007  * @param[in] ifindex
4008  *   Network inferface index to perform cleanup.
4009  */
4010 static void
4011 flow_tcf_encap_local_cleanup(struct mlx5_flow_tcf_context *tcf,
4012                             unsigned int ifindex)
4013 {
4014         struct nlmsghdr *nlh;
4015         struct ifaddrmsg *ifa;
4016         struct tcf_nlcb_context ctx = {
4017                 .ifindex = ifindex,
4018                 .bufsize = MNL_REQUEST_SIZE,
4019                 .nlbuf = LIST_HEAD_INITIALIZER(),
4020         };
4021         int ret;
4022
4023         assert(ifindex);
4024         /*
4025          * Seek and destroy leftovers of local IP addresses with
4026          * matching properties "scope link".
4027          */
4028         nlh = mnl_nlmsg_put_header(tcf->buf);
4029         nlh->nlmsg_type = RTM_GETADDR;
4030         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4031         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
4032         ifa->ifa_family = AF_UNSPEC;
4033         ifa->ifa_index = ifindex;
4034         ifa->ifa_scope = RT_SCOPE_LINK;
4035         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_local_cb, &ctx);
4036         if (ret)
4037                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4038         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4039         if (ret)
4040                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4041 }
4042
4043 /**
4044  * Collect neigh permament rules on specified network device.
4045  * This is callback routine called by libmnl mnl_cb_run() in loop for
4046  * every message in received packet.
4047  *
4048  * @param[in] nlh
4049  *   Pointer to reply header.
4050  * @param[in, out] arg
4051  *   Opaque data pointer for this callback.
4052  *
4053  * @return
4054  *   A positive, nonzero value on success, negative errno value otherwise
4055  *   and rte_errno is set.
4056  */
4057 static int
4058 flow_tcf_collect_neigh_cb(const struct nlmsghdr *nlh, void *arg)
4059 {
4060         struct tcf_nlcb_context *ctx = arg;
4061         struct nlmsghdr *cmd;
4062         struct ndmsg *ndm;
4063         struct nlattr *na;
4064         struct nlattr *na_ip = NULL;
4065         struct nlattr *na_mac = NULL;
4066         unsigned char family;
4067         uint32_t size;
4068
4069         if (nlh->nlmsg_type != RTM_NEWNEIGH) {
4070                 rte_errno = EINVAL;
4071                 return -rte_errno;
4072         }
4073         ndm = mnl_nlmsg_get_payload(nlh);
4074         family = ndm->ndm_family;
4075         if (ndm->ndm_ifindex != (int)ctx->ifindex ||
4076            !(ndm->ndm_state & NUD_PERMANENT) ||
4077            (family != AF_INET && family != AF_INET6))
4078                 return 1;
4079         mnl_attr_for_each(na, nlh, sizeof(*ndm)) {
4080                 switch (mnl_attr_get_type(na)) {
4081                 case NDA_DST:
4082                         na_ip = na;
4083                         break;
4084                 case NDA_LLADDR:
4085                         na_mac = na;
4086                         break;
4087                 }
4088                 if (na_mac && na_ip)
4089                         break;
4090         }
4091         if (!na_mac || !na_ip)
4092                 return 1;
4093         /* Neigh rule with permenent attribute found. */
4094         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4095                MNL_ALIGN(sizeof(struct ndmsg)) +
4096                SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) +
4097                (family == AF_INET6 ? SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
4098                                    : SZ_NLATTR_TYPE_OF(uint32_t));
4099         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4100         if (!cmd) {
4101                 rte_errno = ENOMEM;
4102                 return -rte_errno;
4103         }
4104         cmd = mnl_nlmsg_put_header(cmd);
4105         cmd->nlmsg_type = RTM_DELNEIGH;
4106         cmd->nlmsg_flags = NLM_F_REQUEST;
4107         ndm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ndm));
4108         ndm->ndm_ifindex = ctx->ifindex;
4109         ndm->ndm_state = NUD_PERMANENT;
4110         ndm->ndm_flags = 0;
4111         ndm->ndm_type = 0;
4112         if (family == AF_INET) {
4113                 ndm->ndm_family = AF_INET;
4114                 mnl_attr_put_u32(cmd, NDA_DST, mnl_attr_get_u32(na_ip));
4115         } else {
4116                 ndm->ndm_family = AF_INET6;
4117                 mnl_attr_put(cmd, NDA_DST, IPV6_ADDR_LEN,
4118                              mnl_attr_get_payload(na_ip));
4119         }
4120         mnl_attr_put(cmd, NDA_LLADDR, ETHER_ADDR_LEN,
4121                      mnl_attr_get_payload(na_mac));
4122         assert(size == cmd->nlmsg_len);
4123         return 1;
4124 }
4125
4126 /**
4127  * Cleanup the neigh rules on outer interface.
4128  *
4129  * @param[in] tcf
4130  *   Context object initialized by mlx5_flow_tcf_context_create().
4131  * @param[in] ifindex
4132  *   Network inferface index to perform cleanup.
4133  */
4134 static void
4135 flow_tcf_encap_neigh_cleanup(struct mlx5_flow_tcf_context *tcf,
4136                             unsigned int ifindex)
4137 {
4138         struct nlmsghdr *nlh;
4139         struct ndmsg *ndm;
4140         struct tcf_nlcb_context ctx = {
4141                 .ifindex = ifindex,
4142                 .bufsize = MNL_REQUEST_SIZE,
4143                 .nlbuf = LIST_HEAD_INITIALIZER(),
4144         };
4145         int ret;
4146
4147         assert(ifindex);
4148         /* Seek and destroy leftovers of neigh rules. */
4149         nlh = mnl_nlmsg_put_header(tcf->buf);
4150         nlh->nlmsg_type = RTM_GETNEIGH;
4151         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4152         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4153         ndm->ndm_family = AF_UNSPEC;
4154         ndm->ndm_ifindex = ifindex;
4155         ndm->ndm_state = NUD_PERMANENT;
4156         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_neigh_cb, &ctx);
4157         if (ret)
4158                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4159         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4160         if (ret)
4161                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4162 }
4163
4164 /**
4165  * Collect indices of VXLAN encap/decap interfaces associated with device.
4166  * This is callback routine called by libmnl mnl_cb_run() in loop for
4167  * every message in received packet.
4168  *
4169  * @param[in] nlh
4170  *   Pointer to reply header.
4171  * @param[in, out] arg
4172  *   Opaque data pointer for this callback.
4173  *
4174  * @return
4175  *   A positive, nonzero value on success, negative errno value otherwise
4176  *   and rte_errno is set.
4177  */
4178 static int
4179 flow_tcf_collect_vxlan_cb(const struct nlmsghdr *nlh, void *arg)
4180 {
4181         struct tcf_nlcb_context *ctx = arg;
4182         struct nlmsghdr *cmd;
4183         struct ifinfomsg *ifm;
4184         struct nlattr *na;
4185         struct nlattr *na_info = NULL;
4186         struct nlattr *na_vxlan = NULL;
4187         bool found = false;
4188         unsigned int vxindex;
4189         uint32_t size;
4190
4191         if (nlh->nlmsg_type != RTM_NEWLINK) {
4192                 rte_errno = EINVAL;
4193                 return -rte_errno;
4194         }
4195         ifm = mnl_nlmsg_get_payload(nlh);
4196         if (!ifm->ifi_index) {
4197                 rte_errno = EINVAL;
4198                 return -rte_errno;
4199         }
4200         mnl_attr_for_each(na, nlh, sizeof(*ifm))
4201                 if (mnl_attr_get_type(na) == IFLA_LINKINFO) {
4202                         na_info = na;
4203                         break;
4204                 }
4205         if (!na_info)
4206                 return 1;
4207         mnl_attr_for_each_nested(na, na_info) {
4208                 switch (mnl_attr_get_type(na)) {
4209                 case IFLA_INFO_KIND:
4210                         if (!strncmp("vxlan", mnl_attr_get_str(na),
4211                                      mnl_attr_get_len(na)))
4212                                 found = true;
4213                         break;
4214                 case IFLA_INFO_DATA:
4215                         na_vxlan = na;
4216                         break;
4217                 }
4218                 if (found && na_vxlan)
4219                         break;
4220         }
4221         if (!found || !na_vxlan)
4222                 return 1;
4223         found = false;
4224         mnl_attr_for_each_nested(na, na_vxlan) {
4225                 if (mnl_attr_get_type(na) == IFLA_VXLAN_LINK &&
4226                     mnl_attr_get_u32(na) == ctx->ifindex) {
4227                         found = true;
4228                         break;
4229                 }
4230         }
4231         if (!found)
4232                 return 1;
4233         /* Attached VXLAN device found, store the command to delete. */
4234         vxindex = ifm->ifi_index;
4235         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4236                MNL_ALIGN(sizeof(struct ifinfomsg));
4237         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4238         if (!cmd) {
4239                 rte_errno = ENOMEM;
4240                 return -rte_errno;
4241         }
4242         cmd = mnl_nlmsg_put_header(cmd);
4243         cmd->nlmsg_type = RTM_DELLINK;
4244         cmd->nlmsg_flags = NLM_F_REQUEST;
4245         ifm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifm));
4246         ifm->ifi_family = AF_UNSPEC;
4247         ifm->ifi_index = vxindex;
4248         assert(size == cmd->nlmsg_len);
4249         return 1;
4250 }
4251
4252 /**
4253  * Cleanup the outer interface. Removes all found vxlan devices
4254  * attached to specified index, flushes the meigh and local IP
4255  * datavase.
4256  *
4257  * @param[in] tcf
4258  *   Context object initialized by mlx5_flow_tcf_context_create().
4259  * @param[in] ifindex
4260  *   Network inferface index to perform cleanup.
4261  */
4262 static void
4263 flow_tcf_encap_iface_cleanup(struct mlx5_flow_tcf_context *tcf,
4264                             unsigned int ifindex)
4265 {
4266         struct nlmsghdr *nlh;
4267         struct ifinfomsg *ifm;
4268         struct tcf_nlcb_context ctx = {
4269                 .ifindex = ifindex,
4270                 .bufsize = MNL_REQUEST_SIZE,
4271                 .nlbuf = LIST_HEAD_INITIALIZER(),
4272         };
4273         int ret;
4274
4275         assert(ifindex);
4276         /*
4277          * Seek and destroy leftover VXLAN encap/decap interfaces with
4278          * matching properties.
4279          */
4280         nlh = mnl_nlmsg_put_header(tcf->buf);
4281         nlh->nlmsg_type = RTM_GETLINK;
4282         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4283         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4284         ifm->ifi_family = AF_UNSPEC;
4285         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_vxlan_cb, &ctx);
4286         if (ret)
4287                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4288         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4289         if (ret)
4290                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4291 }
4292
4293 /**
4294  * Emit Netlink message to add/remove local address to the outer device.
4295  * The address being added is visible within the link only (scope link).
4296  *
4297  * Note that an implicit route is maintained by the kernel due to the
4298  * presence of a peer address (IFA_ADDRESS).
4299  *
4300  * These rules are used for encapsultion only and allow to assign
4301  * the outer tunnel source IP address.
4302  *
4303  * @param[in] tcf
4304  *   Libmnl socket context object.
4305  * @param[in] encap
4306  *   Encapsulation properties (source address and its peer).
4307  * @param[in] ifindex
4308  *   Network interface to apply rule.
4309  * @param[in] enable
4310  *   Toggle between add and remove.
4311  * @param[out] error
4312  *   Perform verbose error reporting if not NULL.
4313  *
4314  * @return
4315  *   0 on success, a negative errno value otherwise and rte_errno is set.
4316  */
4317 static int
4318 flow_tcf_rule_local(struct mlx5_flow_tcf_context *tcf,
4319                     const struct flow_tcf_vxlan_encap *encap,
4320                     unsigned int ifindex,
4321                     bool enable,
4322                     struct rte_flow_error *error)
4323 {
4324         struct nlmsghdr *nlh;
4325         struct ifaddrmsg *ifa;
4326         alignas(struct nlmsghdr)
4327         uint8_t buf[mnl_nlmsg_size(sizeof(*ifa) + 128)];
4328
4329         nlh = mnl_nlmsg_put_header(buf);
4330         nlh->nlmsg_type = enable ? RTM_NEWADDR : RTM_DELADDR;
4331         nlh->nlmsg_flags =
4332                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4333         nlh->nlmsg_seq = 0;
4334         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
4335         ifa->ifa_flags = IFA_F_PERMANENT;
4336         ifa->ifa_scope = RT_SCOPE_LINK;
4337         ifa->ifa_index = ifindex;
4338         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4339                 ifa->ifa_family = AF_INET;
4340                 ifa->ifa_prefixlen = 32;
4341                 mnl_attr_put_u32(nlh, IFA_LOCAL, encap->ipv4.src);
4342                 if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST)
4343                         mnl_attr_put_u32(nlh, IFA_ADDRESS,
4344                                               encap->ipv4.dst);
4345         } else {
4346                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4347                 ifa->ifa_family = AF_INET6;
4348                 ifa->ifa_prefixlen = 128;
4349                 mnl_attr_put(nlh, IFA_LOCAL,
4350                                   sizeof(encap->ipv6.src),
4351                                   &encap->ipv6.src);
4352                 if (encap->mask & FLOW_TCF_ENCAP_IPV6_DST)
4353                         mnl_attr_put(nlh, IFA_ADDRESS,
4354                                           sizeof(encap->ipv6.dst),
4355                                           &encap->ipv6.dst);
4356         }
4357         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4358                 return 0;
4359         return rte_flow_error_set(error, rte_errno,
4360                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4361                                   "netlink: cannot complete IFA request"
4362                                   " (ip addr add)");
4363 }
4364
4365 /**
4366  * Emit Netlink message to add/remove neighbor.
4367  *
4368  * @param[in] tcf
4369  *   Libmnl socket context object.
4370  * @param[in] encap
4371  *   Encapsulation properties (destination address).
4372  * @param[in] ifindex
4373  *   Network interface.
4374  * @param[in] enable
4375  *   Toggle between add and remove.
4376  * @param[out] error
4377  *   Perform verbose error reporting if not NULL.
4378  *
4379  * @return
4380  *   0 on success, a negative errno value otherwise and rte_errno is set.
4381  */
4382 static int
4383 flow_tcf_rule_neigh(struct mlx5_flow_tcf_context *tcf,
4384                      const struct flow_tcf_vxlan_encap *encap,
4385                      unsigned int ifindex,
4386                      bool enable,
4387                      struct rte_flow_error *error)
4388 {
4389         struct nlmsghdr *nlh;
4390         struct ndmsg *ndm;
4391         alignas(struct nlmsghdr)
4392         uint8_t buf[mnl_nlmsg_size(sizeof(*ndm) + 128)];
4393
4394         nlh = mnl_nlmsg_put_header(buf);
4395         nlh->nlmsg_type = enable ? RTM_NEWNEIGH : RTM_DELNEIGH;
4396         nlh->nlmsg_flags =
4397                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4398         nlh->nlmsg_seq = 0;
4399         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4400         ndm->ndm_ifindex = ifindex;
4401         ndm->ndm_state = NUD_PERMANENT;
4402         ndm->ndm_flags = 0;
4403         ndm->ndm_type = 0;
4404         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4405                 ndm->ndm_family = AF_INET;
4406                 mnl_attr_put_u32(nlh, NDA_DST, encap->ipv4.dst);
4407         } else {
4408                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4409                 ndm->ndm_family = AF_INET6;
4410                 mnl_attr_put(nlh, NDA_DST, sizeof(encap->ipv6.dst),
4411                                                  &encap->ipv6.dst);
4412         }
4413         if (encap->mask & FLOW_TCF_ENCAP_ETH_SRC && enable)
4414                 DRV_LOG(WARNING,
4415                         "outer ethernet source address cannot be "
4416                         "forced for VXLAN encapsulation");
4417         if (encap->mask & FLOW_TCF_ENCAP_ETH_DST)
4418                 mnl_attr_put(nlh, NDA_LLADDR, sizeof(encap->eth.dst),
4419                                                     &encap->eth.dst);
4420         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4421                 return 0;
4422         return rte_flow_error_set(error, rte_errno,
4423                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4424                                   "netlink: cannot complete ND request"
4425                                   " (ip neigh)");
4426 }
4427
4428 /**
4429  * Manage the local IP addresses and their peers IP addresses on the
4430  * outer interface for encapsulation purposes. The kernel searches the
4431  * appropriate device for tunnel egress traffic using the outer source
4432  * IP, this IP should be assigned to the outer network device, otherwise
4433  * kernel rejects the rule.
4434  *
4435  * Adds or removes the addresses using the Netlink command like this:
4436  *   ip addr add <src_ip> peer <dst_ip> scope link dev <ifouter>
4437  *
4438  * The addresses are local to the netdev ("scope link"), this reduces
4439  * the risk of conflicts. Note that an implicit route is maintained by
4440  * the kernel due to the presence of a peer address (IFA_ADDRESS).
4441  *
4442  * @param[in] tcf
4443  *   Libmnl socket context object.
4444  * @param[in] vtep
4445  *   VTEP object, contains rule database and ifouter index.
4446  * @param[in] dev_flow
4447  *   Flow object, contains the tunnel parameters (for encap only).
4448  * @param[in] enable
4449  *   Toggle between add and remove.
4450  * @param[out] error
4451  *   Perform verbose error reporting if not NULL.
4452  *
4453  * @return
4454  *   0 on success, a negative errno value otherwise and rte_errno is set.
4455  */
4456 static int
4457 flow_tcf_encap_local(struct mlx5_flow_tcf_context *tcf,
4458                      struct tcf_vtep *vtep,
4459                      struct mlx5_flow *dev_flow,
4460                      bool enable,
4461                      struct rte_flow_error *error)
4462 {
4463         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4464         struct tcf_local_rule *rule;
4465         bool found = false;
4466         int ret;
4467
4468         assert(encap);
4469         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4470         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4471                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_DST);
4472                 LIST_FOREACH(rule, &vtep->local, next) {
4473                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_SRC &&
4474                             encap->ipv4.src == rule->ipv4.src &&
4475                             encap->ipv4.dst == rule->ipv4.dst) {
4476                                 found = true;
4477                                 break;
4478                         }
4479                 }
4480         } else {
4481                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4482                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4483                 LIST_FOREACH(rule, &vtep->local, next) {
4484                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_SRC &&
4485                             !memcmp(&encap->ipv6.src, &rule->ipv6.src,
4486                                             sizeof(encap->ipv6.src)) &&
4487                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4488                                             sizeof(encap->ipv6.dst))) {
4489                                 found = true;
4490                                 break;
4491                         }
4492                 }
4493         }
4494         if (found) {
4495                 if (enable) {
4496                         rule->refcnt++;
4497                         return 0;
4498                 }
4499                 if (!rule->refcnt || !--rule->refcnt) {
4500                         LIST_REMOVE(rule, next);
4501                         return flow_tcf_rule_local(tcf, encap,
4502                                         vtep->ifouter, false, error);
4503                 }
4504                 return 0;
4505         }
4506         if (!enable) {
4507                 DRV_LOG(WARNING, "disabling not existing local rule");
4508                 rte_flow_error_set(error, ENOENT,
4509                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4510                                    "disabling not existing local rule");
4511                 return -ENOENT;
4512         }
4513         rule = rte_zmalloc(__func__, sizeof(struct tcf_local_rule),
4514                                 alignof(struct tcf_local_rule));
4515         if (!rule) {
4516                 rte_flow_error_set(error, ENOMEM,
4517                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4518                                    "unable to allocate memory for local rule");
4519                 return -rte_errno;
4520         }
4521         *rule = (struct tcf_local_rule){.refcnt = 0,
4522                                         .mask = 0,
4523                                         };
4524         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4525                 rule->mask = FLOW_TCF_ENCAP_IPV4_SRC
4526                            | FLOW_TCF_ENCAP_IPV4_DST;
4527                 rule->ipv4.src = encap->ipv4.src;
4528                 rule->ipv4.dst = encap->ipv4.dst;
4529         } else {
4530                 rule->mask = FLOW_TCF_ENCAP_IPV6_SRC
4531                            | FLOW_TCF_ENCAP_IPV6_DST;
4532                 memcpy(&rule->ipv6.src, &encap->ipv6.src, IPV6_ADDR_LEN);
4533                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4534         }
4535         ret = flow_tcf_rule_local(tcf, encap, vtep->ifouter, true, error);
4536         if (ret) {
4537                 rte_free(rule);
4538                 return ret;
4539         }
4540         rule->refcnt++;
4541         LIST_INSERT_HEAD(&vtep->local, rule, next);
4542         return 0;
4543 }
4544
4545 /**
4546  * Manage the destination MAC/IP addresses neigh database, kernel uses
4547  * this one to determine the destination MAC address within encapsulation
4548  * header. Adds or removes the entries using the Netlink command like this:
4549  *   ip neigh add dev <ifouter> lladdr <dst_mac> to <dst_ip> nud permanent
4550  *
4551  * @param[in] tcf
4552  *   Libmnl socket context object.
4553  * @param[in] vtep
4554  *   VTEP object, contains rule database and ifouter index.
4555  * @param[in] dev_flow
4556  *   Flow object, contains the tunnel parameters (for encap only).
4557  * @param[in] enable
4558  *   Toggle between add and remove.
4559  * @param[out] error
4560  *   Perform verbose error reporting if not NULL.
4561  *
4562  * @return
4563  *   0 on success, a negative errno value otherwise and rte_errno is set.
4564  */
4565 static int
4566 flow_tcf_encap_neigh(struct mlx5_flow_tcf_context *tcf,
4567                      struct tcf_vtep *vtep,
4568                      struct mlx5_flow *dev_flow,
4569                      bool enable,
4570                      struct rte_flow_error *error)
4571 {
4572         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4573         struct tcf_neigh_rule *rule;
4574         bool found = false;
4575         int ret;
4576
4577         assert(encap);
4578         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4579         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4580                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_SRC);
4581                 LIST_FOREACH(rule, &vtep->neigh, next) {
4582                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_DST &&
4583                             encap->ipv4.dst == rule->ipv4.dst) {
4584                                 found = true;
4585                                 break;
4586                         }
4587                 }
4588         } else {
4589                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4590                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4591                 LIST_FOREACH(rule, &vtep->neigh, next) {
4592                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_DST &&
4593                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4594                                                 sizeof(encap->ipv6.dst))) {
4595                                 found = true;
4596                                 break;
4597                         }
4598                 }
4599         }
4600         if (found) {
4601                 if (memcmp(&encap->eth.dst, &rule->eth,
4602                            sizeof(encap->eth.dst))) {
4603                         DRV_LOG(WARNING, "Destination MAC differs"
4604                                          " in neigh rule");
4605                         rte_flow_error_set(error, EEXIST,
4606                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4607                                            NULL, "Different MAC address"
4608                                            " neigh rule for the same"
4609                                            " destination IP");
4610                                         return -EEXIST;
4611                 }
4612                 if (enable) {
4613                         rule->refcnt++;
4614                         return 0;
4615                 }
4616                 if (!rule->refcnt || !--rule->refcnt) {
4617                         LIST_REMOVE(rule, next);
4618                         return flow_tcf_rule_neigh(tcf, encap,
4619                                                    vtep->ifouter,
4620                                                    false, error);
4621                 }
4622                 return 0;
4623         }
4624         if (!enable) {
4625                 DRV_LOG(WARNING, "Disabling not existing neigh rule");
4626                 rte_flow_error_set(error, ENOENT,
4627                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4628                                    "unable to allocate memory for neigh rule");
4629                 return -ENOENT;
4630         }
4631         rule = rte_zmalloc(__func__, sizeof(struct tcf_neigh_rule),
4632                                 alignof(struct tcf_neigh_rule));
4633         if (!rule) {
4634                 rte_flow_error_set(error, ENOMEM,
4635                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4636                                    "unable to allocate memory for neigh rule");
4637                 return -rte_errno;
4638         }
4639         *rule = (struct tcf_neigh_rule){.refcnt = 0,
4640                                         .mask = 0,
4641                                         };
4642         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4643                 rule->mask = FLOW_TCF_ENCAP_IPV4_DST;
4644                 rule->ipv4.dst = encap->ipv4.dst;
4645         } else {
4646                 rule->mask = FLOW_TCF_ENCAP_IPV6_DST;
4647                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4648         }
4649         memcpy(&rule->eth, &encap->eth.dst, sizeof(rule->eth));
4650         ret = flow_tcf_rule_neigh(tcf, encap, vtep->ifouter, true, error);
4651         if (ret) {
4652                 rte_free(rule);
4653                 return ret;
4654         }
4655         rule->refcnt++;
4656         LIST_INSERT_HEAD(&vtep->neigh, rule, next);
4657         return 0;
4658 }
4659
4660 /* VTEP device list is shared between PMD port instances. */
4661 static LIST_HEAD(, tcf_vtep) vtep_list_vxlan = LIST_HEAD_INITIALIZER();
4662 static pthread_mutex_t vtep_list_mutex = PTHREAD_MUTEX_INITIALIZER;
4663
4664 /**
4665  * Deletes VTEP network device.
4666  *
4667  * @param[in] tcf
4668  *   Context object initialized by mlx5_flow_tcf_context_create().
4669  * @param[in] vtep
4670  *   Object represinting the network device to delete. Memory
4671  *   allocated for this object is freed by routine.
4672  */
4673 static void
4674 flow_tcf_vtep_delete(struct mlx5_flow_tcf_context *tcf,
4675                      struct tcf_vtep *vtep)
4676 {
4677         struct nlmsghdr *nlh;
4678         struct ifinfomsg *ifm;
4679         alignas(struct nlmsghdr)
4680         uint8_t buf[mnl_nlmsg_size(MNL_ALIGN(sizeof(*ifm))) +
4681                     MNL_BUF_EXTRA_SPACE];
4682         int ret;
4683
4684         assert(!vtep->refcnt);
4685         /* Delete only ifaces those we actually created. */
4686         if (vtep->created && vtep->ifindex) {
4687                 DRV_LOG(INFO, "VTEP delete (%d)", vtep->ifindex);
4688                 nlh = mnl_nlmsg_put_header(buf);
4689                 nlh->nlmsg_type = RTM_DELLINK;
4690                 nlh->nlmsg_flags = NLM_F_REQUEST;
4691                 ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4692                 ifm->ifi_family = AF_UNSPEC;
4693                 ifm->ifi_index = vtep->ifindex;
4694                 assert(sizeof(buf) >= nlh->nlmsg_len);
4695                 ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4696                 if (ret)
4697                         DRV_LOG(WARNING, "netlink: error deleting vxlan"
4698                                          " encap/decap ifindex %u",
4699                                          ifm->ifi_index);
4700         }
4701         rte_free(vtep);
4702 }
4703
4704 /**
4705  * Creates VTEP network device.
4706  *
4707  * @param[in] tcf
4708  *   Context object initialized by mlx5_flow_tcf_context_create().
4709  * @param[in] ifouter
4710  *   Outer interface to attach new-created VXLAN device
4711  *   If zero the VXLAN device will not be attached to any device.
4712  *   These VTEPs are used for decapsulation and can be precreated
4713  *   and shared between processes.
4714  * @param[in] port
4715  *   UDP port of created VTEP device.
4716  * @param[out] error
4717  *   Perform verbose error reporting if not NULL.
4718  *
4719  * @return
4720  * Pointer to created device structure on success,
4721  * NULL otherwise and rte_errno is set.
4722  */
4723 #ifdef HAVE_IFLA_VXLAN_COLLECT_METADATA
4724 static struct tcf_vtep*
4725 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf,
4726                      unsigned int ifouter,
4727                      uint16_t port, struct rte_flow_error *error)
4728 {
4729         struct tcf_vtep *vtep;
4730         struct nlmsghdr *nlh;
4731         struct ifinfomsg *ifm;
4732         char name[sizeof(MLX5_VXLAN_DEVICE_PFX) + 24];
4733         alignas(struct nlmsghdr)
4734         uint8_t buf[mnl_nlmsg_size(sizeof(*ifm)) +
4735                     SZ_NLATTR_DATA_OF(sizeof(name)) +
4736                     SZ_NLATTR_NEST * 2 +
4737                     SZ_NLATTR_STRZ_OF("vxlan") +
4738                     SZ_NLATTR_DATA_OF(sizeof(uint32_t)) +
4739                     SZ_NLATTR_DATA_OF(sizeof(uint16_t)) +
4740                     SZ_NLATTR_DATA_OF(sizeof(uint8_t)) * 3 +
4741                     MNL_BUF_EXTRA_SPACE];
4742         struct nlattr *na_info;
4743         struct nlattr *na_vxlan;
4744         rte_be16_t vxlan_port = rte_cpu_to_be_16(port);
4745         int ret;
4746
4747         vtep = rte_zmalloc(__func__, sizeof(*vtep), alignof(struct tcf_vtep));
4748         if (!vtep) {
4749                 rte_flow_error_set(error, ENOMEM,
4750                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4751                                    "unable to allocate memory for VTEP");
4752                 return NULL;
4753         }
4754         *vtep = (struct tcf_vtep){
4755                         .port = port,
4756                         .local = LIST_HEAD_INITIALIZER(),
4757                         .neigh = LIST_HEAD_INITIALIZER(),
4758         };
4759         memset(buf, 0, sizeof(buf));
4760         nlh = mnl_nlmsg_put_header(buf);
4761         nlh->nlmsg_type = RTM_NEWLINK;
4762         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE  | NLM_F_EXCL;
4763         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4764         ifm->ifi_family = AF_UNSPEC;
4765         ifm->ifi_type = 0;
4766         ifm->ifi_index = 0;
4767         ifm->ifi_flags = IFF_UP;
4768         ifm->ifi_change = 0xffffffff;
4769         snprintf(name, sizeof(name), "%s%u", MLX5_VXLAN_DEVICE_PFX, port);
4770         mnl_attr_put_strz(nlh, IFLA_IFNAME, name);
4771         na_info = mnl_attr_nest_start(nlh, IFLA_LINKINFO);
4772         assert(na_info);
4773         mnl_attr_put_strz(nlh, IFLA_INFO_KIND, "vxlan");
4774         na_vxlan = mnl_attr_nest_start(nlh, IFLA_INFO_DATA);
4775         if (ifouter)
4776                 mnl_attr_put_u32(nlh, IFLA_VXLAN_LINK, ifouter);
4777         assert(na_vxlan);
4778         mnl_attr_put_u8(nlh, IFLA_VXLAN_COLLECT_METADATA, 1);
4779         mnl_attr_put_u8(nlh, IFLA_VXLAN_UDP_ZERO_CSUM6_RX, 1);
4780         mnl_attr_put_u8(nlh, IFLA_VXLAN_LEARNING, 0);
4781         mnl_attr_put_u16(nlh, IFLA_VXLAN_PORT, vxlan_port);
4782         mnl_attr_nest_end(nlh, na_vxlan);
4783         mnl_attr_nest_end(nlh, na_info);
4784         assert(sizeof(buf) >= nlh->nlmsg_len);
4785         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4786         if (ret) {
4787                 DRV_LOG(WARNING,
4788                         "netlink: VTEP %s create failure (%d)",
4789                         name, rte_errno);
4790                 if (rte_errno != EEXIST || ifouter)
4791                         /*
4792                          * Some unhandled error occurred or device is
4793                          * for encapsulation and cannot be shared.
4794                          */
4795                         goto error;
4796         } else {
4797                 /*
4798                  * Mark device we actually created.
4799                  * We should explicitly delete
4800                  * when we do not need it anymore.
4801                  */
4802                 vtep->created = 1;
4803         }
4804         /* Try to get ifindex of created of pre-existing device. */
4805         ret = if_nametoindex(name);
4806         if (!ret) {
4807                 DRV_LOG(WARNING,
4808                         "VTEP %s failed to get index (%d)", name, errno);
4809                 rte_flow_error_set
4810                         (error, -errno,
4811                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4812                          "netlink: failed to retrieve VTEP ifindex");
4813                 goto error;
4814         }
4815         vtep->ifindex = ret;
4816         vtep->ifouter = ifouter;
4817         memset(buf, 0, sizeof(buf));
4818         nlh = mnl_nlmsg_put_header(buf);
4819         nlh->nlmsg_type = RTM_NEWLINK;
4820         nlh->nlmsg_flags = NLM_F_REQUEST;
4821         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4822         ifm->ifi_family = AF_UNSPEC;
4823         ifm->ifi_type = 0;
4824         ifm->ifi_index = vtep->ifindex;
4825         ifm->ifi_flags = IFF_UP;
4826         ifm->ifi_change = IFF_UP;
4827         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4828         if (ret) {
4829                 rte_flow_error_set(error, -errno,
4830                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4831                                    "netlink: failed to set VTEP link up");
4832                 DRV_LOG(WARNING, "netlink: VTEP %s set link up failure (%d)",
4833                         name, rte_errno);
4834                 goto clean;
4835         }
4836         ret = mlx5_flow_tcf_init(tcf, vtep->ifindex, error);
4837         if (ret) {
4838                 DRV_LOG(WARNING, "VTEP %s init failure (%d)", name, rte_errno);
4839                 goto clean;
4840         }
4841         DRV_LOG(INFO, "VTEP create (%d, %d)", vtep->port, vtep->ifindex);
4842         vtep->refcnt = 1;
4843         return vtep;
4844 clean:
4845         flow_tcf_vtep_delete(tcf, vtep);
4846         return NULL;
4847 error:
4848         rte_free(vtep);
4849         return NULL;
4850 }
4851 #else
4852 static struct tcf_vtep*
4853 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf __rte_unused,
4854                      unsigned int ifouter __rte_unused,
4855                      uint16_t port __rte_unused,
4856                      struct rte_flow_error *error)
4857 {
4858         rte_flow_error_set(error, ENOTSUP,
4859                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4860                            "netlink: failed to create VTEP, "
4861                            "vxlan metadata are not supported by kernel");
4862         return NULL;
4863 }
4864 #endif /* HAVE_IFLA_VXLAN_COLLECT_METADATA */
4865
4866 /**
4867  * Acquire target interface index for VXLAN tunneling decapsulation.
4868  * In order to share the UDP port within the other interfaces the
4869  * VXLAN device created as not attached to any interface (if created).
4870  *
4871  * @param[in] tcf
4872  *   Context object initialized by mlx5_flow_tcf_context_create().
4873  * @param[in] dev_flow
4874  *   Flow tcf object with tunnel structure pointer set.
4875  * @param[out] error
4876  *   Perform verbose error reporting if not NULL.
4877  * @return
4878  *   Interface descriptor pointer on success,
4879  *   NULL otherwise and rte_errno is set.
4880  */
4881 static struct tcf_vtep*
4882 flow_tcf_decap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
4883                             struct mlx5_flow *dev_flow,
4884                             struct rte_flow_error *error)
4885 {
4886         struct tcf_vtep *vtep;
4887         uint16_t port = dev_flow->tcf.vxlan_decap->udp_port;
4888
4889         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4890                 if (vtep->port == port)
4891                         break;
4892         }
4893         if (vtep && vtep->ifouter) {
4894                 rte_flow_error_set(error, -errno,
4895                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4896                                    "Failed to create decap VTEP with specified"
4897                                    " UDP port, atatched device exists");
4898                 return NULL;
4899         }
4900         if (vtep) {
4901                 /* Device exists, just increment the reference counter. */
4902                 vtep->refcnt++;
4903                 assert(vtep->ifindex);
4904                 return vtep;
4905         }
4906         /* No decapsulation device exists, try to create the new one. */
4907         vtep = flow_tcf_vtep_create(tcf, 0, port, error);
4908         if (vtep)
4909                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
4910         return vtep;
4911 }
4912
4913 /**
4914  * Aqcuire target interface index for VXLAN tunneling encapsulation.
4915  *
4916  * @param[in] tcf
4917  *   Context object initialized by mlx5_flow_tcf_context_create().
4918  * @param[in] ifouter
4919  *   Network interface index to attach VXLAN encap device to.
4920  * @param[in] dev_flow
4921  *   Flow tcf object with tunnel structure pointer set.
4922  * @param[out] error
4923  *   Perform verbose error reporting if not NULL.
4924  * @return
4925  *   Interface descriptor pointer on success,
4926  *   NULL otherwise and rte_errno is set.
4927  */
4928 static struct tcf_vtep*
4929 flow_tcf_encap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
4930                             unsigned int ifouter,
4931                             struct mlx5_flow *dev_flow __rte_unused,
4932                             struct rte_flow_error *error)
4933 {
4934         static uint16_t encap_port = MLX5_VXLAN_PORT_MIN - 1;
4935         struct tcf_vtep *vtep;
4936         int ret;
4937
4938         assert(ifouter);
4939         /* Look whether the attached VTEP for encap is created. */
4940         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4941                 if (vtep->ifouter == ifouter)
4942                         break;
4943         }
4944         if (vtep) {
4945                 /* VTEP already exists, just increment the reference. */
4946                 vtep->refcnt++;
4947         } else {
4948                 uint16_t pcnt;
4949
4950                 /* Not found, we should create the new attached VTEP. */
4951                 flow_tcf_encap_iface_cleanup(tcf, ifouter);
4952                 flow_tcf_encap_local_cleanup(tcf, ifouter);
4953                 flow_tcf_encap_neigh_cleanup(tcf, ifouter);
4954                 for (pcnt = 0; pcnt <= (MLX5_VXLAN_PORT_MAX
4955                                      - MLX5_VXLAN_PORT_MIN); pcnt++) {
4956                         encap_port++;
4957                         /* Wraparound the UDP port index. */
4958                         if (encap_port < MLX5_VXLAN_PORT_MIN ||
4959                             encap_port > MLX5_VXLAN_PORT_MAX)
4960                                 encap_port = MLX5_VXLAN_PORT_MIN;
4961                         /* Check whether UDP port is in already in use. */
4962                         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4963                                 if (vtep->port == encap_port)
4964                                         break;
4965                         }
4966                         if (vtep) {
4967                                 /* Port is in use, try the next one. */
4968                                 vtep = NULL;
4969                                 continue;
4970                         }
4971                         vtep = flow_tcf_vtep_create(tcf, ifouter,
4972                                                     encap_port, error);
4973                         if (vtep) {
4974                                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
4975                                 break;
4976                         }
4977                         if (rte_errno != EEXIST)
4978                                 break;
4979                 }
4980                 if (!vtep)
4981                         return NULL;
4982         }
4983         assert(vtep->ifouter == ifouter);
4984         assert(vtep->ifindex);
4985         /* Create local ipaddr with peer to specify the outer IPs. */
4986         ret = flow_tcf_encap_local(tcf, vtep, dev_flow, true, error);
4987         if (!ret) {
4988                 /* Create neigh rule to specify outer destination MAC. */
4989                 ret = flow_tcf_encap_neigh(tcf, vtep, dev_flow, true, error);
4990                 if (ret)
4991                         flow_tcf_encap_local(tcf, vtep,
4992                                              dev_flow, false, error);
4993         }
4994         if (ret) {
4995                 if (--vtep->refcnt == 0)
4996                         flow_tcf_vtep_delete(tcf, vtep);
4997                 return NULL;
4998         }
4999         return vtep;
5000 }
5001
5002 /**
5003  * Acquires target interface index for tunneling of any type.
5004  * Creates the new VTEP if needed.
5005  *
5006  * @param[in] tcf
5007  *   Context object initialized by mlx5_flow_tcf_context_create().
5008  * @param[in] ifouter
5009  *   Network interface index to attach VXLAN encap device to.
5010  * @param[in] dev_flow
5011  *   Flow tcf object with tunnel structure pointer set.
5012  * @param[out] error
5013  *   Perform verbose error reporting if not NULL.
5014  * @return
5015  *   Interface descriptor pointer on success,
5016  *   NULL otherwise and rte_errno is set.
5017  */
5018 static struct tcf_vtep*
5019 flow_tcf_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
5020                       unsigned int ifouter,
5021                       struct mlx5_flow *dev_flow,
5022                       struct rte_flow_error *error)
5023 {
5024         struct tcf_vtep *vtep = NULL;
5025
5026         assert(dev_flow->tcf.tunnel);
5027         pthread_mutex_lock(&vtep_list_mutex);
5028         switch (dev_flow->tcf.tunnel->type) {
5029         case FLOW_TCF_TUNACT_VXLAN_ENCAP:
5030                 vtep = flow_tcf_encap_vtep_acquire(tcf, ifouter,
5031                                                   dev_flow, error);
5032                 break;
5033         case FLOW_TCF_TUNACT_VXLAN_DECAP:
5034                 vtep = flow_tcf_decap_vtep_acquire(tcf, dev_flow, error);
5035                 break;
5036         default:
5037                 rte_flow_error_set(error, ENOTSUP,
5038                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5039                                    "unsupported tunnel type");
5040                 break;
5041         }
5042         pthread_mutex_unlock(&vtep_list_mutex);
5043         return vtep;
5044 }
5045
5046 /**
5047  * Release tunneling interface by ifindex. Decrements reference
5048  * counter and actually removes the device if counter is zero.
5049  *
5050  * @param[in] tcf
5051  *   Context object initialized by mlx5_flow_tcf_context_create().
5052  * @param[in] vtep
5053  *   VTEP device descriptor structure.
5054  * @param[in] dev_flow
5055  *   Flow tcf object with tunnel structure pointer set.
5056  */
5057 static void
5058 flow_tcf_vtep_release(struct mlx5_flow_tcf_context *tcf,
5059                       struct tcf_vtep *vtep,
5060                       struct mlx5_flow *dev_flow)
5061 {
5062         assert(dev_flow->tcf.tunnel);
5063         pthread_mutex_lock(&vtep_list_mutex);
5064         switch (dev_flow->tcf.tunnel->type) {
5065         case FLOW_TCF_TUNACT_VXLAN_DECAP:
5066                 break;
5067         case FLOW_TCF_TUNACT_VXLAN_ENCAP:
5068                 /* Remove the encap ancillary rules first. */
5069                 flow_tcf_encap_neigh(tcf, vtep, dev_flow, false, NULL);
5070                 flow_tcf_encap_local(tcf, vtep, dev_flow, false, NULL);
5071                 break;
5072         default:
5073                 assert(false);
5074                 DRV_LOG(WARNING, "Unsupported tunnel type");
5075                 break;
5076         }
5077         assert(vtep->refcnt);
5078         if (--vtep->refcnt == 0) {
5079                 LIST_REMOVE(vtep, next);
5080                 flow_tcf_vtep_delete(tcf, vtep);
5081         }
5082         pthread_mutex_unlock(&vtep_list_mutex);
5083 }
5084
5085 struct tcf_nlcb_query {
5086         uint32_t handle;
5087         uint32_t tc_flags;
5088         uint32_t flags_valid:1;
5089 };
5090
5091 /**
5092  * Collect queried rule attributes. This is callback routine called by
5093  * libmnl mnl_cb_run() in loop for every message in received packet.
5094  * Current implementation collects the flower flags only.
5095  *
5096  * @param[in] nlh
5097  *   Pointer to reply header.
5098  * @param[in, out] arg
5099  *   Context pointer for this callback.
5100  *
5101  * @return
5102  *   A positive, nonzero value on success (required by libmnl
5103  *   to continue messages processing).
5104  */
5105 static int
5106 flow_tcf_collect_query_cb(const struct nlmsghdr *nlh, void *arg)
5107 {
5108         struct tcf_nlcb_query *query = arg;
5109         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
5110         struct nlattr *na, *na_opt;
5111         bool flower = false;
5112
5113         if (nlh->nlmsg_type != RTM_NEWTFILTER ||
5114             tcm->tcm_handle != query->handle)
5115                 return 1;
5116         mnl_attr_for_each(na, nlh, sizeof(*tcm)) {
5117                 switch (mnl_attr_get_type(na)) {
5118                 case TCA_KIND:
5119                         if (strcmp(mnl_attr_get_payload(na), "flower")) {
5120                                 /* Not flower filter, drop entire message. */
5121                                 return 1;
5122                         }
5123                         flower = true;
5124                         break;
5125                 case TCA_OPTIONS:
5126                         if (!flower) {
5127                                 /* Not flower options, drop entire message. */
5128                                 return 1;
5129                         }
5130                         /* Check nested flower options. */
5131                         mnl_attr_for_each_nested(na_opt, na) {
5132                                 switch (mnl_attr_get_type(na_opt)) {
5133                                 case TCA_FLOWER_FLAGS:
5134                                         query->flags_valid = 1;
5135                                         query->tc_flags =
5136                                                 mnl_attr_get_u32(na_opt);
5137                                         break;
5138                                 }
5139                         }
5140                         break;
5141                 }
5142         }
5143         return 1;
5144 }
5145
5146 /**
5147  * Query a TC flower rule flags via netlink.
5148  *
5149  * @param[in] tcf
5150  *   Context object initialized by mlx5_flow_tcf_context_create().
5151  * @param[in] dev_flow
5152  *   Pointer to the flow.
5153  * @param[out] pflags
5154  *   pointer to the data retrieved by the query.
5155  *
5156  * @return
5157  *   0 on success, a negative errno value otherwise.
5158  */
5159 static int
5160 flow_tcf_query_flags(struct mlx5_flow_tcf_context *tcf,
5161                      struct mlx5_flow *dev_flow,
5162                      uint32_t *pflags)
5163 {
5164         struct nlmsghdr *nlh;
5165         struct tcmsg *tcm;
5166         struct tcf_nlcb_query query = {
5167                 .handle = dev_flow->tcf.tcm->tcm_handle,
5168         };
5169
5170         nlh = mnl_nlmsg_put_header(tcf->buf);
5171         nlh->nlmsg_type = RTM_GETTFILTER;
5172         nlh->nlmsg_flags = NLM_F_REQUEST;
5173         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5174         memcpy(tcm, dev_flow->tcf.tcm, sizeof(*tcm));
5175         /*
5176          * Ignore Netlink error for filter query operations.
5177          * The reply length is sent by kernel as errno.
5178          * Just check we got the flags option.
5179          */
5180         flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_query_cb, &query);
5181         if (!query.flags_valid) {
5182                 *pflags = 0;
5183                 return -ENOENT;
5184         }
5185         *pflags = query.tc_flags;
5186         return 0;
5187 }
5188
5189 /**
5190  * Query and check the in_hw set for specified rule.
5191  *
5192  * @param[in] tcf
5193  *   Context object initialized by mlx5_flow_tcf_context_create().
5194  * @param[in] dev_flow
5195  *   Pointer to the flow to check.
5196  *
5197  * @return
5198  *   0 on success, a negative errno value otherwise.
5199  */
5200 static int
5201 flow_tcf_check_inhw(struct mlx5_flow_tcf_context *tcf,
5202                     struct mlx5_flow *dev_flow)
5203 {
5204         uint32_t flags;
5205         int ret;
5206
5207         ret = flow_tcf_query_flags(tcf, dev_flow, &flags);
5208         if (ret)
5209                 return ret;
5210         return  (flags & TCA_CLS_FLAGS_IN_HW) ? 0 : -ENOENT;
5211 }
5212
5213 /**
5214  * Remove flow from E-Switch by sending Netlink message.
5215  *
5216  * @param[in] dev
5217  *   Pointer to Ethernet device.
5218  * @param[in, out] flow
5219  *   Pointer to the sub flow.
5220  */
5221 static void
5222 flow_tcf_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5223 {
5224         struct priv *priv = dev->data->dev_private;
5225         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5226         struct mlx5_flow *dev_flow;
5227         struct nlmsghdr *nlh;
5228
5229         if (!flow)
5230                 return;
5231         dev_flow = LIST_FIRST(&flow->dev_flows);
5232         if (!dev_flow)
5233                 return;
5234         /* E-Switch flow can't be expanded. */
5235         assert(!LIST_NEXT(dev_flow, next));
5236         if (dev_flow->tcf.applied) {
5237                 nlh = dev_flow->tcf.nlh;
5238                 nlh->nlmsg_type = RTM_DELTFILTER;
5239                 nlh->nlmsg_flags = NLM_F_REQUEST;
5240                 flow_tcf_nl_ack(ctx, nlh, NULL, NULL);
5241                 if (dev_flow->tcf.tunnel) {
5242                         assert(dev_flow->tcf.tunnel->vtep);
5243                         flow_tcf_vtep_release(ctx,
5244                                 dev_flow->tcf.tunnel->vtep,
5245                                 dev_flow);
5246                         dev_flow->tcf.tunnel->vtep = NULL;
5247                 }
5248                 dev_flow->tcf.applied = 0;
5249         }
5250 }
5251
5252 /**
5253  * Apply flow to E-Switch by sending Netlink message.
5254  *
5255  * @param[in] dev
5256  *   Pointer to Ethernet device.
5257  * @param[in, out] flow
5258  *   Pointer to the sub flow.
5259  * @param[out] error
5260  *   Pointer to the error structure.
5261  *
5262  * @return
5263  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5264  */
5265 static int
5266 flow_tcf_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5267                struct rte_flow_error *error)
5268 {
5269         struct priv *priv = dev->data->dev_private;
5270         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5271         struct mlx5_flow *dev_flow;
5272         struct nlmsghdr *nlh;
5273
5274         dev_flow = LIST_FIRST(&flow->dev_flows);
5275         /* E-Switch flow can't be expanded. */
5276         assert(!LIST_NEXT(dev_flow, next));
5277         if (dev_flow->tcf.applied)
5278                 return 0;
5279         nlh = dev_flow->tcf.nlh;
5280         nlh->nlmsg_type = RTM_NEWTFILTER;
5281         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
5282         if (dev_flow->tcf.tunnel) {
5283                 /*
5284                  * Replace the interface index, target for
5285                  * encapsulation, source for decapsulation.
5286                  */
5287                 assert(!dev_flow->tcf.tunnel->vtep);
5288                 assert(dev_flow->tcf.tunnel->ifindex_ptr);
5289                 /* Acquire actual VTEP device when rule is being applied. */
5290                 dev_flow->tcf.tunnel->vtep =
5291                         flow_tcf_vtep_acquire(ctx,
5292                                         dev_flow->tcf.tunnel->ifindex_org,
5293                                         dev_flow, error);
5294                 if (!dev_flow->tcf.tunnel->vtep)
5295                         return -rte_errno;
5296                 DRV_LOG(INFO, "Replace ifindex: %d->%d",
5297                                 dev_flow->tcf.tunnel->vtep->ifindex,
5298                                 dev_flow->tcf.tunnel->ifindex_org);
5299                 *dev_flow->tcf.tunnel->ifindex_ptr =
5300                         dev_flow->tcf.tunnel->vtep->ifindex;
5301         }
5302         if (!flow_tcf_nl_ack(ctx, nlh, NULL, NULL)) {
5303                 dev_flow->tcf.applied = 1;
5304                 if (*dev_flow->tcf.ptc_flags & TCA_CLS_FLAGS_SKIP_SW)
5305                         return 0;
5306                 /*
5307                  * Rule was applied without skip_sw flag set.
5308                  * We should check whether the rule was acctually
5309                  * accepted by hardware (have look at in_hw flag).
5310                  */
5311                 if (flow_tcf_check_inhw(ctx, dev_flow)) {
5312                         flow_tcf_remove(dev, flow);
5313                         return rte_flow_error_set
5314                                 (error, ENOENT,
5315                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5316                                  "netlink: rule has no in_hw flag set");
5317                 }
5318                 return 0;
5319         }
5320         if (dev_flow->tcf.tunnel) {
5321                 /* Rollback the VTEP configuration if rule apply failed. */
5322                 assert(dev_flow->tcf.tunnel->vtep);
5323                 flow_tcf_vtep_release(ctx, dev_flow->tcf.tunnel->vtep,
5324                                       dev_flow);
5325                 dev_flow->tcf.tunnel->vtep = NULL;
5326         }
5327         return rte_flow_error_set(error, rte_errno,
5328                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5329                                   "netlink: failed to create TC flow rule");
5330 }
5331
5332 /**
5333  * Remove flow from E-Switch and release resources of the device flow.
5334  *
5335  * @param[in] dev
5336  *   Pointer to Ethernet device.
5337  * @param[in, out] flow
5338  *   Pointer to the sub flow.
5339  */
5340 static void
5341 flow_tcf_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5342 {
5343         struct mlx5_flow *dev_flow;
5344
5345         if (!flow)
5346                 return;
5347         flow_tcf_remove(dev, flow);
5348         if (flow->counter) {
5349                 if (--flow->counter->ref_cnt == 0) {
5350                         rte_free(flow->counter);
5351                         flow->counter = NULL;
5352                 }
5353         }
5354         dev_flow = LIST_FIRST(&flow->dev_flows);
5355         if (!dev_flow)
5356                 return;
5357         /* E-Switch flow can't be expanded. */
5358         assert(!LIST_NEXT(dev_flow, next));
5359         LIST_REMOVE(dev_flow, next);
5360         rte_free(dev_flow);
5361 }
5362
5363 /**
5364  * Helper routine for figuring the space size required for a parse buffer.
5365  *
5366  * @param array
5367  *   array of values to use.
5368  * @param idx
5369  *   Current location in array.
5370  * @param value
5371  *   Value to compare with.
5372  *
5373  * @return
5374  *   The maximum between the given value and the array value on index.
5375  */
5376 static uint16_t
5377 flow_tcf_arr_val_max(uint16_t array[], int idx, uint16_t value)
5378 {
5379         return idx < 0 ? (value) : RTE_MAX((array)[idx], value);
5380 }
5381
5382 /**
5383  * Parse rtnetlink message attributes filling the attribute table with the info
5384  * retrieved.
5385  *
5386  * @param tb
5387  *   Attribute table to be filled.
5388  * @param[out] max
5389  *   Maxinum entry in the attribute table.
5390  * @param rte
5391  *   The attributes section in the message to be parsed.
5392  * @param len
5393  *   The length of the attributes section in the message.
5394  */
5395 static void
5396 flow_tcf_nl_parse_rtattr(struct rtattr *tb[], int max,
5397                          struct rtattr *rta, int len)
5398 {
5399         unsigned short type;
5400         memset(tb, 0, sizeof(struct rtattr *) * (max + 1));
5401         while (RTA_OK(rta, len)) {
5402                 type = rta->rta_type;
5403                 if (type <= max && !tb[type])
5404                         tb[type] = rta;
5405                 rta = RTA_NEXT(rta, len);
5406         }
5407 }
5408
5409 /**
5410  * Extract flow counters from flower action.
5411  *
5412  * @param rta
5413  *   flower action stats properties in the Netlink message received.
5414  * @param rta_type
5415  *   The backward sequence of rta_types, as written in the attribute table,
5416  *   we need to traverse in order to get to the requested object.
5417  * @param idx
5418  *   Current location in rta_type table.
5419  * @param[out] data
5420  *   data holding the count statistics of the rte_flow retrieved from
5421  *   the message.
5422  *
5423  * @return
5424  *   0 if data was found and retrieved, -1 otherwise.
5425  */
5426 static int
5427 flow_tcf_nl_action_stats_parse_and_get(struct rtattr *rta,
5428                                        uint16_t rta_type[], int idx,
5429                                        struct gnet_stats_basic *data)
5430 {
5431         int tca_stats_max = flow_tcf_arr_val_max(rta_type, idx,
5432                                                  TCA_STATS_BASIC);
5433         struct rtattr *tbs[tca_stats_max + 1];
5434
5435         if (rta == NULL || idx < 0)
5436                 return -1;
5437         flow_tcf_nl_parse_rtattr(tbs, tca_stats_max,
5438                                  RTA_DATA(rta), RTA_PAYLOAD(rta));
5439         switch (rta_type[idx]) {
5440         case TCA_STATS_BASIC:
5441                 if (tbs[TCA_STATS_BASIC]) {
5442                         memcpy(data, RTA_DATA(tbs[TCA_STATS_BASIC]),
5443                                RTE_MIN(RTA_PAYLOAD(tbs[TCA_STATS_BASIC]),
5444                                sizeof(*data)));
5445                         return 0;
5446                 }
5447                 break;
5448         default:
5449                 break;
5450         }
5451         return -1;
5452 }
5453
5454 /**
5455  * Parse flower single action retrieving the requested action attribute,
5456  * if found.
5457  *
5458  * @param arg
5459  *   flower action properties in the Netlink message received.
5460  * @param rta_type
5461  *   The backward sequence of rta_types, as written in the attribute table,
5462  *   we need to traverse in order to get to the requested object.
5463  * @param idx
5464  *   Current location in rta_type table.
5465  * @param[out] data
5466  *   Count statistics retrieved from the message query.
5467  *
5468  * @return
5469  *   0 if data was found and retrieved, -1 otherwise.
5470  */
5471 static int
5472 flow_tcf_nl_parse_one_action_and_get(struct rtattr *arg,
5473                                      uint16_t rta_type[], int idx, void *data)
5474 {
5475         int tca_act_max = flow_tcf_arr_val_max(rta_type, idx, TCA_ACT_STATS);
5476         struct rtattr *tb[tca_act_max + 1];
5477
5478         if (arg == NULL || idx < 0)
5479                 return -1;
5480         flow_tcf_nl_parse_rtattr(tb, tca_act_max,
5481                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5482         if (tb[TCA_ACT_KIND] == NULL)
5483                 return -1;
5484         switch (rta_type[idx]) {
5485         case TCA_ACT_STATS:
5486                 if (tb[TCA_ACT_STATS])
5487                         return flow_tcf_nl_action_stats_parse_and_get
5488                                         (tb[TCA_ACT_STATS],
5489                                          rta_type, --idx,
5490                                          (struct gnet_stats_basic *)data);
5491                 break;
5492         default:
5493                 break;
5494         }
5495         return -1;
5496 }
5497
5498 /**
5499  * Parse flower action section in the message retrieving the requested
5500  * attribute from the first action that provides it.
5501  *
5502  * @param opt
5503  *   flower section in the Netlink message received.
5504  * @param rta_type
5505  *   The backward sequence of rta_types, as written in the attribute table,
5506  *   we need to traverse in order to get to the requested object.
5507  * @param idx
5508  *   Current location in rta_type table.
5509  * @param[out] data
5510  *   data retrieved from the message query.
5511  *
5512  * @return
5513  *   0 if data was found and retrieved, -1 otherwise.
5514  */
5515 static int
5516 flow_tcf_nl_action_parse_and_get(struct rtattr *arg,
5517                                  uint16_t rta_type[], int idx, void *data)
5518 {
5519         struct rtattr *tb[TCA_ACT_MAX_PRIO + 1];
5520         int i;
5521
5522         if (arg == NULL || idx < 0)
5523                 return -1;
5524         flow_tcf_nl_parse_rtattr(tb, TCA_ACT_MAX_PRIO,
5525                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5526         switch (rta_type[idx]) {
5527         /*
5528          * flow counters are stored in the actions defined by the flow
5529          * and not in the flow itself, therefore we need to traverse the
5530          * flower chain of actions in search for them.
5531          *
5532          * Note that the index is not decremented here.
5533          */
5534         case TCA_ACT_STATS:
5535                 for (i = 0; i <= TCA_ACT_MAX_PRIO; i++) {
5536                         if (tb[i] &&
5537                         !flow_tcf_nl_parse_one_action_and_get(tb[i],
5538                                                               rta_type,
5539                                                               idx, data))
5540                                 return 0;
5541                 }
5542                 break;
5543         default:
5544                 break;
5545         }
5546         return -1;
5547 }
5548
5549 /**
5550  * Parse flower classifier options in the message, retrieving the requested
5551  * attribute if found.
5552  *
5553  * @param opt
5554  *   flower section in the Netlink message received.
5555  * @param rta_type
5556  *   The backward sequence of rta_types, as written in the attribute table,
5557  *   we need to traverse in order to get to the requested object.
5558  * @param idx
5559  *   Current location in rta_type table.
5560  * @param[out] data
5561  *   data retrieved from the message query.
5562  *
5563  * @return
5564  *   0 if data was found and retrieved, -1 otherwise.
5565  */
5566 static int
5567 flow_tcf_nl_opts_parse_and_get(struct rtattr *opt,
5568                                uint16_t rta_type[], int idx, void *data)
5569 {
5570         int tca_flower_max = flow_tcf_arr_val_max(rta_type, idx,
5571                                                   TCA_FLOWER_ACT);
5572         struct rtattr *tb[tca_flower_max + 1];
5573
5574         if (!opt || idx < 0)
5575                 return -1;
5576         flow_tcf_nl_parse_rtattr(tb, tca_flower_max,
5577                                  RTA_DATA(opt), RTA_PAYLOAD(opt));
5578         switch (rta_type[idx]) {
5579         case TCA_FLOWER_ACT:
5580                 if (tb[TCA_FLOWER_ACT])
5581                         return flow_tcf_nl_action_parse_and_get
5582                                                         (tb[TCA_FLOWER_ACT],
5583                                                          rta_type, --idx, data);
5584                 break;
5585         default:
5586                 break;
5587         }
5588         return -1;
5589 }
5590
5591 /**
5592  * Parse Netlink reply on filter query, retrieving the flow counters.
5593  *
5594  * @param nlh
5595  *   Message received from Netlink.
5596  * @param rta_type
5597  *   The backward sequence of rta_types, as written in the attribute table,
5598  *   we need to traverse in order to get to the requested object.
5599  * @param idx
5600  *   Current location in rta_type table.
5601  * @param[out] data
5602  *   data retrieved from the message query.
5603  *
5604  * @return
5605  *   0 if data was found and retrieved, -1 otherwise.
5606  */
5607 static int
5608 flow_tcf_nl_filter_parse_and_get(struct nlmsghdr *cnlh,
5609                                  uint16_t rta_type[], int idx, void *data)
5610 {
5611         struct nlmsghdr *nlh = cnlh;
5612         struct tcmsg *t = NLMSG_DATA(nlh);
5613         int len = nlh->nlmsg_len;
5614         int tca_max = flow_tcf_arr_val_max(rta_type, idx, TCA_OPTIONS);
5615         struct rtattr *tb[tca_max + 1];
5616
5617         if (idx < 0)
5618                 return -1;
5619         if (nlh->nlmsg_type != RTM_NEWTFILTER &&
5620             nlh->nlmsg_type != RTM_GETTFILTER &&
5621             nlh->nlmsg_type != RTM_DELTFILTER)
5622                 return -1;
5623         len -= NLMSG_LENGTH(sizeof(*t));
5624         if (len < 0)
5625                 return -1;
5626         flow_tcf_nl_parse_rtattr(tb, tca_max, TCA_RTA(t), len);
5627         /* Not a TC flower flow - bail out */
5628         if (!tb[TCA_KIND] ||
5629             strcmp(RTA_DATA(tb[TCA_KIND]), "flower"))
5630                 return -1;
5631         switch (rta_type[idx]) {
5632         case TCA_OPTIONS:
5633                 if (tb[TCA_OPTIONS])
5634                         return flow_tcf_nl_opts_parse_and_get(tb[TCA_OPTIONS],
5635                                                               rta_type,
5636                                                               --idx, data);
5637                 break;
5638         default:
5639                 break;
5640         }
5641         return -1;
5642 }
5643
5644 /**
5645  * A callback to parse Netlink reply on TC flower query.
5646  *
5647  * @param nlh
5648  *   Message received from Netlink.
5649  * @param[out] data
5650  *   Pointer to data area to be filled by the parsing routine.
5651  *   assumed to be a pointer to struct flow_tcf_stats_basic.
5652  *
5653  * @return
5654  *   MNL_CB_OK value.
5655  */
5656 static int
5657 flow_tcf_nl_message_get_stats_basic(const struct nlmsghdr *nlh, void *data)
5658 {
5659         /*
5660          * The backward sequence of rta_types to pass in order to get
5661          *  to the counters.
5662          */
5663         uint16_t rta_type[] = { TCA_STATS_BASIC, TCA_ACT_STATS,
5664                                 TCA_FLOWER_ACT, TCA_OPTIONS };
5665         struct flow_tcf_stats_basic *sb_data = data;
5666         union {
5667                 const struct nlmsghdr *c;
5668                 struct nlmsghdr *nc;
5669         } tnlh = { .c = nlh };
5670
5671         if (!flow_tcf_nl_filter_parse_and_get(tnlh.nc, rta_type,
5672                                               RTE_DIM(rta_type) - 1,
5673                                               (void *)&sb_data->counters))
5674                 sb_data->valid = true;
5675         return MNL_CB_OK;
5676 }
5677
5678 /**
5679  * Query a TC flower rule for its statistics via netlink.
5680  *
5681  * @param[in] dev
5682  *   Pointer to Ethernet device.
5683  * @param[in] flow
5684  *   Pointer to the sub flow.
5685  * @param[out] data
5686  *   data retrieved by the query.
5687  * @param[out] error
5688  *   Perform verbose error reporting if not NULL.
5689  *
5690  * @return
5691  *   0 on success, a negative errno value otherwise and rte_errno is set.
5692  */
5693 static int
5694 flow_tcf_query_count(struct rte_eth_dev *dev,
5695                           struct rte_flow *flow,
5696                           void *data,
5697                           struct rte_flow_error *error)
5698 {
5699         struct flow_tcf_stats_basic sb_data;
5700         struct rte_flow_query_count *qc = data;
5701         struct priv *priv = dev->data->dev_private;
5702         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5703         struct mnl_socket *nl = ctx->nl;
5704         struct mlx5_flow *dev_flow;
5705         struct nlmsghdr *nlh;
5706         uint32_t seq = priv->tcf_context->seq++;
5707         ssize_t ret;
5708         assert(qc);
5709
5710         memset(&sb_data, 0, sizeof(sb_data));
5711         dev_flow = LIST_FIRST(&flow->dev_flows);
5712         /* E-Switch flow can't be expanded. */
5713         assert(!LIST_NEXT(dev_flow, next));
5714         if (!dev_flow->flow->counter)
5715                 goto notsup_exit;
5716         nlh = dev_flow->tcf.nlh;
5717         nlh->nlmsg_type = RTM_GETTFILTER;
5718         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_ECHO;
5719         nlh->nlmsg_seq = seq;
5720         if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) == -1)
5721                 goto error_exit;
5722         do {
5723                 ret = mnl_socket_recvfrom(nl, ctx->buf, ctx->buf_size);
5724                 if (ret <= 0)
5725                         break;
5726                 ret = mnl_cb_run(ctx->buf, ret, seq,
5727                                  mnl_socket_get_portid(nl),
5728                                  flow_tcf_nl_message_get_stats_basic,
5729                                  (void *)&sb_data);
5730         } while (ret > 0);
5731         /* Return the delta from last reset. */
5732         if (sb_data.valid) {
5733                 /* Return the delta from last reset. */
5734                 qc->hits_set = 1;
5735                 qc->bytes_set = 1;
5736                 qc->hits = sb_data.counters.packets - flow->counter->hits;
5737                 qc->bytes = sb_data.counters.bytes - flow->counter->bytes;
5738                 if (qc->reset) {
5739                         flow->counter->hits = sb_data.counters.packets;
5740                         flow->counter->bytes = sb_data.counters.bytes;
5741                 }
5742                 return 0;
5743         }
5744         return rte_flow_error_set(error, EINVAL,
5745                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5746                                   NULL,
5747                                   "flow does not have counter");
5748 error_exit:
5749         return rte_flow_error_set
5750                         (error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5751                          NULL, "netlink: failed to read flow rule counters");
5752 notsup_exit:
5753         return rte_flow_error_set
5754                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5755                          NULL, "counters are not available.");
5756 }
5757
5758 /**
5759  * Query a flow.
5760  *
5761  * @see rte_flow_query()
5762  * @see rte_flow_ops
5763  */
5764 static int
5765 flow_tcf_query(struct rte_eth_dev *dev,
5766                struct rte_flow *flow,
5767                const struct rte_flow_action *actions,
5768                void *data,
5769                struct rte_flow_error *error)
5770 {
5771         int ret = -EINVAL;
5772
5773         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5774                 switch (actions->type) {
5775                 case RTE_FLOW_ACTION_TYPE_VOID:
5776                         break;
5777                 case RTE_FLOW_ACTION_TYPE_COUNT:
5778                         ret = flow_tcf_query_count(dev, flow, data, error);
5779                         break;
5780                 default:
5781                         return rte_flow_error_set(error, ENOTSUP,
5782                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5783                                                   actions,
5784                                                   "action not supported");
5785                 }
5786         }
5787         return ret;
5788 }
5789
5790 const struct mlx5_flow_driver_ops mlx5_flow_tcf_drv_ops = {
5791         .validate = flow_tcf_validate,
5792         .prepare = flow_tcf_prepare,
5793         .translate = flow_tcf_translate,
5794         .apply = flow_tcf_apply,
5795         .remove = flow_tcf_remove,
5796         .destroy = flow_tcf_destroy,
5797         .query = flow_tcf_query,
5798 };
5799
5800 /**
5801  * Create and configure a libmnl socket for Netlink flow rules.
5802  *
5803  * @return
5804  *   A valid libmnl socket object pointer on success, NULL otherwise and
5805  *   rte_errno is set.
5806  */
5807 static struct mnl_socket *
5808 flow_tcf_mnl_socket_create(void)
5809 {
5810         struct mnl_socket *nl = mnl_socket_open(NETLINK_ROUTE);
5811
5812         if (nl) {
5813                 mnl_socket_setsockopt(nl, NETLINK_CAP_ACK, &(int){ 1 },
5814                                       sizeof(int));
5815                 if (!mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID))
5816                         return nl;
5817         }
5818         rte_errno = errno;
5819         if (nl)
5820                 mnl_socket_close(nl);
5821         return NULL;
5822 }
5823
5824 /**
5825  * Destroy a libmnl socket.
5826  *
5827  * @param nl
5828  *   Libmnl socket of the @p NETLINK_ROUTE kind.
5829  */
5830 static void
5831 flow_tcf_mnl_socket_destroy(struct mnl_socket *nl)
5832 {
5833         if (nl)
5834                 mnl_socket_close(nl);
5835 }
5836
5837 /**
5838  * Initialize ingress qdisc of a given network interface.
5839  *
5840  * @param ctx
5841  *   Pointer to tc-flower context to use.
5842  * @param ifindex
5843  *   Index of network interface to initialize.
5844  * @param[out] error
5845  *   Perform verbose error reporting if not NULL.
5846  *
5847  * @return
5848  *   0 on success, a negative errno value otherwise and rte_errno is set.
5849  */
5850 int
5851 mlx5_flow_tcf_init(struct mlx5_flow_tcf_context *ctx,
5852                    unsigned int ifindex, struct rte_flow_error *error)
5853 {
5854         struct nlmsghdr *nlh;
5855         struct tcmsg *tcm;
5856         alignas(struct nlmsghdr)
5857         uint8_t buf[mnl_nlmsg_size(sizeof(*tcm)) +
5858                     SZ_NLATTR_STRZ_OF("ingress") +
5859                     MNL_BUF_EXTRA_SPACE];
5860
5861         /* Destroy existing ingress qdisc and everything attached to it. */
5862         nlh = mnl_nlmsg_put_header(buf);
5863         nlh->nlmsg_type = RTM_DELQDISC;
5864         nlh->nlmsg_flags = NLM_F_REQUEST;
5865         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5866         tcm->tcm_family = AF_UNSPEC;
5867         tcm->tcm_ifindex = ifindex;
5868         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
5869         tcm->tcm_parent = TC_H_INGRESS;
5870         assert(sizeof(buf) >= nlh->nlmsg_len);
5871         /* Ignore errors when qdisc is already absent. */
5872         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL) &&
5873             rte_errno != EINVAL && rte_errno != ENOENT)
5874                 return rte_flow_error_set(error, rte_errno,
5875                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5876                                           "netlink: failed to remove ingress"
5877                                           " qdisc");
5878         /* Create fresh ingress qdisc. */
5879         nlh = mnl_nlmsg_put_header(buf);
5880         nlh->nlmsg_type = RTM_NEWQDISC;
5881         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
5882         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5883         tcm->tcm_family = AF_UNSPEC;
5884         tcm->tcm_ifindex = ifindex;
5885         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
5886         tcm->tcm_parent = TC_H_INGRESS;
5887         mnl_attr_put_strz_check(nlh, sizeof(buf), TCA_KIND, "ingress");
5888         assert(sizeof(buf) >= nlh->nlmsg_len);
5889         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL))
5890                 return rte_flow_error_set(error, rte_errno,
5891                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5892                                           "netlink: failed to create ingress"
5893                                           " qdisc");
5894         return 0;
5895 }
5896
5897 /**
5898  * Create libmnl context for Netlink flow rules.
5899  *
5900  * @return
5901  *   A valid libmnl socket object pointer on success, NULL otherwise and
5902  *   rte_errno is set.
5903  */
5904 struct mlx5_flow_tcf_context *
5905 mlx5_flow_tcf_context_create(void)
5906 {
5907         struct mlx5_flow_tcf_context *ctx = rte_zmalloc(__func__,
5908                                                         sizeof(*ctx),
5909                                                         sizeof(uint32_t));
5910         if (!ctx)
5911                 goto error;
5912         ctx->nl = flow_tcf_mnl_socket_create();
5913         if (!ctx->nl)
5914                 goto error;
5915         ctx->buf_size = MNL_SOCKET_BUFFER_SIZE;
5916         ctx->buf = rte_zmalloc(__func__,
5917                                ctx->buf_size, sizeof(uint32_t));
5918         if (!ctx->buf)
5919                 goto error;
5920         ctx->seq = random();
5921         return ctx;
5922 error:
5923         mlx5_flow_tcf_context_destroy(ctx);
5924         return NULL;
5925 }
5926
5927 /**
5928  * Destroy a libmnl context.
5929  *
5930  * @param ctx
5931  *   Libmnl socket of the @p NETLINK_ROUTE kind.
5932  */
5933 void
5934 mlx5_flow_tcf_context_destroy(struct mlx5_flow_tcf_context *ctx)
5935 {
5936         if (!ctx)
5937                 return;
5938         flow_tcf_mnl_socket_destroy(ctx->nl);
5939         rte_free(ctx->buf);
5940         rte_free(ctx);
5941 }