lib/librte_net/rte_net.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2016 6WIND S.A.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35
  36 #include <rte_mbuf.h>
  37 #include <rte_mbuf_ptype.h>
  38 #include <rte_byteorder.h>
  39 #include <rte_ether.h>
  40 #include <rte_ip.h>
  41 #include <rte_tcp.h>
  42 #include <rte_udp.h>
  43 #include <rte_sctp.h>
  44 #include <rte_gre.h>
  45 #include <rte_net.h>
  46
  47 /* get l3 packet type from ip6 next protocol */
  48 static uint32_t
  49 ptype_l3_ip6(uint8_t ip6_proto)
  50 {
  51         static const uint32_t ip6_ext_proto_map[256] = {
  52                 [IPPROTO_HOPOPTS] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6,
  53                 [IPPROTO_ROUTING] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6,
  54                 [IPPROTO_FRAGMENT] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6,
  55                 [IPPROTO_ESP] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6,
  56                 [IPPROTO_AH] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6,
  57                 [IPPROTO_DSTOPTS] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6,
  58         };
  59
  60         return RTE_PTYPE_L3_IPV6 + ip6_ext_proto_map[ip6_proto];
  61 }
  62
  63 /* get l3 packet type from ip version and header length */
  64 static uint32_t
  65 ptype_l3_ip(uint8_t ipv_ihl)
  66 {
  67         static const uint32_t ptype_l3_ip_proto_map[256] = {
  68                 [0x45] = RTE_PTYPE_L3_IPV4,
  69                 [0x46] = RTE_PTYPE_L3_IPV4_EXT,
  70                 [0x47] = RTE_PTYPE_L3_IPV4_EXT,
  71                 [0x48] = RTE_PTYPE_L3_IPV4_EXT,
  72                 [0x49] = RTE_PTYPE_L3_IPV4_EXT,
  73                 [0x4A] = RTE_PTYPE_L3_IPV4_EXT,
  74                 [0x4B] = RTE_PTYPE_L3_IPV4_EXT,
  75                 [0x4C] = RTE_PTYPE_L3_IPV4_EXT,
  76                 [0x4D] = RTE_PTYPE_L3_IPV4_EXT,
  77                 [0x4E] = RTE_PTYPE_L3_IPV4_EXT,
  78                 [0x4F] = RTE_PTYPE_L3_IPV4_EXT,
  79         };
  80
  81         return ptype_l3_ip_proto_map[ipv_ihl];
  82 }
  83
  84 /* get l4 packet type from proto */
  85 static uint32_t
  86 ptype_l4(uint8_t proto)
  87 {
  88         static const uint32_t ptype_l4_proto[256] = {
  89                 [IPPROTO_UDP] = RTE_PTYPE_L4_UDP,
  90                 [IPPROTO_TCP] = RTE_PTYPE_L4_TCP,
  91                 [IPPROTO_SCTP] = RTE_PTYPE_L4_SCTP,
  92         };
  93
  94         return ptype_l4_proto[proto];
  95 }
  96
  97 /* get inner l3 packet type from ip6 next protocol */
  98 static uint32_t
  99 ptype_inner_l3_ip6(uint8_t ip6_proto)
 100 {
 101         static const uint32_t ptype_inner_ip6_ext_proto_map[256] = {
 102                 [IPPROTO_HOPOPTS] = RTE_PTYPE_INNER_L3_IPV6_EXT -
 103                         RTE_PTYPE_INNER_L3_IPV6,
 104                 [IPPROTO_ROUTING] = RTE_PTYPE_INNER_L3_IPV6_EXT -
 105                         RTE_PTYPE_INNER_L3_IPV6,
 106                 [IPPROTO_FRAGMENT] = RTE_PTYPE_INNER_L3_IPV6_EXT -
 107                         RTE_PTYPE_INNER_L3_IPV6,
 108                 [IPPROTO_ESP] = RTE_PTYPE_INNER_L3_IPV6_EXT -
 109                         RTE_PTYPE_INNER_L3_IPV6,
 110                 [IPPROTO_AH] = RTE_PTYPE_INNER_L3_IPV6_EXT -
 111                         RTE_PTYPE_INNER_L3_IPV6,
 112                 [IPPROTO_DSTOPTS] = RTE_PTYPE_INNER_L3_IPV6_EXT -
 113                         RTE_PTYPE_INNER_L3_IPV6,
 114         };
 115
 116         return RTE_PTYPE_INNER_L3_IPV6 +
 117                 ptype_inner_ip6_ext_proto_map[ip6_proto];
 118 }
 119
 120 /* get inner l3 packet type from ip version and header length */
 121 static uint32_t
 122 ptype_inner_l3_ip(uint8_t ipv_ihl)
 123 {
 124         static const uint32_t ptype_inner_l3_ip_proto_map[256] = {
 125                 [0x45] = RTE_PTYPE_INNER_L3_IPV4,
 126                 [0x46] = RTE_PTYPE_INNER_L3_IPV4_EXT,
 127                 [0x47] = RTE_PTYPE_INNER_L3_IPV4_EXT,
 128                 [0x48] = RTE_PTYPE_INNER_L3_IPV4_EXT,
 129                 [0x49] = RTE_PTYPE_INNER_L3_IPV4_EXT,
 130                 [0x4A] = RTE_PTYPE_INNER_L3_IPV4_EXT,
 131                 [0x4B] = RTE_PTYPE_INNER_L3_IPV4_EXT,
 132                 [0x4C] = RTE_PTYPE_INNER_L3_IPV4_EXT,
 133                 [0x4D] = RTE_PTYPE_INNER_L3_IPV4_EXT,
 134                 [0x4E] = RTE_PTYPE_INNER_L3_IPV4_EXT,
 135                 [0x4F] = RTE_PTYPE_INNER_L3_IPV4_EXT,
 136         };
 137
 138         return ptype_inner_l3_ip_proto_map[ipv_ihl];
 139 }
 140
 141 /* get inner l4 packet type from proto */
 142 static uint32_t
 143 ptype_inner_l4(uint8_t proto)
 144 {
 145         static const uint32_t ptype_inner_l4_proto[256] = {
 146                 [IPPROTO_UDP] = RTE_PTYPE_INNER_L4_UDP,
 147                 [IPPROTO_TCP] = RTE_PTYPE_INNER_L4_TCP,
 148                 [IPPROTO_SCTP] = RTE_PTYPE_INNER_L4_SCTP,
 149         };
 150
 151         return ptype_inner_l4_proto[proto];
 152 }
 153
 154 /* get the tunnel packet type if any, update proto and off. */
 155 static uint32_t
 156 ptype_tunnel(uint16_t *proto, const struct rte_mbuf *m,
 157         uint32_t *off)
 158 {
 159         switch (*proto) {
 160         case IPPROTO_GRE: {
 161                 static const uint8_t opt_len[16] = {
 162                         [0x0] = 4,
 163                         [0x1] = 8,
 164                         [0x2] = 8,
 165                         [0x8] = 8,
 166                         [0x3] = 12,
 167                         [0x9] = 12,
 168                         [0xa] = 12,
 169                         [0xb] = 16,
 170                 };
 171                 const struct gre_hdr *gh;
 172                 struct gre_hdr gh_copy;
 173                 uint16_t flags;
 174
 175                 gh = rte_pktmbuf_read(m, *off, sizeof(*gh), &gh_copy);
 176                 if (unlikely(gh == NULL))
 177                         return 0;
 178
 179                 flags = rte_be_to_cpu_16(*(const uint16_t *)gh);
 180                 flags >>= 12;
 181                 if (opt_len[flags] == 0)
 182                         return 0;
 183
 184                 *off += opt_len[flags];
 185                 *proto = gh->proto;
 186                 if (*proto == rte_cpu_to_be_16(ETHER_TYPE_TEB))
 187                         return RTE_PTYPE_TUNNEL_NVGRE;
 188                 else
 189                         return RTE_PTYPE_TUNNEL_GRE;
 190         }
 191         case IPPROTO_IPIP:
 192                 *proto = rte_cpu_to_be_16(ETHER_TYPE_IPv4);
 193                 return RTE_PTYPE_TUNNEL_IP;
 194         case IPPROTO_IPV6:
 195                 *proto = rte_cpu_to_be_16(ETHER_TYPE_IPv6);
 196                 return RTE_PTYPE_TUNNEL_IP; /* IP is also valid for IPv6 */
 197         default:
 198                 return 0;
 199         }
 200 }
 201
 202 /* get the ipv4 header length */
 203 static uint8_t
 204 ip4_hlen(const struct ipv4_hdr *hdr)
 205 {
 206         return (hdr->version_ihl & 0xf) * 4;
 207 }
 208
 209 /* parse ipv6 extended headers, update offset and return next proto */
 210 static uint16_t
 211 skip_ip6_ext(uint16_t proto, const struct rte_mbuf *m, uint32_t *off,
 212         int *frag)
 213 {
 214         struct ext_hdr {
 215                 uint8_t next_hdr;
 216                 uint8_t len;
 217         };
 218         const struct ext_hdr *xh;
 219         struct ext_hdr xh_copy;
 220         unsigned int i;
 221
 222         *frag = 0;
 223
 224 #define MAX_EXT_HDRS 5
 225         for (i = 0; i < MAX_EXT_HDRS; i++) {
 226                 switch (proto) {
 227                 case IPPROTO_HOPOPTS:
 228                 case IPPROTO_ROUTING:
 229                 case IPPROTO_DSTOPTS:
 230                         xh = rte_pktmbuf_read(m, *off, sizeof(*xh),
 231                                 &xh_copy);
 232                         if (xh == NULL)
 233                                 return 0;
 234                         *off += (xh->len + 1) * 8;
 235                         proto = xh->next_hdr;
 236                         break;
 237                 case IPPROTO_FRAGMENT:
 238                         xh = rte_pktmbuf_read(m, *off, sizeof(*xh),
 239                                 &xh_copy);
 240                         if (xh == NULL)
 241                                 return 0;
 242                         *off += 8;
 243                         proto = xh->next_hdr;
 244                         *frag = 1;
 245                         return proto; /* this is always the last ext hdr */
 246                 case IPPROTO_NONE:
 247                         return 0;
 248                 default:
 249                         return proto;
 250                 }
 251         }
 252         return 0;
 253 }
 254
 255 /* parse mbuf data to get packet type */
 256 uint32_t rte_net_get_ptype(const struct rte_mbuf *m,
 257         struct rte_net_hdr_lens *hdr_lens, uint32_t layers)
 258 {
 259         struct rte_net_hdr_lens local_hdr_lens;
 260         const struct ether_hdr *eh;
 261         struct ether_hdr eh_copy;
 262         uint32_t pkt_type = RTE_PTYPE_L2_ETHER;
 263         uint32_t off = 0;
 264         uint16_t proto;
 265
 266         if (hdr_lens == NULL)
 267                 hdr_lens = &local_hdr_lens;
 268
 269         eh = rte_pktmbuf_read(m, off, sizeof(*eh), &eh_copy);
 270         if (unlikely(eh == NULL))
 271                 return 0;
 272         proto = eh->ether_type;
 273         off = sizeof(*eh);
 274         hdr_lens->l2_len = off;
 275
 276         if ((layers & RTE_PTYPE_L2_MASK) == 0)
 277                 return 0;
 278
 279         if (proto == rte_cpu_to_be_16(ETHER_TYPE_IPv4))
 280                 goto l3; /* fast path if packet is IPv4 */
 281
 282         if (proto == rte_cpu_to_be_16(ETHER_TYPE_VLAN)) {
 283                 const struct vlan_hdr *vh;
 284                 struct vlan_hdr vh_copy;
 285
 286                 pkt_type = RTE_PTYPE_L2_ETHER_VLAN;
 287                 vh = rte_pktmbuf_read(m, off, sizeof(*vh), &vh_copy);
 288                 if (unlikely(vh == NULL))
 289                         return pkt_type;
 290                 off += sizeof(*vh);
 291                 hdr_lens->l2_len += sizeof(*vh);
 292                 proto = vh->eth_proto;
 293         } else if (proto == rte_cpu_to_be_16(ETHER_TYPE_QINQ)) {
 294                 const struct vlan_hdr *vh;
 295                 struct vlan_hdr vh_copy;
 296
 297                 pkt_type = RTE_PTYPE_L2_ETHER_QINQ;
 298                 vh = rte_pktmbuf_read(m, off + sizeof(*vh), sizeof(*vh),
 299                         &vh_copy);
 300                 if (unlikely(vh == NULL))
 301                         return pkt_type;
 302                 off += 2 * sizeof(*vh);
 303                 hdr_lens->l2_len += 2 * sizeof(*vh);
 304                 proto = vh->eth_proto;
 305         }
 306
 307  l3:
 308         if ((layers & RTE_PTYPE_L3_MASK) == 0)
 309                 return pkt_type;
 310
 311         if (proto == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) {
 312                 const struct ipv4_hdr *ip4h;
 313                 struct ipv4_hdr ip4h_copy;
 314
 315                 ip4h = rte_pktmbuf_read(m, off, sizeof(*ip4h), &ip4h_copy);
 316                 if (unlikely(ip4h == NULL))
 317                         return pkt_type;
 318
 319                 pkt_type |= ptype_l3_ip(ip4h->version_ihl);
 320                 hdr_lens->l3_len = ip4_hlen(ip4h);
 321                 off += hdr_lens->l3_len;
 322
 323                 if ((layers & RTE_PTYPE_L4_MASK) == 0)
 324                         return pkt_type;
 325
 326                 if (ip4h->fragment_offset & rte_cpu_to_be_16(
 327                                 IPV4_HDR_OFFSET_MASK | IPV4_HDR_MF_FLAG)) {
 328                         pkt_type |= RTE_PTYPE_L4_FRAG;
 329                         hdr_lens->l4_len = 0;
 330                         return pkt_type;
 331                 }
 332                 proto = ip4h->next_proto_id;
 333                 pkt_type |= ptype_l4(proto);
 334         } else if (proto == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) {
 335                 const struct ipv6_hdr *ip6h;
 336                 struct ipv6_hdr ip6h_copy;
 337                 int frag = 0;
 338
 339                 ip6h = rte_pktmbuf_read(m, off, sizeof(*ip6h), &ip6h_copy);
 340                 if (unlikely(ip6h == NULL))
 341                         return pkt_type;
 342
 343                 proto = ip6h->proto;
 344                 hdr_lens->l3_len = sizeof(*ip6h);
 345                 off += hdr_lens->l3_len;
 346                 pkt_type |= ptype_l3_ip6(proto);
 347                 if ((pkt_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV6_EXT) {
 348                         proto = skip_ip6_ext(proto, m, &off, &frag);
 349                         hdr_lens->l3_len = off - hdr_lens->l2_len;
 350                 }
 351                 if (proto == 0)
 352                         return pkt_type;
 353
 354                 if ((layers & RTE_PTYPE_L4_MASK) == 0)
 355                         return pkt_type;
 356
 357                 if (frag) {
 358                         pkt_type |= RTE_PTYPE_L4_FRAG;
 359                         hdr_lens->l4_len = 0;
 360                         return pkt_type;
 361                 }
 362                 pkt_type |= ptype_l4(proto);
 363         }
 364
 365         if ((pkt_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP) {
 366                 hdr_lens->l4_len = sizeof(struct udp_hdr);
 367                 return pkt_type;
 368         } else if ((pkt_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) {
 369                 const struct tcp_hdr *th;
 370                 struct tcp_hdr th_copy;
 371
 372                 th = rte_pktmbuf_read(m, off, sizeof(*th), &th_copy);
 373                 if (unlikely(th == NULL))
 374                         return pkt_type & (RTE_PTYPE_L2_MASK |
 375                                 RTE_PTYPE_L3_MASK);
 376                 hdr_lens->l4_len = (th->data_off & 0xf0) >> 2;
 377                 return pkt_type;
 378         } else if ((pkt_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP) {
 379                 hdr_lens->l4_len = sizeof(struct sctp_hdr);
 380                 return pkt_type;
 381         } else {
 382                 uint32_t prev_off = off;
 383
 384                 hdr_lens->l4_len = 0;
 385
 386                 if ((layers & RTE_PTYPE_TUNNEL_MASK) == 0)
 387                         return pkt_type;
 388
 389                 pkt_type |= ptype_tunnel(&proto, m, &off);
 390                 hdr_lens->tunnel_len = off - prev_off;
 391         }
 392
 393         /* same job for inner header: we need to duplicate the code
 394          * because the packet types do not have the same value.
 395          */
 396         if ((layers & RTE_PTYPE_INNER_L2_MASK) == 0)
 397                 return pkt_type;
 398
 399         if (proto == rte_cpu_to_be_16(ETHER_TYPE_TEB)) {
 400                 eh = rte_pktmbuf_read(m, off, sizeof(*eh), &eh_copy);
 401                 if (unlikely(eh == NULL))
 402                         return pkt_type;
 403                 pkt_type |= RTE_PTYPE_INNER_L2_ETHER;
 404                 proto = eh->ether_type;
 405                 off += sizeof(*eh);
 406                 hdr_lens->inner_l2_len = sizeof(*eh);
 407         }
 408
 409         if (proto == rte_cpu_to_be_16(ETHER_TYPE_VLAN)) {
 410                 const struct vlan_hdr *vh;
 411                 struct vlan_hdr vh_copy;
 412
 413                 pkt_type &= ~RTE_PTYPE_INNER_L2_MASK;
 414                 pkt_type |= RTE_PTYPE_INNER_L2_ETHER_VLAN;
 415                 vh = rte_pktmbuf_read(m, off, sizeof(*vh), &vh_copy);
 416                 if (unlikely(vh == NULL))
 417                         return pkt_type;
 418                 off += sizeof(*vh);
 419                 hdr_lens->inner_l2_len += sizeof(*vh);
 420                 proto = vh->eth_proto;
 421         } else if (proto == rte_cpu_to_be_16(ETHER_TYPE_QINQ)) {
 422                 const struct vlan_hdr *vh;
 423                 struct vlan_hdr vh_copy;
 424
 425                 pkt_type &= ~RTE_PTYPE_INNER_L2_MASK;
 426                 pkt_type |= RTE_PTYPE_INNER_L2_ETHER_QINQ;
 427                 vh = rte_pktmbuf_read(m, off + sizeof(*vh), sizeof(*vh),
 428                         &vh_copy);
 429                 if (unlikely(vh == NULL))
 430                         return pkt_type;
 431                 off += 2 * sizeof(*vh);
 432                 hdr_lens->inner_l2_len += 2 * sizeof(*vh);
 433                 proto = vh->eth_proto;
 434         }
 435
 436         if ((layers & RTE_PTYPE_INNER_L3_MASK) == 0)
 437                 return pkt_type;
 438
 439         if (proto == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) {
 440                 const struct ipv4_hdr *ip4h;
 441                 struct ipv4_hdr ip4h_copy;
 442
 443                 ip4h = rte_pktmbuf_read(m, off, sizeof(*ip4h), &ip4h_copy);
 444                 if (unlikely(ip4h == NULL))
 445                         return pkt_type;
 446
 447                 pkt_type |= ptype_inner_l3_ip(ip4h->version_ihl);
 448                 hdr_lens->inner_l3_len = ip4_hlen(ip4h);
 449                 off += hdr_lens->inner_l3_len;
 450
 451                 if ((layers & RTE_PTYPE_INNER_L4_MASK) == 0)
 452                         return pkt_type;
 453                 if (ip4h->fragment_offset &
 454                                 rte_cpu_to_be_16(IPV4_HDR_OFFSET_MASK |
 455                                         IPV4_HDR_MF_FLAG)) {
 456                         pkt_type |= RTE_PTYPE_INNER_L4_FRAG;
 457                         hdr_lens->inner_l4_len = 0;
 458                         return pkt_type;
 459                 }
 460                 proto = ip4h->next_proto_id;
 461                 pkt_type |= ptype_inner_l4(proto);
 462         } else if (proto == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) {
 463                 const struct ipv6_hdr *ip6h;
 464                 struct ipv6_hdr ip6h_copy;
 465                 int frag = 0;
 466
 467                 ip6h = rte_pktmbuf_read(m, off, sizeof(*ip6h), &ip6h_copy);
 468                 if (unlikely(ip6h == NULL))
 469                         return pkt_type;
 470
 471                 proto = ip6h->proto;
 472                 hdr_lens->inner_l3_len = sizeof(*ip6h);
 473                 off += hdr_lens->inner_l3_len;
 474                 pkt_type |= ptype_inner_l3_ip6(proto);
 475                 if ((pkt_type & RTE_PTYPE_INNER_L3_MASK) ==
 476                                 RTE_PTYPE_INNER_L3_IPV6_EXT) {
 477                         uint32_t prev_off;
 478
 479                         prev_off = off;
 480                         proto = skip_ip6_ext(proto, m, &off, &frag);
 481                         hdr_lens->inner_l3_len += off - prev_off;
 482                 }
 483                 if (proto == 0)
 484                         return pkt_type;
 485
 486                 if ((layers & RTE_PTYPE_INNER_L4_MASK) == 0)
 487                         return pkt_type;
 488
 489                 if (frag) {
 490                         pkt_type |= RTE_PTYPE_INNER_L4_FRAG;
 491                         hdr_lens->inner_l4_len = 0;
 492                         return pkt_type;
 493                 }
 494                 pkt_type |= ptype_inner_l4(proto);
 495         }
 496
 497         if ((pkt_type & RTE_PTYPE_INNER_L4_MASK) == RTE_PTYPE_INNER_L4_UDP) {
 498                 hdr_lens->inner_l4_len = sizeof(struct udp_hdr);
 499         } else if ((pkt_type & RTE_PTYPE_INNER_L4_MASK) ==
 500                         RTE_PTYPE_INNER_L4_TCP) {
 501                 const struct tcp_hdr *th;
 502                 struct tcp_hdr th_copy;
 503
 504                 th = rte_pktmbuf_read(m, off, sizeof(*th), &th_copy);
 505                 if (unlikely(th == NULL))
 506                         return pkt_type & (RTE_PTYPE_INNER_L2_MASK |
 507                                 RTE_PTYPE_INNER_L3_MASK);
 508                 hdr_lens->inner_l4_len = (th->data_off & 0xf0) >> 2;
 509         } else if ((pkt_type & RTE_PTYPE_INNER_L4_MASK) ==
 510                         RTE_PTYPE_INNER_L4_SCTP) {
 511                 hdr_lens->inner_l4_len = sizeof(struct sctp_hdr);
 512         } else {
 513                 hdr_lens->inner_l4_len = 0;
 514         }
 515
 516         return pkt_type;
 517 }