New upstream version 17.11-rc3

[deb_dpdk.git] / drivers / net / ixgbe / ixgbe_rxtx_vec_sse.c

diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
index e704a7f..486239b 100644 (file)
--- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
+++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
@@ -86,8 +86,8 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq)
                mb0 = rxep[0].mbuf;
                mb1 = rxep[1].mbuf;
 
-               /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */
-               RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_physaddr) !=
+               /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
+               RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
                                offsetof(struct rte_mbuf, buf_addr) + 8);
                vaddr0 = _mm_loadu_si128((__m128i *)&(mb0->buf_addr));
                vaddr1 = _mm_loadu_si128((__m128i *)&(mb1->buf_addr));
@@ -122,6 +122,43 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq)
        IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id);
 }
 
+#ifdef RTE_LIBRTE_SECURITY
+static inline void
+desc_to_olflags_v_ipsec(__m128i descs[4], struct rte_mbuf **rx_pkts)
+{
+       __m128i sterr, rearm, tmp_e, tmp_p;
+       uint32_t *rearm0 = (uint32_t *)rx_pkts[0]->rearm_data + 2;
+       uint32_t *rearm1 = (uint32_t *)rx_pkts[1]->rearm_data + 2;
+       uint32_t *rearm2 = (uint32_t *)rx_pkts[2]->rearm_data + 2;
+       uint32_t *rearm3 = (uint32_t *)rx_pkts[3]->rearm_data + 2;
+       const __m128i ipsec_sterr_msk =
+                       _mm_set1_epi32(IXGBE_RXDADV_IPSEC_STATUS_SECP |
+                                      IXGBE_RXDADV_IPSEC_ERROR_AUTH_FAILED);
+       const __m128i ipsec_proc_msk  =
+                       _mm_set1_epi32(IXGBE_RXDADV_IPSEC_STATUS_SECP);
+       const __m128i ipsec_err_flag  =
+                       _mm_set1_epi32(PKT_RX_SEC_OFFLOAD_FAILED |
+                                      PKT_RX_SEC_OFFLOAD);
+       const __m128i ipsec_proc_flag = _mm_set1_epi32(PKT_RX_SEC_OFFLOAD);
+
+       rearm = _mm_set_epi32(*rearm3, *rearm2, *rearm1, *rearm0);
+       sterr = _mm_set_epi32(_mm_extract_epi32(descs[3], 2),
+                             _mm_extract_epi32(descs[2], 2),
+                             _mm_extract_epi32(descs[1], 2),
+                             _mm_extract_epi32(descs[0], 2));
+       sterr = _mm_and_si128(sterr, ipsec_sterr_msk);
+       tmp_e = _mm_cmpeq_epi32(sterr, ipsec_sterr_msk);
+       tmp_p = _mm_cmpeq_epi32(sterr, ipsec_proc_msk);
+       sterr = _mm_or_si128(_mm_and_si128(tmp_e, ipsec_err_flag),
+                               _mm_and_si128(tmp_p, ipsec_proc_flag));
+       rearm = _mm_or_si128(rearm, sterr);
+       *rearm0 = _mm_extract_epi32(rearm, 0);
+       *rearm1 = _mm_extract_epi32(rearm, 1);
+       *rearm2 = _mm_extract_epi32(rearm, 2);
+       *rearm3 = _mm_extract_epi32(rearm, 3);
+}
+#endif
+
 static inline void
 desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags,
        struct rte_mbuf **rx_pkts)
@@ -310,6 +347,9 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
        volatile union ixgbe_adv_rx_desc *rxdp;
        struct ixgbe_rx_entry *sw_ring;
        uint16_t nb_pkts_recd;
+#ifdef RTE_LIBRTE_SECURITY
+       uint8_t use_ipsec = rxq->using_ipsec;
+#endif
        int pos;
        uint64_t var;
        __m128i shuf_msk;
@@ -397,7 +437,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
        sw_ring = &rxq->sw_ring[rxq->rx_tail];
 
        /* ensure these 2 flags are in the lower 8 bits */
-       RTE_BUILD_BUG_ON((PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED) > UINT8_MAX);
+       RTE_BUILD_BUG_ON((PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED) > UINT8_MAX);
        vlan_flags = rxq->vlan_flags & UINT8_MAX;
 
        /* A. load 4 packet in one loop
@@ -473,6 +513,11 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
                /* set ol_flags with vlan packet type */
                desc_to_olflags_v(descs, mbuf_init, vlan_flags, &rx_pkts[pos]);
 
+#ifdef RTE_LIBRTE_SECURITY
+               if (unlikely(use_ipsec))
+                       desc_to_olflags_v_ipsec(descs, &rx_pkts[pos]);
+#endif
+
                /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
                pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);
                pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust);
@@ -604,7 +649,7 @@ vtx1(volatile union ixgbe_adv_tx_desc *txdp,
 {
        __m128i descriptor = _mm_set_epi64x((uint64_t)pkt->pkt_len << 46 |
                        flags | pkt->data_len,
-                       pkt->buf_physaddr + pkt->data_off);
+                       pkt->buf_iova + pkt->data_off);
        _mm_store_si128((__m128i *)&txdp->read, descriptor);
 }