X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=drivers%2Fnet%2Fixgbe%2Fixgbe_rxtx_vec_sse.c;h=dd7d1778286c1449f41a8339d0b4c36d385223f9;hb=9365d6cfd641d5ade83591c5f5dfa2bf32808e4e;hp=4f95debd38b8844fb9688a90504e45490b2e7788;hpb=809f08006d56e7ba4ce190b0a63d44acf62d8044;p=deb_dpdk.git

diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
index 4f95debd..dd7d1778 100644
--- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
+++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
@@ -145,7 +145,7 @@ static inline void
 desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags,
 	struct rte_mbuf **rx_pkts)
 {
-	__m128i ptype0, ptype1, vtag0, vtag1;
+	__m128i ptype0, ptype1, vtag0, vtag1, csum;
 	union {
 		uint16_t e[4];
 		uint64_t dword;
@@ -156,24 +156,45 @@ desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags,
 			0x0000, 0x0000, 0x0000, 0x0000,
 			0x000F, 0x000F, 0x000F, 0x000F);
 
+	/* mask the lower byte of ol_flags */
+	const __m128i ol_flags_msk = _mm_set_epi16(
+			0x0000, 0x0000, 0x0000, 0x0000,
+			0x00FF, 0x00FF, 0x00FF, 0x00FF);
+
 	/* map rss type to rss hash flag */
 	const __m128i rss_flags = _mm_set_epi8(PKT_RX_FDIR, 0, 0, 0,
 			0, 0, 0, PKT_RX_RSS_HASH,
 			PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH, 0,
 			PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, 0);
 
-	/* mask everything except vlan present bit */
-	const __m128i vlan_msk = _mm_set_epi16(
-			0x0000, 0x0000,
-			0x0000, 0x0000,
-			IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP,
-			IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP);
-	/* map vlan present (0x8) to ol_flags */
-	const __m128i vlan_map = _mm_set_epi8(
+	/* mask everything except vlan present and l4/ip csum error */
+	const __m128i vlan_csum_msk = _mm_set_epi16(
+		(IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 16,
+		(IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 16,
+		(IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 16,
+		(IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 16,
+		IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP,
+		IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP);
+	/* map vlan present (0x8), IPE (0x2), L4E (0x1) to ol_flags */
+	const __m128i vlan_csum_map_lo = _mm_set_epi8(
+		0, 0, 0, 0,
+		vlan_flags | PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD,
+		vlan_flags | PKT_RX_IP_CKSUM_BAD,
+		vlan_flags | PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD,
+		vlan_flags | PKT_RX_IP_CKSUM_GOOD,
+		0, 0, 0, 0,
+		PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD,
+		PKT_RX_IP_CKSUM_BAD,
+		PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD,
+		PKT_RX_IP_CKSUM_GOOD);
+
+	const __m128i vlan_csum_map_hi = _mm_set_epi8(
 		0, 0, 0, 0,
-		0, 0, 0, vlan_flags,
+		0, PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0,
+		PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t),
 		0, 0, 0, 0,
-		0, 0, 0, 0);
+		0, PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0,
+		PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t));
 
 	ptype0 = _mm_unpacklo_epi16(descs[0], descs[1]);
 	ptype1 = _mm_unpacklo_epi16(descs[2], descs[3]);
@@ -185,8 +206,26 @@ desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags,
 	ptype0 = _mm_shuffle_epi8(rss_flags, ptype0);
 
 	vtag1 = _mm_unpacklo_epi32(vtag0, vtag1);
-	vtag1 = _mm_and_si128(vtag1, vlan_msk);
-	vtag1 = _mm_shuffle_epi8(vlan_map, vtag1);
+	vtag1 = _mm_and_si128(vtag1, vlan_csum_msk);
+
+	/* csum bits are in the most significant, to use shuffle we need to
+	 * shift them. Change mask to 0xc000 to 0x0003.
+	 */
+	csum = _mm_srli_epi16(vtag1, 14);
+
+	/* now or the most significant 64 bits containing the checksum
+	 * flags with the vlan present flags.
+	 */
+	csum = _mm_srli_si128(csum, 8);
+	vtag1 = _mm_or_si128(csum, vtag1);
+
+	/* convert VP, IPE, L4E to ol_flags */
+	vtag0 = _mm_shuffle_epi8(vlan_csum_map_hi, vtag1);
+	vtag0 = _mm_slli_epi16(vtag0, sizeof(uint8_t));
+
+	vtag1 = _mm_shuffle_epi8(vlan_csum_map_lo, vtag1);
+	vtag1 = _mm_and_si128(vtag1, ol_flags_msk);
+	vtag1 = _mm_or_si128(vtag0, vtag1);
 
 	vtag1 = _mm_or_si128(ptype0, vtag1);
 	vol.dword = _mm_cvtsi128_si64(vtag1);
@@ -197,7 +236,9 @@ desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags,
 	rx_pkts[3]->ol_flags = vol.e[3];
 }
 #else
-#define desc_to_olflags_v(desc, rx_pkts) do {} while (0)
+#define desc_to_olflags_v(desc, vlan_flags, rx_pkts) do { \
+		RTE_SET_USED(vlan_flags); \
+	} while (0)
 #endif
 
 /*
@@ -208,7 +249,6 @@ desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags,
  * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST
  *   numbers of DD bit
  * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two
- * - don't support ol_flags for rss and csum err
  */
 static inline uint16_t
 _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
@@ -241,7 +281,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
 	 */
 	rxdp = rxq->rx_ring + rxq->rx_tail;
 
-	_mm_prefetch((const void *)rxdp, _MM_HINT_T0);
+	rte_prefetch0(rxdp);
 
 	/* See if we need to rearm the RX queue - gives the prefetch a bit
 	 * of time to act
@@ -295,28 +335,39 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
 		__m128i descs[RTE_IXGBE_DESCS_PER_LOOP];
 		__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
 		__m128i zero, staterr, sterr_tmp1, sterr_tmp2;
-		__m128i mbp1, mbp2; /* two mbuf pointer in one XMM reg. */
+		/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */
+		__m128i mbp1;
+#if defined(RTE_ARCH_X86_64)
+		__m128i mbp2;
+#endif
 
-		/* B.1 load 1 mbuf point */
+		/* B.1 load 2 (64 bit) or 4 (32 bit) mbuf points */
 		mbp1 = _mm_loadu_si128((__m128i *)&sw_ring[pos]);
 
 		/* Read desc statuses backwards to avoid race condition */
 		/* A.1 load 4 pkts desc */
 		descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3));
+		rte_compiler_barrier();
 
-		/* B.2 copy 2 mbuf point into rx_pkts  */
+		/* B.2 copy 2 64 bit or 4 32 bit mbuf point into rx_pkts */
 		_mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1);
 
-		/* B.1 load 1 mbuf point */
+#if defined(RTE_ARCH_X86_64)
+		/* B.1 load 2 64 bit mbuf points */
 		mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]);
+#endif
 
 		descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2));
+		rte_compiler_barrier();
 		/* B.1 load 2 mbuf point */
 		descs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1));
+		rte_compiler_barrier();
 		descs[0] = _mm_loadu_si128((__m128i *)(rxdp));
 
+#if defined(RTE_ARCH_X86_64)
 		/* B.2 copy 2 mbuf point into rx_pkts  */
 		_mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2);
+#endif
 
 		if (split_packet) {
 			rte_mbuf_prefetch_part2(rx_pkts[pos]);
@@ -423,7 +474,6 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
  * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST
  *   numbers of DD bit
  * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two
- * - don't support ol_flags for rss and csum err
  */
 uint16_t
 ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
@@ -436,7 +486,6 @@ ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
  * vPMD receive routine that reassembles scattered packets
  *
  * Notice:
- * - don't support ol_flags for rss and csum err
  * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet
  * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST
  *   numbers of DD bit