misc: replace CLIB_PREFETCH with clib_prefetch_{load,store}

Type: refactor
Change-Id: Id10cbf52e8f2dd809080a228d8fa282308be84ac
Signed-off-by: Damjan Marion <damarion@cisco.com>

diff --git a/src/examples/sample-plugin/sample/node.c b/src/examples/sample-plugin/sample/node.c
index 2c71b48..a31c3e8 100644 (file)
--- a/src/examples/sample-plugin/sample/node.c
+++ b/src/examples/sample-plugin/sample/node.c
@@ -124,8 +124,8 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p2->data);
+           clib_prefetch_store (p3->data);
          }
 
          /* speculatively enqueue b0 and b1 to the current next frame */
@@ -322,8 +322,8 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p2->data);
+           clib_prefetch_store (p3->data);
          }
 
          /* speculatively enqueue b0 and b1 to the current next frame */
@@ -507,10 +507,10 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
          vlib_prefetch_buffer_header (b[5], STORE);
          vlib_prefetch_buffer_header (b[6], STORE);
          vlib_prefetch_buffer_header (b[7], STORE);
-         CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, STORE);
+         clib_prefetch_store (&b[4]->data);
+         clib_prefetch_store (&b[5]->data);
+         clib_prefetch_store (&b[6]->data);
+         clib_prefetch_store (&b[7]->data);
        }
 
       src_dst0 = ((u8x16 *) vlib_buffer_get_current (b[0]))[0];

diff --git a/src/plugins/acl/dataplane_node.c b/src/plugins/acl/dataplane_node.c
index 3a351a0..1a050f5 100644 (file)
--- a/src/plugins/acl/dataplane_node.c
+++ b/src/plugins/acl/dataplane_node.c
@@ -277,7 +277,7 @@ acl_fa_node_common_prepare_fn (vlib_main_t * vm,
        for (ii = ACL_PLUGIN_PREFETCH_GAP * vec_sz;
             ii < (ACL_PLUGIN_PREFETCH_GAP + 1) * vec_sz; ii++)
          {
-           CLIB_PREFETCH (b[ii], CLIB_CACHE_LINE_BYTES, LOAD);
+           clib_prefetch_load (b[ii]);
            CLIB_PREFETCH (b[ii]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
          }
       }

diff --git a/src/plugins/adl/ip4_allowlist.c b/src/plugins/adl/ip4_allowlist.c
index ca7b4c1..316e2cb 100644 (file)
--- a/src/plugins/adl/ip4_allowlist.c
+++ b/src/plugins/adl/ip4_allowlist.c
@@ -98,30 +98,28 @@ VLIB_NODE_FN (ip4_adl_allowlist_node) (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
-         }
-
-          /* speculatively enqueue b0 and b1 to the current next frame */
-         to_next[0] = bi0 = from[0];
-         to_next[1] = bi1 = from[1];
-         from += 2;
-         to_next += 2;
-         n_left_from -= 2;
-         n_left_to_next -= 2;
-
-         b0 = vlib_get_buffer (vm, bi0);
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+           clib_prefetch_store (p2->data);
+           clib_prefetch_store (p3->data);
+         }
+
+         /* speculatively enqueue b0 and b1 to the current next frame */
+         to_next[0] = bi0 = from[0];
+         to_next[1] = bi1 = from[1];
+         from += 2;
+         to_next += 2;
+         n_left_from -= 2;
+         n_left_to_next -= 2;
 
-         ip0 = vlib_buffer_get_current (b0);
+         b0 = vlib_get_buffer (vm, bi0);
+         sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 
-         ccm0 = cm->adl_config_mains + VNET_ADL_IP4;
+         ip0 = vlib_buffer_get_current (b0);
 
-         c0 = vnet_get_config_data
-              (&ccm0->config_main,
-               &adl_buffer (b0)->adl.current_config_index,
-               &next0,
-               sizeof (c0[0]));
+         ccm0 = cm->adl_config_mains + VNET_ADL_IP4;
+
+         c0 = vnet_get_config_data (
+           &ccm0->config_main, &adl_buffer (b0)->adl.current_config_index,
+           &next0, sizeof (c0[0]));
 
          mtrie0 = &ip4_fib_get (c0->fib_index)->mtrie;
 

diff --git a/src/plugins/adl/ip6_allowlist.c b/src/plugins/adl/ip6_allowlist.c
index d272f10..5f38484 100644 (file)
--- a/src/plugins/adl/ip6_allowlist.c
+++ b/src/plugins/adl/ip6_allowlist.c
@@ -96,8 +96,8 @@ VLIB_NODE_FN (ip6_adl_allowlist_node) (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p2->data);
+           clib_prefetch_store (p3->data);
          }
 
           /* speculatively enqueue b0 and b1 to the current next frame */

diff --git a/src/plugins/adl/node.c b/src/plugins/adl/node.c
index 7dfaf3b..5578c13 100644 (file)
--- a/src/plugins/adl/node.c
+++ b/src/plugins/adl/node.c
@@ -88,8 +88,8 @@ VLIB_NODE_FN (adl_input_node) (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p2->data);
+           clib_prefetch_store (p3->data);
          }
 
           /* speculatively enqueue b0 and b1 to the current next frame */

diff --git a/src/plugins/avf/input.c b/src/plugins/avf/input.c
index 4d037b4..1406d78 100644 (file)
--- a/src/plugins/avf/input.c
+++ b/src/plugins/avf/input.c
@@ -309,14 +309,10 @@ avf_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
       if (next + 11 < size)
        {
          int stride = 8;
-         CLIB_PREFETCH ((void *) (fd + (next + stride)),
-                        CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH ((void *) (fd + (next + stride + 1)),
-                        CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH ((void *) (fd + (next + stride + 2)),
-                        CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH ((void *) (fd + (next + stride + 3)),
-                        CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load ((void *) (fd + (next + stride)));
+         clib_prefetch_load ((void *) (fd + (next + stride + 1)));
+         clib_prefetch_load ((void *) (fd + (next + stride + 2)));
+         clib_prefetch_load ((void *) (fd + (next + stride + 3)));
        }
 
 #ifdef CLIB_HAVE_VEC256
@@ -375,8 +371,7 @@ avf_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
       continue;
     one_by_one:
 #endif
-      CLIB_PREFETCH ((void *) (fd + ((next + 8) & mask)),
-                    CLIB_CACHE_LINE_BYTES, LOAD);
+      clib_prefetch_load ((void *) (fd + ((next + 8) & mask)));
 
       if (avf_rxd_is_not_dd (d))
        break;

diff --git a/src/plugins/cnat/cnat_session.c b/src/plugins/cnat/cnat_session.c
index bb04de0..216d257 100644 (file)
--- a/src/plugins/cnat/cnat_session.c
+++ b/src/plugins/cnat/cnat_session.c
@@ -192,13 +192,13 @@ cnat_session_scan (vlib_main_t * vm, f64 start_time, int i)
        {
          BVT (clib_bihash_bucket) * b =
            BV (clib_bihash_get_bucket) (h, i + 3);
-         CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (b);
          b = BV (clib_bihash_get_bucket) (h, i + 1);
          if (!BV (clib_bihash_bucket_is_empty) (b))
            {
              BVT (clib_bihash_value) * v =
                BV (clib_bihash_get_value) (h, b->offset);
-             CLIB_PREFETCH (v, CLIB_CACHE_LINE_BYTES, LOAD);
+             clib_prefetch_load (v);
            }
        }
 

diff --git a/src/plugins/crypto_sw_scheduler/main.c b/src/plugins/crypto_sw_scheduler/main.c
index 2842b18..b0548fa 100644 (file)
--- a/src/plugins/crypto_sw_scheduler/main.c
+++ b/src/plugins/crypto_sw_scheduler/main.c
@@ -370,7 +370,7 @@ crypto_sw_scheduler_dequeue_aead (vlib_main_t * vm,
       while (n_elts--)
        {
          if (n_elts > 1)
-           CLIB_PREFETCH (fe + 1, CLIB_CACHE_LINE_BYTES, LOAD);
+           clib_prefetch_load (fe + 1);
 
          crypto_sw_scheduler_convert_aead (vm, ptd, fe, fe - f->elts, bi[0],
                                            sync_op_id, aad_len, tag_len);
@@ -438,7 +438,7 @@ crypto_sw_scheduler_dequeue_link (vlib_main_t * vm,
       while (n_elts--)
        {
          if (n_elts > 1)
-           CLIB_PREFETCH (fe + 1, CLIB_CACHE_LINE_BYTES, LOAD);
+           clib_prefetch_load (fe + 1);
 
          crypto_sw_scheduler_convert_link_crypto (vm, ptd,
                                                   cm->keys + fe->key_index,

diff --git a/src/plugins/ct6/ct6_in2out.c b/src/plugins/ct6/ct6_in2out.c
index 39da789..b8bda18 100644 (file)
--- a/src/plugins/ct6/ct6_in2out.c
+++ b/src/plugins/ct6/ct6_in2out.c
@@ -165,10 +165,10 @@ ct6_in2out_inline (vlib_main_t * vm,
          vlib_prefetch_buffer_header (b[5], STORE);
          vlib_prefetch_buffer_header (b[6], STORE);
          vlib_prefetch_buffer_header (b[7], STORE);
-         CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, STORE);
+         clib_prefetch_store (b[4]->data);
+         clib_prefetch_store (b[5]->data);
+         clib_prefetch_store (b[6]->data);
+         clib_prefetch_store (b[7]->data);
        }
 
       /* $$$$ process 4x pkts right here */

diff --git a/src/plugins/ct6/ct6_out2in.c b/src/plugins/ct6/ct6_out2in.c
index bbe9c2e..ebb6da5 100644 (file)
--- a/src/plugins/ct6/ct6_out2in.c
+++ b/src/plugins/ct6/ct6_out2in.c
@@ -102,10 +102,10 @@ ct6_out2in_inline (vlib_main_t * vm,
          vlib_prefetch_buffer_header (b[5], STORE);
          vlib_prefetch_buffer_header (b[6], STORE);
          vlib_prefetch_buffer_header (b[7], STORE);
-         CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, STORE);
+         clib_prefetch_store (b[4]->data);
+         clib_prefetch_store (b[5]->data);
+         clib_prefetch_store (b[6]->data);
+         clib_prefetch_store (b[7]->data);
        }
 
       /* $$$$ process 4x pkts right here */

diff --git a/src/plugins/dns/reply_node.c b/src/plugins/dns/reply_node.c
index 5d7e735..b15943a 100644 (file)
--- a/src/plugins/dns/reply_node.c
+++ b/src/plugins/dns/reply_node.c
@@ -93,8 +93,8 @@ dns46_reply_node_fn (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p2->data);
+           clib_prefetch_store (p3->data);
          }
 
          /* speculatively enqueue b0 and b1 to the current next frame */

diff --git a/src/plugins/dns/request_node.c b/src/plugins/dns/request_node.c
index a808769..72a76d1 100644 (file)
--- a/src/plugins/dns/request_node.c
+++ b/src/plugins/dns/request_node.c
@@ -95,8 +95,8 @@ dns46_request_inline (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p2->data);
+           clib_prefetch_store (p3->data);
          }
 
          /* speculatively enqueue b0 and b1 to the current next frame */

diff --git a/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c b/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c
index 86f8e04..4545e24 100644 (file)
--- a/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c
+++ b/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c
@@ -171,8 +171,8 @@ cryptodev_frame_linked_algs_enqueue (vlib_main_t *vm,
        {
          CLIB_PREFETCH (cop[1], CLIB_CACHE_LINE_BYTES * 3, STORE);
          CLIB_PREFETCH (cop[2], CLIB_CACHE_LINE_BYTES * 3, STORE);
-         CLIB_PREFETCH (&fe[1], CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&fe[2], CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (&fe[1]);
+         clib_prefetch_load (&fe[2]);
        }
       if (last_key_index != fe->key_index)
        {
@@ -288,8 +288,8 @@ cryptodev_frame_aead_enqueue (vlib_main_t *vm,
        {
          CLIB_PREFETCH (cop[1], CLIB_CACHE_LINE_BYTES * 3, STORE);
          CLIB_PREFETCH (cop[2], CLIB_CACHE_LINE_BYTES * 3, STORE);
-         CLIB_PREFETCH (&fe[1], CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&fe[2], CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (&fe[1]);
+         clib_prefetch_load (&fe[2]);
        }
       if (last_key_index != fe->key_index)
        {

diff --git a/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c b/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c
index c045d78..41a1e0c 100644 (file)
--- a/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c
+++ b/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c
@@ -144,8 +144,8 @@ cryptodev_frame_linked_algs_enqueue (vlib_main_t *vm,
 
       if (n_elts > 2)
        {
-         CLIB_PREFETCH (&fe[1], CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&fe[2], CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (&fe[1]);
+         clib_prefetch_load (&fe[2]);
          vlib_prefetch_buffer_header (b[1], LOAD);
          vlib_prefetch_buffer_header (b[2], LOAD);
        }
@@ -275,7 +275,7 @@ cryptodev_raw_aead_enqueue (vlib_main_t *vm, vnet_crypto_async_frame_t *frame,
 
       if (n_elts > 1)
        {
-         CLIB_PREFETCH (&fe[1], CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (&fe[1]);
          vlib_prefetch_buffer_header (b[1], LOAD);
        }
 
@@ -449,8 +449,8 @@ cryptodev_raw_dequeue (vlib_main_t *vm, u32 *nb_elts_processed,
              vnet_crypto_async_frame_t *f1, *f2;
              GET_RING_OBJ (cet->cached_frame, i + 1, f1);
              GET_RING_OBJ (cet->cached_frame, i + 2, f2);
-             CLIB_PREFETCH (f1, CLIB_CACHE_LINE_BYTES, LOAD);
-             CLIB_PREFETCH (f2, CLIB_CACHE_LINE_BYTES, LOAD);
+             clib_prefetch_load (f1);
+             clib_prefetch_load (f2);
            }
 
          n_left = f->state & 0x7f;

diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index 6d130a9..7c083e1 100644 (file)
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -212,7 +212,7 @@ dpdk_prefetch_buffer (vlib_main_t * vm, struct rte_mbuf *mb)
 {
   vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb);
   CLIB_PREFETCH (mb, sizeof (struct rte_mbuf), STORE);
-  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
+  clib_prefetch_load (b);
 }
 
 static_always_inline void
@@ -357,12 +357,12 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm,
       vlib_buffer_t *b2, *b3;
       u32 or_flags;
 
-      CLIB_PREFETCH (mb[2], CLIB_CACHE_LINE_BYTES, STORE);
-      CLIB_PREFETCH (mb[3], CLIB_CACHE_LINE_BYTES, STORE);
+      clib_prefetch_store (mb[2]);
+      clib_prefetch_store (mb[3]);
       b2 = vlib_buffer_from_rte_mbuf (mb[2]);
-      CLIB_PREFETCH (b2, CLIB_CACHE_LINE_BYTES, LOAD);
+      clib_prefetch_load (b2);
       b3 = vlib_buffer_from_rte_mbuf (mb[3]);
-      CLIB_PREFETCH (b3, CLIB_CACHE_LINE_BYTES, LOAD);
+      clib_prefetch_load (b3);
 
       b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
       b[1] = vlib_buffer_from_rte_mbuf (mb[1]);

diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c
index 98cda4c..2522285 100644 (file)
--- a/src/plugins/dpdk/device/node.c
+++ b/src/plugins/dpdk/device/node.c
@@ -86,10 +86,10 @@ dpdk_process_subseq_segs (vlib_main_t * vm, vlib_buffer_t * b,
 static_always_inline void
 dpdk_prefetch_mbuf_x4 (struct rte_mbuf *mb[])
 {
-  CLIB_PREFETCH (mb[0], CLIB_CACHE_LINE_BYTES, LOAD);
-  CLIB_PREFETCH (mb[1], CLIB_CACHE_LINE_BYTES, LOAD);
-  CLIB_PREFETCH (mb[2], CLIB_CACHE_LINE_BYTES, LOAD);
-  CLIB_PREFETCH (mb[3], CLIB_CACHE_LINE_BYTES, LOAD);
+  clib_prefetch_load (mb[0]);
+  clib_prefetch_load (mb[1]);
+  clib_prefetch_load (mb[2]);
+  clib_prefetch_load (mb[3]);
 }
 
 static_always_inline void
@@ -97,13 +97,13 @@ dpdk_prefetch_buffer_x4 (struct rte_mbuf *mb[])
 {
   vlib_buffer_t *b;
   b = vlib_buffer_from_rte_mbuf (mb[0]);
-  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
+  clib_prefetch_load (b);
   b = vlib_buffer_from_rte_mbuf (mb[1]);
-  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
+  clib_prefetch_load (b);
   b = vlib_buffer_from_rte_mbuf (mb[2]);
-  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
+  clib_prefetch_load (b);
   b = vlib_buffer_from_rte_mbuf (mb[3]);
-  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
+  clib_prefetch_load (b);
 }
 
 /** \brief Main DPDK input node

diff --git a/src/plugins/flowprobe/node.c b/src/plugins/flowprobe/node.c
index ab3640c..e7a39a7 100644 (file)
--- a/src/plugins/flowprobe/node.c
+++ b/src/plugins/flowprobe/node.c
@@ -755,8 +755,8 @@ flowprobe_node_fn (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p2->data);
+           clib_prefetch_store (p3->data);
          }
 
          /* speculatively enqueue b0 and b1 to the current next frame */

diff --git a/src/plugins/memif/device.c b/src/plugins/memif/device.c
index fce4ab2..fc66420 100644 (file)
--- a/src/plugins/memif/device.c
+++ b/src/plugins/memif/device.c
@@ -143,8 +143,7 @@ retry:
       u32 saved_ptd_buffers_len = _vec_len (ptd->buffers);
       u16 saved_slot = slot;
 
-      CLIB_PREFETCH (&ring->desc[(slot + 8) & mask], CLIB_CACHE_LINE_BYTES,
-                    LOAD);
+      clib_prefetch_load (&ring->desc[(slot + 8) & mask]);
 
       d0 = &ring->desc[slot & mask];
       if (PREDICT_FALSE (last_region != d0->region))
@@ -235,10 +234,10 @@ no_free_slots:
   co = ptd->copy_ops;
   while (n_copy_op >= 8)
     {
-      CLIB_PREFETCH (co[4].data, CLIB_CACHE_LINE_BYTES, LOAD);
-      CLIB_PREFETCH (co[5].data, CLIB_CACHE_LINE_BYTES, LOAD);
-      CLIB_PREFETCH (co[6].data, CLIB_CACHE_LINE_BYTES, LOAD);
-      CLIB_PREFETCH (co[7].data, CLIB_CACHE_LINE_BYTES, LOAD);
+      clib_prefetch_load (co[4].data);
+      clib_prefetch_load (co[5].data);
+      clib_prefetch_load (co[6].data);
+      clib_prefetch_load (co[7].data);
 
       b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]);
       b1 = vlib_get_buffer (vm, ptd->buffers[co[1].buffer_vec_index]);
@@ -315,8 +314,7 @@ retry:
       memif_desc_t *d0;
       u32 bi0;
 
-      CLIB_PREFETCH (&ring->desc[(slot + 8) & mask], CLIB_CACHE_LINE_BYTES,
-                    STORE);
+      clib_prefetch_store (&ring->desc[(slot + 8) & mask]);
 
       if (PREDICT_TRUE (n_left >= 4))
        vlib_prefetch_buffer_header (vlib_get_buffer (vm, buffers[3]), LOAD);

diff --git a/src/plugins/memif/node.c b/src/plugins/memif/node.c
index a69ab7f..8bb3758 100644 (file)
--- a/src/plugins/memif/node.c
+++ b/src/plugins/memif/node.c
@@ -207,8 +207,7 @@ memif_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
       dst_off = start_offset;
 
     next_slot:
-      CLIB_PREFETCH (&ring->desc[(cur_slot + 8) & mask],
-                    CLIB_CACHE_LINE_BYTES, LOAD);
+      clib_prefetch_load (&ring->desc[(cur_slot + 8) & mask]);
       s0 = cur_slot & mask;
       d0 = &ring->desc[s0];
       n_bytes_left = d0->length;
@@ -277,10 +276,10 @@ memif_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
   co = ptd->copy_ops;
   while (n_left >= 8)
     {
-      CLIB_PREFETCH (co[4].data, CLIB_CACHE_LINE_BYTES, LOAD);
-      CLIB_PREFETCH (co[5].data, CLIB_CACHE_LINE_BYTES, LOAD);
-      CLIB_PREFETCH (co[6].data, CLIB_CACHE_LINE_BYTES, LOAD);
-      CLIB_PREFETCH (co[7].data, CLIB_CACHE_LINE_BYTES, LOAD);
+      clib_prefetch_load (co[4].data);
+      clib_prefetch_load (co[5].data);
+      clib_prefetch_load (co[6].data);
+      clib_prefetch_load (co[7].data);
 
       b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]);
       b1 = vlib_get_buffer (vm, ptd->buffers[co[1].buffer_vec_index]);
@@ -574,8 +573,7 @@ memif_device_input_zc_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
       bi0 = mq->buffers[s0];
       ptd->buffers[n_rx_packets++] = bi0;
 
-      CLIB_PREFETCH (&ring->desc[(cur_slot + 8) & mask],
-                    CLIB_CACHE_LINE_BYTES, LOAD);
+      clib_prefetch_load (&ring->desc[(cur_slot + 8) & mask]);
       d0 = &ring->desc[s0];
       hb = b0 = vlib_get_buffer (vm, bi0);
       b0->current_data = start_offset;

diff --git a/src/plugins/nat/det44/det44_in2out.c b/src/plugins/nat/det44/det44_in2out.c
index 0857ca1..5fe4a9a 100644 (file)
--- a/src/plugins/nat/det44/det44_in2out.c
+++ b/src/plugins/nat/det44/det44_in2out.c
@@ -451,8 +451,8 @@ VLIB_NODE_FN (det44_in2out_node) (vlib_main_t * vm,
          vlib_prefetch_buffer_header (p2, LOAD);
          vlib_prefetch_buffer_header (p3, LOAD);
 
-         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p2->data);
+         clib_prefetch_load (p3->data);
        }
 
       next0 = DET44_IN2OUT_NEXT_LOOKUP;

diff --git a/src/plugins/nat/det44/det44_out2in.c b/src/plugins/nat/det44/det44_out2in.c
index 48b816b..111bc61 100644 (file)
--- a/src/plugins/nat/det44/det44_out2in.c
+++ b/src/plugins/nat/det44/det44_out2in.c
@@ -415,8 +415,8 @@ VLIB_NODE_FN (det44_out2in_node) (vlib_main_t * vm,
          vlib_prefetch_buffer_header (p2, LOAD);
          vlib_prefetch_buffer_header (p3, LOAD);
 
-         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p2->data);
+         clib_prefetch_load (p3->data);
        }
 
 

diff --git a/src/plugins/nat/nat44-ed/nat44_ed_handoff.c b/src/plugins/nat/nat44-ed/nat44_ed_handoff.c
index 52d52b4..918d772 100644 (file)
--- a/src/plugins/nat/nat44-ed/nat44_ed_handoff.c
+++ b/src/plugins/nat/nat44-ed/nat44_ed_handoff.c
@@ -112,10 +112,10 @@ nat44_worker_handoff_fn_inline (vlib_main_t * vm,
          vlib_prefetch_buffer_header (b[5], LOAD);
          vlib_prefetch_buffer_header (b[6], LOAD);
          vlib_prefetch_buffer_header (b[7], LOAD);
-         CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (&b[4]->data);
+         clib_prefetch_load (&b[5]->data);
+         clib_prefetch_load (&b[6]->data);
+         clib_prefetch_load (&b[7]->data);
        }
 
       if (is_output)

diff --git a/src/plugins/nat/nat44-ed/nat44_ed_in2out.c b/src/plugins/nat/nat44-ed/nat44_ed_in2out.c
index 523d262..bfabdbd 100644 (file)
--- a/src/plugins/nat/nat44-ed/nat44_ed_in2out.c
+++ b/src/plugins/nat/nat44-ed/nat44_ed_in2out.c
@@ -1010,7 +1010,7 @@ nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t *vm,
 
          vlib_prefetch_buffer_header (p2, LOAD);
 
-         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p2->data);
        }
 
       if (is_output_feature)

diff --git a/src/plugins/nat/nat44-ed/nat44_ed_inlines.h b/src/plugins/nat/nat44-ed/nat44_ed_inlines.h
index 680bdef..0d75e73 100644 (file)
--- a/src/plugins/nat/nat44-ed/nat44_ed_inlines.h
+++ b/src/plugins/nat/nat44-ed/nat44_ed_inlines.h
@@ -638,8 +638,8 @@ nat_pre_node_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
          vlib_prefetch_buffer_header (p2, LOAD);
          vlib_prefetch_buffer_header (p3, LOAD);
 
-         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p2->data);
+         clib_prefetch_load (p3->data);
        }
 
       next0 = def_next;

diff --git a/src/plugins/nat/nat44-ed/nat44_ed_out2in.c b/src/plugins/nat/nat44-ed/nat44_ed_out2in.c
index 995260a..eaf8993 100644 (file)
--- a/src/plugins/nat/nat44-ed/nat44_ed_out2in.c
+++ b/src/plugins/nat/nat44-ed/nat44_ed_out2in.c
@@ -838,7 +838,7 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 
          vlib_prefetch_buffer_header (p2, LOAD);
 
-         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p2->data);
        }
 
       next[0] = vnet_buffer2 (b0)->nat.arc_next;

diff --git a/src/plugins/nat/nat44-ei/nat44_ei_handoff.c b/src/plugins/nat/nat44-ei/nat44_ei_handoff.c
index 567f539..c7a1317 100644 (file)
--- a/src/plugins/nat/nat44-ei/nat44_ei_handoff.c
+++ b/src/plugins/nat/nat44-ei/nat44_ei_handoff.c
@@ -108,10 +108,10 @@ nat44_ei_worker_handoff_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
          vlib_prefetch_buffer_header (b[5], LOAD);
          vlib_prefetch_buffer_header (b[6], LOAD);
          vlib_prefetch_buffer_header (b[7], LOAD);
-         CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (&b[4]->data);
+         clib_prefetch_load (&b[5]->data);
+         clib_prefetch_load (&b[6]->data);
+         clib_prefetch_load (&b[7]->data);
        }
 
       if (is_output)

diff --git a/src/plugins/nat/nat44-ei/nat44_ei_in2out.c b/src/plugins/nat/nat44-ei/nat44_ei_in2out.c
index 42923aa..7ac1a92 100644 (file)
--- a/src/plugins/nat/nat44-ei/nat44_ei_in2out.c
+++ b/src/plugins/nat/nat44-ei/nat44_ei_in2out.c
@@ -964,8 +964,8 @@ nat44_ei_in2out_node_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
          vlib_prefetch_buffer_header (p2, LOAD);
          vlib_prefetch_buffer_header (p3, LOAD);
 
-         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p2->data);
+         clib_prefetch_load (p3->data);
        }
 
       if (is_output_feature)

diff --git a/src/plugins/nat/nat44-ei/nat44_ei_out2in.c b/src/plugins/nat/nat44-ei/nat44_ei_out2in.c
index 1f04ba0..7796b11 100644 (file)
--- a/src/plugins/nat/nat44-ei/nat44_ei_out2in.c
+++ b/src/plugins/nat/nat44-ei/nat44_ei_out2in.c
@@ -758,8 +758,8 @@ VLIB_NODE_FN (nat44_ei_out2in_node)
          vlib_prefetch_buffer_header (p2, LOAD);
          vlib_prefetch_buffer_header (p3, LOAD);
 
-         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p2->data);
+         clib_prefetch_load (p3->data);
        }
 
       vnet_buffer (b0)->snat.flags = 0;

diff --git a/src/plugins/nsim/nsim_input.c b/src/plugins/nsim/nsim_input.c
index 1976d05..65a7ae1 100644 (file)
--- a/src/plugins/nsim/nsim_input.c
+++ b/src/plugins/nsim/nsim_input.c
@@ -94,7 +94,7 @@ nsim_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
     {
       /* prefetch one line / 2 entries ahead */
       if ((((uword) ep) & (CLIB_CACHE_LINE_BYTES - 1)) == 0)
-       CLIB_PREFETCH ((ep + 2), CLIB_CACHE_LINE_BYTES, LOAD);
+       clib_prefetch_load ((ep + 2));
 
       ep = wp->entries + wp->head;
       from[0] = ep->buffer_index;

diff --git a/src/plugins/rdma/input.c b/src/plugins/rdma/input.c
index df101e5..f1c508a 100644 (file)
--- a/src/plugins/rdma/input.c
+++ b/src/plugins/rdma/input.c
@@ -389,10 +389,10 @@ rdma_device_input_bufs (vlib_main_t * vm, const rdma_device_t * rd,
     {
       if (PREDICT_TRUE (n_left_from >= 8))
        {
-         CLIB_PREFETCH (&wc[4 + 0], CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&wc[4 + 1], CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&wc[4 + 2], CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (&wc[4 + 3], CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (&wc[4 + 0]);
+         clib_prefetch_load (&wc[4 + 1]);
+         clib_prefetch_load (&wc[4 + 2]);
+         clib_prefetch_load (&wc[4 + 3]);
          vlib_prefetch_buffer_header (b[4 + 0], STORE);
          vlib_prefetch_buffer_header (b[4 + 1], STORE);
          vlib_prefetch_buffer_header (b[4 + 2], STORE);

diff --git a/src/plugins/unittest/mpcap_node.c b/src/plugins/unittest/mpcap_node.c
index f3c2484..083c22e 100644 (file)
--- a/src/plugins/unittest/mpcap_node.c
+++ b/src/plugins/unittest/mpcap_node.c
@@ -116,10 +116,10 @@ mpcap_inline (vlib_main_t * vm,
          vlib_prefetch_buffer_header (b[5], STORE);
          vlib_prefetch_buffer_header (b[6], STORE);
          vlib_prefetch_buffer_header (b[7], STORE);
-         CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, STORE);
+         clib_prefetch_store (b[4]->data);
+         clib_prefetch_store (b[5]->data);
+         clib_prefetch_store (b[6]->data);
+         clib_prefetch_store (b[7]->data);
        }
 
       next[0] = 0;

diff --git a/src/vlib/buffer_node.h b/src/vlib/buffer_node.h
index 9fece67..9ca43d4 100644 (file)
--- a/src/vlib/buffer_node.h
+++ b/src/vlib/buffer_node.h
@@ -287,8 +287,8 @@ generic_buffer_node_inline (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, 64, LOAD);
-           CLIB_PREFETCH (p3->data, 64, LOAD);
+           clib_prefetch_load (p2->data);
+           clib_prefetch_load (p3->data);
          }
 
          pi0 = to_next[0] = from[0];

diff --git a/src/vlib/counter.h b/src/vlib/counter.h
index 01aefd8..9f56542 100644 (file)
--- a/src/vlib/counter.h
+++ b/src/vlib/counter.h
@@ -254,7 +254,7 @@ vlib_prefetch_combined_counter (const vlib_combined_counter_main_t * cm,
    * This CPU's index is assumed to already be in cache
    */
   cpu_counters = cm->counters[thread_index];
-  CLIB_PREFETCH (cpu_counters + index, CLIB_CACHE_LINE_BYTES, STORE);
+  clib_prefetch_store (cpu_counters + index);
 }
 
 

diff --git a/src/vnet/bonding/device.c b/src/vnet/bonding/device.c
index 8a0c109..9e949b8 100644 (file)
--- a/src/vnet/bonding/device.c
+++ b/src/vnet/bonding/device.c
@@ -416,10 +416,10 @@ bond_tx_inline (vlib_main_t * vm, bond_if_t * bif, vlib_buffer_t ** b,
          vlib_prefetch_buffer_header (pb[2], LOAD);
          vlib_prefetch_buffer_header (pb[3], LOAD);
 
-         CLIB_PREFETCH (pb[0]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (pb[1]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (pb[2]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (pb[3]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (pb[0]->data);
+         clib_prefetch_load (pb[1]->data);
+         clib_prefetch_load (pb[2]->data);
+         clib_prefetch_load (pb[3]->data);
        }
 
       if (lb_alg == BOND_LB_L2)

diff --git a/src/vnet/classify/flow_classify_node.c b/src/vnet/classify/flow_classify_node.c
index 9462d46..4989bf0 100644 (file)
--- a/src/vnet/classify/flow_classify_node.c
+++ b/src/vnet/classify/flow_classify_node.c
@@ -97,9 +97,9 @@ flow_classify_inline (vlib_main_t * vm,
        p2 = vlib_get_buffer (vm, from[2]);
 
        vlib_prefetch_buffer_header (p1, STORE);
-       CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p1->data);
        vlib_prefetch_buffer_header (p2, STORE);
-       CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p2->data);
       }
 
       bi0 = from[0];

diff --git a/src/vnet/classify/ip_classify.c b/src/vnet/classify/ip_classify.c
index 5a5cfed..a5c0445 100644 (file)
--- a/src/vnet/classify/ip_classify.c
+++ b/src/vnet/classify/ip_classify.c
@@ -102,9 +102,9 @@ ip_classify_inline (vlib_main_t * vm,
        p2 = vlib_get_buffer (vm, from[2]);
 
        vlib_prefetch_buffer_header (p1, STORE);
-       CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p1->data);
        vlib_prefetch_buffer_header (p2, STORE);
-       CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p2->data);
       }
 
       bi0 = from[0];

diff --git a/src/vnet/classify/vnet_classify.h b/src/vnet/classify/vnet_classify.h
index fbac77c..06784e0 100644 (file)
--- a/src/vnet/classify/vnet_classify.h
+++ b/src/vnet/classify/vnet_classify.h
@@ -328,7 +328,7 @@ vnet_classify_prefetch_bucket (vnet_classify_table_t * t, u64 hash)
 
   bucket_index = hash & (t->nbuckets - 1);
 
-  CLIB_PREFETCH (&t->buckets[bucket_index], CLIB_CACHE_LINE_BYTES, LOAD);
+  clib_prefetch_load (&t->buckets[bucket_index]);
 }
 
 static inline vnet_classify_entry_t *
@@ -389,7 +389,7 @@ vnet_classify_prefetch_entry (vnet_classify_table_t * t, u64 hash)
 
   e = vnet_classify_entry_at_index (t, e, value_index);
 
-  CLIB_PREFETCH (e, CLIB_CACHE_LINE_BYTES, LOAD);
+  clib_prefetch_load (e);
 }
 
 vnet_classify_entry_t *vnet_classify_find_entry (vnet_classify_table_t * t,

diff --git a/src/vnet/devices/pipe/pipe.c b/src/vnet/devices/pipe/pipe.c
index ec50f63..eb92b3c 100644 (file)
--- a/src/vnet/devices/pipe/pipe.c
+++ b/src/vnet/devices/pipe/pipe.c
@@ -324,8 +324,8 @@ pipe_rx (vlib_main_t * vm,
            p3 = vlib_get_buffer (vm, from[3]);
            vlib_prefetch_buffer_header (p2, STORE);
            vlib_prefetch_buffer_header (p3, STORE);
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
+           clib_prefetch_load (p2->data);
+           clib_prefetch_load (p3->data);
          }
 
          bi0 = from[0];

diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c
index 6abc1fc..bdb3d27 100644 (file)
--- a/src/vnet/devices/virtio/vhost_user_input.c
+++ b/src/vnet/devices/virtio/vhost_user_input.c
@@ -163,8 +163,8 @@ vhost_user_input_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy,
              (!(src3 = map_guest_mem (vui, cpy[3].src, map_hint))))
            return 1;
 
-         CLIB_PREFETCH (src2, 64, LOAD);
-         CLIB_PREFETCH (src3, 64, LOAD);
+         clib_prefetch_load (src2);
+         clib_prefetch_load (src3);
 
          clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len);
          clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len);
@@ -872,10 +872,10 @@ vhost_user_input_copy_packed (vhost_user_intf_t * vui, vhost_copy_t * cpy,
       bad = (src4 == 0) + (src5 == 0) + (src6 == 0) + (src7 == 0);
       if (PREDICT_FALSE (bad))
        goto one_by_one;
-      CLIB_PREFETCH (src4, 64, LOAD);
-      CLIB_PREFETCH (src5, 64, LOAD);
-      CLIB_PREFETCH (src6, 64, LOAD);
-      CLIB_PREFETCH (src7, 64, LOAD);
+      clib_prefetch_load (src4);
+      clib_prefetch_load (src5);
+      clib_prefetch_load (src6);
+      clib_prefetch_load (src7);
 
       while (PREDICT_TRUE (copy_len >= 8))
        {
@@ -892,10 +892,10 @@ vhost_user_input_copy_packed (vhost_user_intf_t * vui, vhost_copy_t * cpy,
          if (PREDICT_FALSE (bad))
            break;
 
-         CLIB_PREFETCH (src4, 64, LOAD);
-         CLIB_PREFETCH (src5, 64, LOAD);
-         CLIB_PREFETCH (src6, 64, LOAD);
-         CLIB_PREFETCH (src7, 64, LOAD);
+         clib_prefetch_load (src4);
+         clib_prefetch_load (src5);
+         clib_prefetch_load (src6);
+         clib_prefetch_load (src7);
 
          clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len);
          clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len);

diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c
index 59ea810..40faefa 100644 (file)
--- a/src/vnet/devices/virtio/vhost_user_output.c
+++ b/src/vnet/devices/virtio/vhost_user_output.c
@@ -194,8 +194,8 @@ vhost_user_tx_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy,
              (!(dst3 = map_guest_mem (vui, cpy[3].dst, map_hint))))
            return 1;
 
-         CLIB_PREFETCH ((void *) cpy[2].src, 64, LOAD);
-         CLIB_PREFETCH ((void *) cpy[3].src, 64, LOAD);
+         clib_prefetch_load ((void *) cpy[2].src);
+         clib_prefetch_load ((void *) cpy[3].src);
 
          clib_memcpy_fast (dst0, (void *) cpy[0].src, cpy[0].len);
          clib_memcpy_fast (dst1, (void *) cpy[1].src, cpy[1].len);
@@ -601,7 +601,7 @@ retry:
          buffer_map_addr += cpy->len;
          desc_len += cpy->len;
 
-         CLIB_PREFETCH (&rxvq->packed_desc, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (&rxvq->packed_desc);
 
          /* Check if vlib buffer has more data. If not, get more or break */
          if (PREDICT_TRUE (!bytes_left))
@@ -922,7 +922,7 @@ retry:
            buffer_map_addr += cpy->len;
            desc_len += cpy->len;
 
-           CLIB_PREFETCH (&rxvq->desc, CLIB_CACHE_LINE_BYTES, LOAD);
+           clib_prefetch_load (&rxvq->desc);
          }
 
          // Check if vlib buffer has more data. If not, get more or break.

diff --git a/src/vnet/dpo/lookup_dpo.c b/src/vnet/dpo/lookup_dpo.c
index 23a1285..3f34cef 100644 (file)
--- a/src/vnet/dpo/lookup_dpo.c
+++ b/src/vnet/dpo/lookup_dpo.c
@@ -377,8 +377,8 @@ lookup_dpo_ip4_inline (vlib_main_t * vm,
                vlib_prefetch_buffer_header (p2, LOAD);
                vlib_prefetch_buffer_header (p3, LOAD);
 
-               CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-               CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+               clib_prefetch_store (p2->data);
+               clib_prefetch_store (p3->data);
            }
 
            bi0 = from[0];
@@ -734,8 +734,8 @@ lookup_dpo_ip6_inline (vlib_main_t * vm,
                vlib_prefetch_buffer_header (p2, LOAD);
                vlib_prefetch_buffer_header (p3, LOAD);
 
-               CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-               CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+               clib_prefetch_store (p2->data);
+               clib_prefetch_store (p3->data);
            }
 
            bi0 = from[0];

diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c
index f470c1c..cc0a0c3 100644 (file)
--- a/src/vnet/ethernet/node.c
+++ b/src/vnet/ethernet/node.c
@@ -1505,7 +1505,7 @@ ethernet_input_inline (vlib_main_t * vm,
          if (n_left_from > 1)
            {
              vlib_prefetch_buffer_header (b[1], STORE);
-             CLIB_PREFETCH (b[1]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+             clib_prefetch_load (b[1]->data);
            }
 
          bi0 = from[0];

diff --git a/src/vnet/ip/icmp4.c b/src/vnet/ip/icmp4.c
index 05262e6..0363092 100644 (file)
--- a/src/vnet/ip/icmp4.c
+++ b/src/vnet/ip/icmp4.c
@@ -176,7 +176,7 @@ ip4_icmp_input (vlib_main_t * vm,
              vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
              p0 = vlib_get_buffer (vm, from[1]);
              ip0 = vlib_buffer_get_current (p0);
-             CLIB_PREFETCH (ip0, CLIB_CACHE_LINE_BYTES, LOAD);
+             clib_prefetch_load (ip0);
            }
 
          bi0 = to_next[0] = from[0];

diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 36f05a2..750b75f 100644 (file)
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -1762,8 +1762,8 @@ ip4_local_inline (vlib_main_t * vm,
        vlib_prefetch_buffer_header (b[4], LOAD);
        vlib_prefetch_buffer_header (b[5], LOAD);
 
-       CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-       CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+       clib_prefetch_load (b[4]->data);
+       clib_prefetch_load (b[5]->data);
       }
 
       error[0] = error[1] = IP4_ERROR_UNKNOWN_PROTOCOL;
@@ -2148,12 +2148,12 @@ ip4_rewrite_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
       vnet_buffer (b[1])->ip.save_rewrite_length = rw_len1;
 
       p = vlib_buffer_get_current (b[2]);
-      CLIB_PREFETCH (p - CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES, STORE);
-      CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
+      clib_prefetch_store (p - CLIB_CACHE_LINE_BYTES);
+      clib_prefetch_load (p);
 
       p = vlib_buffer_get_current (b[3]);
-      CLIB_PREFETCH (p - CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES, STORE);
-      CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
+      clib_prefetch_store (p - CLIB_CACHE_LINE_BYTES);
+      clib_prefetch_load (p);
 
       /* Check MTU of outgoing interface. */
       u16 ip0_len = clib_net_to_host_u16 (ip0->length);

diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index fd742d6..ba616eb 100644 (file)
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -1780,8 +1780,8 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->pre_data, 32, STORE);
-           CLIB_PREFETCH (p3->pre_data, 32, STORE);
+           clib_prefetch_store (p2->pre_data);
+           clib_prefetch_store (p3->pre_data);
 
            CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
            CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);

diff --git a/src/vnet/ip/ip6_hop_by_hop.c b/src/vnet/ip/ip6_hop_by_hop.c
index 9923b41..e66084c 100644 (file)
--- a/src/vnet/ip/ip6_hop_by_hop.c
+++ b/src/vnet/ip/ip6_hop_by_hop.c
@@ -620,8 +620,8 @@ VLIB_NODE_FN (ip6_pop_hop_by_hop_node) (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p2->data);
+           clib_prefetch_store (p3->data);
          }
 
          /* speculatively enqueue b0 and b1 to the current next frame */
@@ -879,10 +879,10 @@ ip6_local_hop_by_hop_inline (vlib_main_t * vm,
          vlib_prefetch_buffer_header (b[5], STORE);
          vlib_prefetch_buffer_header (b[6], STORE);
          vlib_prefetch_buffer_header (b[7], STORE);
-         CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, STORE);
-         CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, STORE);
+         clib_prefetch_store (b[4]->data);
+         clib_prefetch_store (b[5]->data);
+         clib_prefetch_store (b[6]->data);
+         clib_prefetch_store (b[7]->data);
        }
 
       /*

diff --git a/src/vnet/ip/ip_in_out_acl.c b/src/vnet/ip/ip_in_out_acl.c
index 2f73e13..a5e652e 100644 (file)
--- a/src/vnet/ip/ip_in_out_acl.c
+++ b/src/vnet/ip/ip_in_out_acl.c
@@ -220,8 +220,8 @@ ip_in_out_acl_inline (vlib_main_t * vm,
          vlib_prefetch_buffer_header (b[4], LOAD);
          vlib_prefetch_buffer_header (b[5], LOAD);
 
-         CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (b[4]->data);
+         clib_prefetch_load (b[5]->data);
        }
 
       /* calculate hashes for b[2] & b[3] */

diff --git a/src/vnet/ip/reass/ip4_sv_reass.c b/src/vnet/ip/reass/ip4_sv_reass.c
index 43390ea..9b3f1b9 100644 (file)
--- a/src/vnet/ip/reass/ip4_sv_reass.c
+++ b/src/vnet/ip/reass/ip4_sv_reass.c
@@ -468,8 +468,8 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
          vlib_prefetch_buffer_header (p2, LOAD);
          vlib_prefetch_buffer_header (p3, LOAD);
 
-         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p2->data);
+         clib_prefetch_load (p3->data);
        }
 
       ip4_header_t *ip0 =

diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c
index b700f2c..e30fc9e 100644 (file)
--- a/src/vnet/ipsec/esp_decrypt.c
+++ b/src/vnet/ipsec/esp_decrypt.c
@@ -1068,9 +1068,9 @@ esp_decrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
          u8 *p;
          vlib_prefetch_buffer_header (b[2], LOAD);
          p = vlib_buffer_get_current (b[1]);
-         CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p);
          p -= CLIB_CACHE_LINE_BYTES;
-         CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p);
        }
 
       u32 n_bufs = vlib_buffer_chain_linearize (vm, b[0]);
@@ -1095,7 +1095,7 @@ esp_decrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
          sa0 = ipsec_sa_get (current_sa_index);
 
          /* fetch the second cacheline ASAP */
-         CLIB_PREFETCH (sa0->cacheline1, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (sa0->cacheline1);
          cpd.icv_sz = sa0->integ_icv_size;
          cpd.iv_sz = sa0->crypto_iv_size;
          cpd.flags = sa0->flags;

diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c
index da9c56a..d102bd6 100644 (file)
--- a/src/vnet/ipsec/esp_encrypt.c
+++ b/src/vnet/ipsec/esp_encrypt.c
@@ -626,9 +626,9 @@ esp_encrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
          u8 *p;
          vlib_prefetch_buffer_header (b[2], LOAD);
          p = vlib_buffer_get_current (b[1]);
-         CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p);
          p -= CLIB_CACHE_LINE_BYTES;
-         CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (p);
          /* speculate that the trailer goes in the first buffer */
          CLIB_PREFETCH (vlib_buffer_get_tail (b[1]),
                         CLIB_CACHE_LINE_BYTES, LOAD);
@@ -656,7 +656,7 @@ esp_encrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
          sa0 = ipsec_sa_get (sa_index0);
 
          /* fetch the second cacheline ASAP */
-         CLIB_PREFETCH (sa0->cacheline1, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (sa0->cacheline1);
 
          current_sa_index = sa_index0;
          spi = clib_net_to_host_u32 (sa0->spi);

diff --git a/src/vnet/l2/l2_efp_filter.c b/src/vnet/l2/l2_efp_filter.c
index 3d152ad..ad325b8 100644 (file)
--- a/src/vnet/l2/l2_efp_filter.c
+++ b/src/vnet/l2/l2_efp_filter.c
@@ -227,8 +227,8 @@ VLIB_NODE_FN (l2_efp_filter_node) (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p4, LOAD);
            vlib_prefetch_buffer_header (p5, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
 
            /*
             * Prefetch the input config for the N+1 loop iteration
@@ -238,8 +238,10 @@ VLIB_NODE_FN (l2_efp_filter_node) (vlib_main_t * vm,
            sw_if_index3 = vnet_buffer (p3)->sw_if_index[VLIB_TX];
            /*
             * $$$ TODO
-            * CLIB_PREFETCH (vec_elt_at_index(l2output_main.configs, sw_if_index2), CLIB_CACHE_LINE_BYTES, LOAD);
-            * CLIB_PREFETCH (vec_elt_at_index(l2output_main.configs, sw_if_index3), CLIB_CACHE_LINE_BYTES, LOAD);
+            * clib_prefetch_load (vec_elt_at_index(l2output_main.configs,
+            *                     sw_if_index2));
+            * clib_prefetch_load (vec_elt_at_index(l2output_main.configs,
+            *                     sw_if_index3));
             */
          }
 

diff --git a/src/vnet/l2/l2_fib.c b/src/vnet/l2/l2_fib.c
index 478f676..d1ee822 100644 (file)
--- a/src/vnet/l2/l2_fib.c
+++ b/src/vnet/l2/l2_fib.c
@@ -1131,13 +1131,13 @@ l2fib_scan (vlib_main_t * vm, f64 start_time, u8 event_only)
        {
          BVT (clib_bihash_bucket) * b =
            BV (clib_bihash_get_bucket) (h, i + 3);
-         CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (b);
          b = BV (clib_bihash_get_bucket) (h, i + 1);
          if (!BV (clib_bihash_bucket_is_empty) (b))
            {
              BVT (clib_bihash_value) * v =
                BV (clib_bihash_get_value) (h, b->offset);
-             CLIB_PREFETCH (v, CLIB_CACHE_LINE_BYTES, LOAD);
+             clib_prefetch_load (v);
            }
        }
 

diff --git a/src/vnet/l2/l2_fwd.c b/src/vnet/l2/l2_fwd.c
index e7accc2..3414f6c 100644 (file)
--- a/src/vnet/l2/l2_fwd.c
+++ b/src/vnet/l2/l2_fwd.c
@@ -269,10 +269,10 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
        vlib_prefetch_buffer_header (b[6], LOAD);
        vlib_prefetch_buffer_header (b[7], LOAD);
 
-       CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-       CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-       CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-       CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+       clib_prefetch_load (b[4]->data);
+       clib_prefetch_load (b[5]->data);
+       clib_prefetch_load (b[6]->data);
+       clib_prefetch_load (b[7]->data);
       }
 
       /* RX interface handles */

diff --git a/src/vnet/l2/l2_in_out_acl.c b/src/vnet/l2/l2_in_out_acl.c
index df23367..f8293c1 100644 (file)
--- a/src/vnet/l2/l2_in_out_acl.c
+++ b/src/vnet/l2/l2_in_out_acl.c
@@ -172,9 +172,9 @@ l2_in_out_acl_node_fn (vlib_main_t * vm,
        p2 = vlib_get_buffer (vm, from[2]);
 
        vlib_prefetch_buffer_header (p1, STORE);
-       CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p1->data);
        vlib_prefetch_buffer_header (p2, STORE);
-       CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p2->data);
       }
 
       bi0 = from[0];

diff --git a/src/vnet/l2/l2_in_out_feat_arc.c b/src/vnet/l2/l2_in_out_feat_arc.c
index 37efa07..b3b4a8c 100644 (file)
--- a/src/vnet/l2/l2_in_out_feat_arc.c
+++ b/src/vnet/l2/l2_in_out_feat_arc.c
@@ -131,7 +131,7 @@ buffer_prefetch_xN (int vector_sz, vlib_buffer_t ** b)
 {
   int ii;
   for (ii = 0; ii < vector_sz; ii++)
-    CLIB_PREFETCH (b[ii], CLIB_CACHE_LINE_BYTES, STORE);
+    clib_prefetch_store (b[ii]);
 }
 
 static_always_inline void

diff --git a/src/vnet/l2/l2_input_classify.c b/src/vnet/l2/l2_input_classify.c
index ffd1078..53d4639 100644 (file)
--- a/src/vnet/l2/l2_input_classify.c
+++ b/src/vnet/l2/l2_input_classify.c
@@ -190,9 +190,9 @@ VLIB_NODE_FN (l2_input_classify_node) (vlib_main_t * vm,
        p3 = vlib_get_buffer (vm, from[3]);
 
        vlib_prefetch_buffer_header (p2, STORE);
-       CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p2->data);
        vlib_prefetch_buffer_header (p3, STORE);
-       CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p3->data);
       }
 
       bi0 = from[0];

diff --git a/src/vnet/l2/l2_input_node.c b/src/vnet/l2/l2_input_node.c
index 59a437e..3638a8a 100644 (file)
--- a/src/vnet/l2/l2_input_node.c
+++ b/src/vnet/l2/l2_input_node.c
@@ -258,10 +258,10 @@ l2input_node_inline (vlib_main_t * vm,
            vlib_prefetch_buffer_header (b[6], LOAD);
            vlib_prefetch_buffer_header (b[7], LOAD);
 
-           CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (b[4]->data);
+           clib_prefetch_store (b[5]->data);
+           clib_prefetch_store (b[6]->data);
+           clib_prefetch_store (b[7]->data);
          }
 
          classify_and_dispatch (msm, b[0], &next[0]);

diff --git a/src/vnet/l2/l2_input_vtr.c b/src/vnet/l2/l2_input_vtr.c
index be3e6ba..3c1235b 100644 (file)
--- a/src/vnet/l2/l2_input_vtr.c
+++ b/src/vnet/l2/l2_input_vtr.c
@@ -126,8 +126,8 @@ VLIB_NODE_FN (l2_invtr_node) (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p4, LOAD);
            vlib_prefetch_buffer_header (p5, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
 
            /*
             * Prefetch the input config for the N+1 loop iteration

diff --git a/src/vnet/l2/l2_learn.c b/src/vnet/l2/l2_learn.c
index 3b9b1db..6d90cee 100644 (file)
--- a/src/vnet/l2/l2_learn.c
+++ b/src/vnet/l2/l2_learn.c
@@ -292,10 +292,10 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
        vlib_prefetch_buffer_header (b[6], LOAD);
        vlib_prefetch_buffer_header (b[7], LOAD);
 
-       CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-       CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-       CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-       CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+       clib_prefetch_load (b[4]->data);
+       clib_prefetch_load (b[5]->data);
+       clib_prefetch_load (b[6]->data);
+       clib_prefetch_load (b[7]->data);
       }
 
       /* RX interface handles */

diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c
index 2c472b5..ba40de3 100644 (file)
--- a/src/vnet/l2/l2_output.c
+++ b/src/vnet/l2/l2_output.c
@@ -147,10 +147,10 @@ l2output_process_batch_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       /* prefetch eth headers only if we need to touch them */
       if (l2_vtr || l2_pbb || shg_set)
        {
-         CLIB_PREFETCH (b[4]->data + cdo[4], CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (b[5]->data + cdo[5], CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (b[6]->data + cdo[6], CLIB_CACHE_LINE_BYTES, LOAD);
-         CLIB_PREFETCH (b[7]->data + cdo[7], CLIB_CACHE_LINE_BYTES, LOAD);
+         clib_prefetch_load (b[4]->data + cdo[4]);
+         clib_prefetch_load (b[5]->data + cdo[5]);
+         clib_prefetch_load (b[6]->data + cdo[6]);
+         clib_prefetch_load (b[7]->data + cdo[7]);
        }
 
       if (update_feature_bitmap)

diff --git a/src/vnet/l2/l2_output_classify.c b/src/vnet/l2/l2_output_classify.c
index 1e55afc..96d0b14 100644 (file)
--- a/src/vnet/l2/l2_output_classify.c
+++ b/src/vnet/l2/l2_output_classify.c
@@ -183,9 +183,9 @@ VLIB_NODE_FN (l2_output_classify_node) (vlib_main_t * vm,
        p3 = vlib_get_buffer (vm, from[3]);
 
        vlib_prefetch_buffer_header (p2, STORE);
-       CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p2->data);
        vlib_prefetch_buffer_header (p3, STORE);
-       CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p3->data);
       }
 
       bi0 = from[0];

diff --git a/src/vnet/l2/l2_xcrw.c b/src/vnet/l2/l2_xcrw.c
index 942cd11..c2c325a 100644 (file)
--- a/src/vnet/l2/l2_xcrw.c
+++ b/src/vnet/l2/l2_xcrw.c
@@ -107,8 +107,8 @@ VLIB_NODE_FN (l2_xcrw_node) (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p2->data);
+           clib_prefetch_store (p3->data);
          }
 
          /* speculatively enqueue b0 and b1 to the current next frame */

diff --git a/src/vnet/lawful-intercept/node.c b/src/vnet/lawful-intercept/node.c
index 03b507a..c5328e6 100644 (file)
--- a/src/vnet/lawful-intercept/node.c
+++ b/src/vnet/lawful-intercept/node.c
@@ -119,8 +119,8 @@ VLIB_NODE_FN (li_hit_node) (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p2->data);
+           clib_prefetch_store (p3->data);
          }
 
          /* speculatively enqueue b0 and b1 to the current next frame */

diff --git a/src/vnet/pipeline.h b/src/vnet/pipeline.h
index 2ac1a9a..afb3c38 100644 (file)
--- a/src/vnet/pipeline.h
+++ b/src/vnet/pipeline.h
@@ -75,7 +75,7 @@ generic_stage0 (vlib_main_t * vm,
                vlib_node_runtime_t * node, vlib_buffer_t * b AUX_DATA_ARG)
 {
   vlib_prefetch_buffer_header (b, STORE);
-  CLIB_PREFETCH (b->data, CLIB_CACHE_LINE_BYTES, STORE);
+  clib_prefetch_store (b->data);
 }
 
 #if NSTAGES == 2

diff --git a/src/vnet/policer/node_funcs.c b/src/vnet/policer/node_funcs.c
index c7b6e8c..21b9393 100644 (file)
--- a/src/vnet/policer/node_funcs.c
+++ b/src/vnet/policer/node_funcs.c
@@ -387,9 +387,9 @@ policer_classify_inline (vlib_main_t * vm,
        p2 = vlib_get_buffer (vm, from[2]);
 
        vlib_prefetch_buffer_header (p1, STORE);
-       CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p1->data);
        vlib_prefetch_buffer_header (p2, STORE);
-       CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+       clib_prefetch_store (p2->data);
       }
 
       bi0 = from[0];

diff --git a/src/vnet/srv6/sr_localsid.c b/src/vnet/srv6/sr_localsid.c
index 9361907..a055c92 100644 (file)
--- a/src/vnet/srv6/sr_localsid.c
+++ b/src/vnet/srv6/sr_localsid.c
@@ -1195,10 +1195,10 @@ sr_localsid_d_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p6, LOAD);
            vlib_prefetch_buffer_header (p7, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
+           clib_prefetch_store (p6->data);
+           clib_prefetch_store (p7->data);
          }
 
          to_next[0] = bi0 = from[0];
@@ -1502,10 +1502,10 @@ sr_localsid_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p6, LOAD);
            vlib_prefetch_buffer_header (p7, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
+           clib_prefetch_store (p6->data);
+           clib_prefetch_store (p7->data);
          }
 
          to_next[0] = bi0 = from[0];
@@ -1812,10 +1812,10 @@ sr_localsid_un_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p6, LOAD);
            vlib_prefetch_buffer_header (p7, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
+           clib_prefetch_store (p6->data);
+           clib_prefetch_store (p7->data);
          }
 
          to_next[0] = bi0 = from[0];
@@ -2117,10 +2117,10 @@ sr_localsid_un_perf_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p6, LOAD);
            vlib_prefetch_buffer_header (p7, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
+           clib_prefetch_store (p6->data);
+           clib_prefetch_store (p7->data);
          }
 
          to_next[0] = bi0 = from[0];

diff --git a/src/vnet/srv6/sr_policy_rewrite.c b/src/vnet/srv6/sr_policy_rewrite.c
index 98fae3b..79de779 100644 (file)
--- a/src/vnet/srv6/sr_policy_rewrite.c
+++ b/src/vnet/srv6/sr_policy_rewrite.c
@@ -1307,10 +1307,10 @@ sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p6, LOAD);
            vlib_prefetch_buffer_header (p7, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
+           clib_prefetch_store (p6->data);
+           clib_prefetch_store (p7->data);
          }
 
          to_next[0] = bi0 = from[0];
@@ -1600,10 +1600,10 @@ sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p6, LOAD);
            vlib_prefetch_buffer_header (p7, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
+           clib_prefetch_store (p6->data);
+           clib_prefetch_store (p7->data);
          }
 
          to_next[0] = bi0 = from[0];
@@ -1904,10 +1904,10 @@ sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p6, LOAD);
            vlib_prefetch_buffer_header (p7, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
+           clib_prefetch_store (p6->data);
+           clib_prefetch_store (p7->data);
          }
 
          to_next[0] = bi0 = from[0];
@@ -2300,10 +2300,10 @@ sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p6, LOAD);
            vlib_prefetch_buffer_header (p7, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
+           clib_prefetch_store (p6->data);
+           clib_prefetch_store (p7->data);
          }
 
          to_next[0] = bi0 = from[0];
@@ -2722,10 +2722,10 @@ sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p6, LOAD);
            vlib_prefetch_buffer_header (p7, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
+           clib_prefetch_store (p6->data);
+           clib_prefetch_store (p7->data);
          }
 
          to_next[0] = bi0 = from[0];
@@ -3165,10 +3165,10 @@ sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node,
            vlib_prefetch_buffer_header (p6, LOAD);
            vlib_prefetch_buffer_header (p7, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
+           clib_prefetch_store (p6->data);
+           clib_prefetch_store (p7->data);
          }
 
          to_next[0] = bi0 = from[0];

diff --git a/src/vnet/tcp/tcp_syn_filter4.c b/src/vnet/tcp/tcp_syn_filter4.c
index b107899..ef7a328 100644 (file)
--- a/src/vnet/tcp/tcp_syn_filter4.c
+++ b/src/vnet/tcp/tcp_syn_filter4.c
@@ -135,10 +135,10 @@ VLIB_NODE_FN (syn_filter4_node) (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p6, LOAD);
            vlib_prefetch_buffer_header (p7, LOAD);
 
-           CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+           clib_prefetch_store (p4->data);
+           clib_prefetch_store (p5->data);
+           clib_prefetch_store (p6->data);
+           clib_prefetch_store (p7->data);
          }
 
          /* speculatively enqueue b0 and b1 to the current next frame */

diff --git a/src/vnet/vxlan/decap.c b/src/vnet/vxlan/decap.c
index 2ba24d8..729293f 100644 (file)
--- a/src/vnet/vxlan/decap.c
+++ b/src/vnet/vxlan/decap.c
@@ -249,8 +249,8 @@ vxlan_input (vlib_main_t * vm,
        vxlan6_find_tunnel (vxm, &last6, fi1, ip6_1, vxlan1, &stats_if1);
 
       /* Prefetch next iteration. */
-      CLIB_PREFETCH (b[2]->data, CLIB_CACHE_LINE_BYTES, LOAD);
-      CLIB_PREFETCH (b[3]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+      clib_prefetch_load (b[2]->data);
+      clib_prefetch_load (b[3]->data);
 
       u32 len0 = vlib_buffer_length_in_chain (vm, b[0]);
       u32 len1 = vlib_buffer_length_in_chain (vm, b[1]);

diff --git a/src/vppinfra/callback_data.h b/src/vppinfra/callback_data.h
index 9a1ad0a..23a7f90 100644 (file)
--- a/src/vppinfra/callback_data.h
+++ b/src/vppinfra/callback_data.h
@@ -273,16 +273,18 @@ do {                                                       \
  * @param vec_ the callback vector
  * @param varargs additional callback parameters
  */
-#define clib_callback_data_call_vec(vec_, ...)                     \
-do {                                                               \
-  u32 sz_ = vec_len (vec_);                                        \
-  u32 i_;                                                          \
-  for (i_ = 0; i_ < sz_; i_++)                                     \
-    {                                                              \
-      CLIB_PREFETCH (&vec_[i_+1], CLIB_CACHE_LINE_BYTES, STORE);   \
-      (vec_[i_].fp) (&vec_[i_], __VA_ARGS__);                      \
-    }                                                              \
-} while (0)
+#define clib_callback_data_call_vec(vec_, ...)                                \
+  do                                                                          \
+    {                                                                         \
+      u32 sz_ = vec_len (vec_);                                               \
+      u32 i_;                                                                 \
+      for (i_ = 0; i_ < sz_; i_++)                                            \
+       {                                                                     \
+         clib_prefetch_store (&vec_[i_ + 1]);                                \
+         (vec_[i_].fp) (&vec_[i_], __VA_ARGS__);                             \
+       }                                                                     \
+    }                                                                         \
+  while (0)
 
 /** @brief Call the specified callback set
  * @param set_ the callback set
@@ -297,12 +299,13 @@ do {                                                                 \
 /** @brief prefetch the callback set
  * @param set_ The callback set
  */
-#define clib_callback_data_prefetch(set_)                        \
-do {                                                             \
-  if (PREDICT_FALSE ((set_)->curr))                              \
-    CLIB_PREFETCH ((set_)->curr, CLIB_CACHE_LINE_BYTES, STORE);  \
-} while (0)
-
+#define clib_callback_data_prefetch(set_)                                     \
+  do                                                                          \
+    {                                                                         \
+      if (PREDICT_FALSE ((set_)->curr))                                       \
+       clib_prefetch_store ((set_)->curr);                                   \
+    }                                                                         \
+  while (0)
 
 #endif /* included_callback_data_h */