pg: misc improvements and fixes 46/41246/5
authorBenoît Ganne <[email protected]>
Fri, 5 Jul 2024 12:09:35 +0000 (14:09 +0200)
committerDave Wallace <[email protected]>
Thu, 12 Dec 2024 14:45:14 +0000 (14:45 +0000)
1) pg can typically injects packets in ethernet-input, ip4-input or
ip6-input. Make sure offload offsets are correctly set for ip4-input and
ip6-input.
2) add hw-addr support for ethernet mode (only available through cli)
3) refactor pg creation code to improve the readability by using
data structure pg_interface_args_t
4) fix the pg input and output traces to use headers according to
pg interface mode
5) introduce pg interface flags i.e. checksum, gso, gro

Type: improvement

Change-Id: Iffed502e9c6357d7ef8e8a72217867e8297236aa
Signed-off-by: Benoît Ganne <[email protected]>
Signed-off-by: Mohsin Kazmi <[email protected]>
src/vnet/pg/cli.c
src/vnet/pg/input.c
src/vnet/pg/output.c
src/vnet/pg/pg.h
src/vnet/pg/pg_api.c
src/vnet/pg/stream.c
test/test_pg_stream.py

index 6cd9cbd..1478243 100644 (file)
@@ -40,6 +40,7 @@
 #include <sys/stat.h>
 
 #include <vnet/vnet.h>
+#include <vnet/ethernet/ethernet.h>
 #include <vnet/pg/pg.h>
 
 #include <strings.h>
@@ -663,23 +664,29 @@ create_pg_if_cmd_fn (vlib_main_t * vm,
 {
   pg_main_t *pg = &pg_main;
   unformat_input_t _line_input, *line_input = &_line_input;
-  u32 if_id = ~0, gso_enabled = 0, gso_size = 0, coalesce_enabled = 0;
+  pg_interface_args_t args = { 0 };
   clib_error_t *error = NULL;
-  pg_interface_mode_t mode = PG_MODE_ETHERNET;
+
+  args.if_id = ~0;
+  args.flags = 0;
+  args.rv = -1;
+  args.hw_addr_set = 0;
+  args.gso_size = 0;
+  args.mode = PG_MODE_ETHERNET;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
 
   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
     {
-      if (unformat (line_input, "pg%u", &if_id))
+      if (unformat (line_input, "pg%u", &args.if_id))
        ;
       else if (unformat (line_input, "coalesce-enabled"))
-       coalesce_enabled = 1;
+       args.flags |= PG_INTERFACE_FLAG_GRO_COALESCE;
       else if (unformat (line_input, "gso-enabled"))
        {
-         gso_enabled = 1;
-         if (unformat (line_input, "gso-size %u", &gso_size))
+         args.flags |= PG_INTERFACE_FLAG_GSO;
+         if (unformat (line_input, "gso-size %u", &args.gso_size))
            ;
          else
            {
@@ -687,10 +694,13 @@ create_pg_if_cmd_fn (vlib_main_t * vm,
              goto done;
            }
        }
+      else if (unformat (line_input, "hw-addr %U", unformat_ethernet_address,
+                        args.hw_addr.bytes))
+       args.hw_addr_set = 1;
       else if (unformat (line_input, "mode ip4"))
-       mode = PG_MODE_IP4;
+       args.mode = PG_MODE_IP4;
       else if (unformat (line_input, "mode ip6"))
-       mode = PG_MODE_IP6;
+       args.mode = PG_MODE_IP6;
       else
        {
          error = clib_error_create ("unknown input `%U'",
@@ -699,8 +709,7 @@ create_pg_if_cmd_fn (vlib_main_t * vm,
        }
     }
 
-  pg_interface_add_or_get (pg, if_id, gso_enabled, gso_size, coalesce_enabled,
-                          mode);
+  pg_interface_add_or_get (pg, &args);
 
 done:
   unformat_free (line_input);
@@ -710,9 +719,10 @@ done:
 
 VLIB_CLI_COMMAND (create_pg_if_cmd, static) = {
   .path = "create packet-generator interface",
-  .short_help = "create packet-generator interface <interface name>"
-               " [gso-enabled gso-size <size> [coalesce-enabled]]"
-               " [mode <ethernet | ip4 | ip6>]",
+  .short_help =
+    "create packet-generator interface <interface name>"
+    " [hw-addr <addr>] [gso-enabled gso-size <size> [coalesce-enabled]]"
+    " [mode <ethernet | ip4 | ip6>]",
   .function = create_pg_if_cmd_fn,
 };
 
index 321472c..4f89c73 100644 (file)
@@ -1407,6 +1407,7 @@ typedef struct
 
   u32 packet_length;
   u32 sw_if_index;
+  u32 next_index;
 
   /* Use pre data for packet data. */
   vlib_buffer_t buffer;
@@ -1422,6 +1423,7 @@ format_pg_input_trace (u8 * s, va_list * va)
   pg_stream_t *stream;
   vlib_node_t *n;
   u32 indent = format_get_indent (s);
+  const char *next_nodes[] = VNET_DEVICE_INPUT_NEXT_NODES;
 
   stream = 0;
   if (!pool_is_free_index (pg->streams, t->stream_index))
@@ -1434,6 +1436,10 @@ format_pg_input_trace (u8 * s, va_list * va)
 
   s = format (s, ", %d bytes", t->packet_length);
   s = format (s, ", sw_if_index %d", t->sw_if_index);
+  if (t->next_index < VNET_DEVICE_INPUT_N_NEXT_NODES)
+    s = format (s, ", next_node %s", next_nodes[t->next_index]);
+  else
+    s = format (s, ", next_node %d", t->next_index);
 
   s = format (s, "\n%U%U", format_white_space, indent,
              format_vnet_buffer_no_chain, &t->buffer);
@@ -1497,6 +1503,9 @@ pg_input_trace (pg_main_t * pg,
       t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
       t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 
+      t0->next_index = next_index;
+      t1->next_index = next_index;
+
       clib_memcpy_fast (&t0->buffer, b0,
                        sizeof (b0[0]) - sizeof (b0->pre_data));
       clib_memcpy_fast (&t1->buffer, b1,
@@ -1527,6 +1536,7 @@ pg_input_trace (pg_main_t * pg,
       t0->stream_index = stream_index;
       t0->packet_length = vlib_buffer_length_in_chain (vm, b0);
       t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
+      t0->next_index = next_index;
       clib_memcpy_fast (&t0->buffer, b0,
                        sizeof (b0[0]) - sizeof (b0->pre_data));
       clib_memcpy_fast (t0->buffer.pre_data, b0->data,
@@ -1537,31 +1547,49 @@ pg_input_trace (pg_main_t * pg,
 }
 
 static_always_inline void
-fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers,
-                          u32 buffer_oflags, int gso_enabled, u32 gso_size)
+fill_buffer_offload_flags (vlib_main_t *vm, u32 next_index, u32 *buffers,
+                          u32 n_buffers, u32 buffer_oflags, int gso_enabled,
+                          u32 gso_size)
 {
   for (int i = 0; i < n_buffers; i++)
     {
       vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[i]);
       u8 l4_proto = 0;
       vnet_buffer_oflags_t oflags = 0;
+      u16 ethertype;
+      u16 l2hdr_sz;
 
-      ethernet_header_t *eh =
-       (ethernet_header_t *) vlib_buffer_get_current (b0);
-      u16 ethertype = clib_net_to_host_u16 (eh->type);
-      u16 l2hdr_sz = sizeof (ethernet_header_t);
-
-      if (ethernet_frame_is_tagged (ethertype))
+      /* pg can typically injects to ethernet-input or ip4/6-input */
+      if (VNET_DEVICE_INPUT_NEXT_IP4_INPUT == next_index)
+       {
+         ethertype = ETHERNET_TYPE_IP4;
+         l2hdr_sz = 0;
+       }
+      else if (VNET_DEVICE_INPUT_NEXT_IP6_INPUT == next_index)
        {
-         ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1);
+         ethertype = ETHERNET_TYPE_IP6;
+         l2hdr_sz = 0;
+       }
+      else
+       {
+         /* default to ethernet */
+         ethernet_header_t *eh = (ethernet_header_t *) b0->data;
+         ethertype = clib_net_to_host_u16 (eh->type);
+         l2hdr_sz = sizeof (ethernet_header_t);
 
-         ethertype = clib_net_to_host_u16 (vlan->type);
-         l2hdr_sz += sizeof (*vlan);
-         if (ethertype == ETHERNET_TYPE_VLAN)
+         if (ethernet_frame_is_tagged (ethertype))
            {
-             vlan++;
+             ethernet_vlan_header_t *vlan =
+               (ethernet_vlan_header_t *) (eh + 1);
+
              ethertype = clib_net_to_host_u16 (vlan->type);
              l2hdr_sz += sizeof (*vlan);
+             if (ethertype == ETHERNET_TYPE_VLAN)
+               {
+                 vlan++;
+                 ethertype = clib_net_to_host_u16 (vlan->type);
+                 l2hdr_sz += sizeof (*vlan);
+               }
            }
        }
 
@@ -1570,8 +1598,7 @@ fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers,
 
       if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4))
        {
-         ip4_header_t *ip4 =
-           (ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
+         ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l2hdr_sz);
          vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
          l4_proto = ip4->protocol;
          b0->flags |=
@@ -1583,8 +1610,7 @@ fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers,
        }
       else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
        {
-         ip6_header_t *ip6 =
-           (ip6_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
+         ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l2hdr_sz);
          vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + sizeof (ip6_header_t);
          /* FIXME IPv6 EH traversal */
          l4_proto = ip6->protocol;
@@ -1604,8 +1630,7 @@ fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers,
            {
              b0->flags |= VNET_BUFFER_F_GSO;
              tcp_header_t *tcp =
-               (tcp_header_t *) (vlib_buffer_get_current (b0) +
-                                 vnet_buffer (b0)->l4_hdr_offset);
+               (tcp_header_t *) (b0->data + vnet_buffer (b0)->l4_hdr_offset);
              vnet_buffer2 (b0)->gso_l4_hdr_sz = tcp_header_bytes (tcp);
              vnet_buffer2 (b0)->gso_size = gso_size;
            }
@@ -1723,7 +1748,11 @@ pg_generate_packets (vlib_node_runtime_t * node,
 
       if (pi->gso_enabled || (s->buffer_flags & VNET_BUFFER_F_OFFLOAD))
        {
-         fill_buffer_offload_flags (vm, to_next, n_this_frame,
+         /* we use s->next_index and not next_index on purpose here: we want
+          * the original node set by the user (typically ethernet-input,
+          * ip4-input or ip6-input) whereas next_index can be overwritten by
+          * device-input features */
+         fill_buffer_offload_flags (vm, s->next_index, to_next, n_this_frame,
                                     s->buffer_oflags, pi->gso_enabled,
                                     pi->gso_size);
        }
@@ -1731,9 +1760,8 @@ pg_generate_packets (vlib_node_runtime_t * node,
       n_trace = vlib_get_trace_count (vm, node);
       if (PREDICT_FALSE (n_trace > 0))
        {
-         n_trace =
-           pg_input_trace (pg, node, s - pg->streams, next_index, to_next,
-                           n_this_frame, n_trace);
+         n_trace = pg_input_trace (pg, node, s - pg->streams, next_index,
+                                   to_next, n_this_frame, n_trace);
          vlib_set_trace_count (vm, node, n_trace);
        }
       n_packets_to_generate -= n_this_frame;
index fa1a14c..5287f3e 100644 (file)
@@ -77,6 +77,7 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
       if (b->flags & VLIB_BUFFER_IS_TRACED)
        {
          pg_output_trace_t *t = vlib_add_trace (vm, node, b, sizeof (*t));
+         t->mode = pif->mode;
          t->buffer_index = bi0;
          clib_memcpy_fast (&t->buffer, b,
                            sizeof (b[0]) - sizeof (b->pre_data));
index 5e63b58..7c5d698 100644 (file)
@@ -348,6 +348,29 @@ pg_intf_mode_to_pcap_packet_type (pg_interface_mode_t mode)
     return PCAP_PACKET_TYPE_ethernet;
 }
 
+#define foreach_pg_interface_flags                                            \
+  _ (CSUM_OFFLOAD, 0)                                                         \
+  _ (GSO, 1)                                                                  \
+  _ (GRO_COALESCE, 2)
+
+typedef enum
+{
+#define _(a, b) PG_INTERFACE_FLAG_##a = (1 << b),
+  foreach_pg_interface_flags
+#undef _
+} pg_interface_flags_t;
+
+typedef struct
+{
+  u32 if_id;
+  pg_interface_mode_t mode;
+  pg_interface_flags_t flags;
+  u32 gso_size;
+  mac_address_t hw_addr;
+  u8 hw_addr_set;
+  int rv;
+} pg_interface_args_t;
+
 typedef struct
 {
   /* TX lock */
@@ -359,6 +382,8 @@ typedef struct
   /* Identifies stream for this interface. */
   u32 id;
 
+  mac_address_t hw_addr;
+
   u8 coalesce_enabled;
   gro_flow_table_t *flow_table;
   u8 gso_enabled;
@@ -425,9 +450,7 @@ void pg_interface_enable_disable_coalesce (pg_interface_t * pi, u8 enable,
                                           u32 tx_node_index);
 
 /* Find/create free packet-generator interface index. */
-u32 pg_interface_add_or_get (pg_main_t *pg, u32 stream_index, u8 gso_enabled,
-                            u32 gso_size, u8 coalesce_enabled,
-                            pg_interface_mode_t mode);
+u32 pg_interface_add_or_get (pg_main_t *pg, pg_interface_args_t *args);
 
 int pg_interface_delete (u32 sw_if_index);
 
@@ -459,6 +482,7 @@ clib_error_t *pg_capture (pg_capture_args_t * a);
 
 typedef struct
 {
+  pg_interface_mode_t mode;
   u32 buffer_index;
   vlib_buffer_t buffer;
 }
index 57fb40c..6895353 100644 (file)
@@ -33,11 +33,24 @@ vl_api_pg_create_interface_t_handler (vl_api_pg_create_interface_t * mp)
 {
   vl_api_pg_create_interface_reply_t *rmp;
   int rv = 0;
+  u32 pg_if_id = ~0;
 
   pg_main_t *pg = &pg_main;
-  u32 pg_if_id =
-    pg_interface_add_or_get (pg, ntohl (mp->interface_id), mp->gso_enabled,
-                            ntohl (mp->gso_size), 0, PG_MODE_ETHERNET);
+  pg_interface_args_t args = { 0 };
+
+  args.mode = PG_MODE_ETHERNET;
+  args.gso_size = 0;
+  args.hw_addr_set = 0;
+  args.flags = 0;
+  args.if_id = ntohl (mp->interface_id);
+
+  if (mp->gso_enabled)
+    {
+      args.flags = PG_INTERFACE_FLAG_GSO;
+      args.gso_size = ntohl (mp->gso_size);
+    }
+
+  pg_if_id = pg_interface_add_or_get (pg, &args);
   pg_interface_t *pi = pool_elt_at_index (pg->interfaces, pg_if_id);
 
   REPLY_MACRO2(VL_API_PG_CREATE_INTERFACE_REPLY,
@@ -51,11 +64,25 @@ vl_api_pg_create_interface_v2_t_handler (vl_api_pg_create_interface_v2_t *mp)
 {
   vl_api_pg_create_interface_v2_reply_t *rmp;
   int rv = 0;
+  u32 pg_if_id = ~0;
 
   pg_main_t *pg = &pg_main;
-  u32 pg_if_id =
-    pg_interface_add_or_get (pg, ntohl (mp->interface_id), mp->gso_enabled,
-                            ntohl (mp->gso_size), 0, (u8) mp->mode);
+
+  pg_interface_args_t args = { 0 };
+
+  args.mode = (pg_interface_mode_t) mp->mode;
+  args.gso_size = 0;
+  args.hw_addr_set = 0;
+  args.flags = 0;
+  args.if_id = ntohl (mp->interface_id);
+
+  if (mp->gso_enabled)
+    {
+      args.flags = PG_INTERFACE_FLAG_GSO;
+      args.gso_size = ntohl (mp->gso_size);
+    }
+
+  pg_if_id = pg_interface_add_or_get (pg, &args);
   pg_interface_t *pi = pool_elt_at_index (pg->interfaces, pg_if_id);
 
   REPLY_MACRO2 (VL_API_PG_CREATE_INTERFACE_V2_REPLY,
index 440e285..31a41e8 100644 (file)
@@ -105,9 +105,16 @@ format_pg_output_trace (u8 * s, va_list * va)
   s = format (s, "%Ubuffer 0x%x: %U", format_white_space, indent,
              t->buffer_index, format_vnet_buffer_no_chain, &t->buffer);
 
-  s = format (s, "\n%U%U", format_white_space, indent,
-             format_ethernet_header_with_length, t->buffer.pre_data,
-             sizeof (t->buffer.pre_data));
+  if (t->mode == PG_MODE_IP4)
+    s = format (s, "\n%U%U", format_white_space, indent, format_ip4_header,
+               t->buffer.pre_data, sizeof (t->buffer.pre_data));
+  else if (t->mode == PG_MODE_IP6)
+    s = format (s, "\n%U%U", format_white_space, indent, format_ip6_header,
+               t->buffer.pre_data, sizeof (t->buffer.pre_data));
+  else
+    s = format (s, "\n%U%U", format_white_space, indent,
+               format_ethernet_header_with_length, t->buffer.pre_data,
+               sizeof (t->buffer.pre_data));
 
   return s;
 }
@@ -245,18 +252,15 @@ VNET_HW_INTERFACE_CLASS (pg_tun_hw_interface_class) = {
 };
 
 u32
-pg_interface_add_or_get (pg_main_t *pg, u32 if_id, u8 gso_enabled,
-                        u32 gso_size, u8 coalesce_enabled,
-                        pg_interface_mode_t mode)
+pg_interface_add_or_get (pg_main_t *pg, pg_interface_args_t *args)
 {
   vnet_main_t *vnm = vnet_get_main ();
-  vlib_main_t *vm = vlib_get_main ();
   pg_interface_t *pi;
   vnet_hw_interface_t *hi;
   uword *p;
   u32 i;
 
-  p = hash_get (pg->if_index_by_if_id, if_id);
+  p = hash_get (pg->if_index_by_if_id, args->if_id);
 
   if (p)
     {
@@ -264,32 +268,26 @@ pg_interface_add_or_get (pg_main_t *pg, u32 if_id, u8 gso_enabled,
     }
   else
     {
-      vnet_eth_interface_registration_t eir = {};
-      u8 hw_addr[6];
-      f64 now = vlib_time_now (vm);
-      u32 rnd;
-
       pool_get (pg->interfaces, pi);
       i = pi - pg->interfaces;
-
-      rnd = (u32) (now * 1e6);
-      rnd = random_u32 (&rnd);
-      clib_memcpy_fast (hw_addr + 2, &rnd, sizeof (rnd));
-      hw_addr[0] = 2;
-      hw_addr[1] = 0xfe;
-
-      pi->id = if_id;
-      pi->mode = mode;
+      pi->id = args->if_id;
+      pi->mode = args->mode;
 
       switch (pi->mode)
        {
        case PG_MODE_ETHERNET:
-         eir.dev_class_index = pg_dev_class.index;
-         eir.dev_instance = i;
-         eir.address = hw_addr;
-         eir.cb.flag_change = pg_eth_flag_change;
-         pi->hw_if_index = vnet_eth_register_interface (vnm, &eir);
-         break;
+         {
+           vnet_eth_interface_registration_t eir = { 0 };
+           if (!args->hw_addr_set)
+             ethernet_mac_address_generate (args->hw_addr.bytes);
+           clib_memcpy (pi->hw_addr.bytes, args->hw_addr.bytes, 6);
+           eir.dev_class_index = pg_dev_class.index;
+           eir.dev_instance = i;
+           eir.address = pi->hw_addr.bytes;
+           eir.cb.flag_change = pg_eth_flag_change;
+           pi->hw_if_index = vnet_eth_register_interface (vnm, &eir);
+           break;
+         }
        case PG_MODE_IP4:
        case PG_MODE_IP6:
          pi->hw_if_index = vnet_register_interface (
@@ -297,19 +295,19 @@ pg_interface_add_or_get (pg_main_t *pg, u32 if_id, u8 gso_enabled,
          break;
        }
       hi = vnet_get_hw_interface (vnm, pi->hw_if_index);
-      if (gso_enabled)
+      if (args->flags & PG_INTERFACE_FLAG_GSO)
        {
          vnet_hw_if_set_caps (vnm, pi->hw_if_index, VNET_HW_IF_CAP_TCP_GSO);
          pi->gso_enabled = 1;
-         pi->gso_size = gso_size;
-         if (coalesce_enabled)
+         pi->gso_size = args->gso_size;
+         if (args->flags & PG_INTERFACE_FLAG_GRO_COALESCE)
            {
              pg_interface_enable_disable_coalesce (pi, 1, hi->tx_node_index);
            }
        }
       pi->sw_if_index = hi->sw_if_index;
 
-      hash_set (pg->if_index_by_if_id, if_id, i);
+      hash_set (pg->if_index_by_if_id, pi->id, i);
 
       vec_validate (pg->if_index_by_sw_if_index, hi->sw_if_index);
       pg->if_index_by_sw_if_index[hi->sw_if_index] = i;
@@ -585,10 +583,16 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
     vec_resize (s->buffer_indices, n);
   }
 
+  pg_interface_args_t args = {
+    .if_id = s->if_id,
+    .mode = PG_MODE_ETHERNET,
+    .flags = 0,              /* gso_enabled and coalesce_enabled */
+    .gso_size = 0,    /* gso_size */
+    .hw_addr_set = 0, /* mac address set */
+  };
+
   /* Find an interface to use. */
-  s->pg_if_index = pg_interface_add_or_get (
-    pg, s->if_id, 0 /* gso_enabled */, 0 /* gso_size */,
-    0 /* coalesce_enabled */, PG_MODE_ETHERNET);
+  s->pg_if_index = pg_interface_add_or_get (pg, &args);
 
   if (s->sw_if_index[VLIB_RX] == ~0)
     {
index 6c01d3b..471c85c 100644 (file)
@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 
 import unittest
+import time
+import re
 
 from scapy.packet import Raw
 from scapy.layers.l2 import Ether
@@ -50,16 +52,20 @@ class TestPgStream(VppTestCase):
         count = str(count)
 
         cmds = [
+            "clear trace",
+            "trace add pg-input 1000",
             "packet-generator new {{\n"
-            "  name pg0-stream\n"
+            "  name pg0-pg1-stream\n"
             "  limit {count}\n"
             "  node ethernet-input\n"
             "  source pg0\n"
             "  rate {rate}\n"
             "  size {packet_size}+{packet_size}\n"
+            "  buffer-flags ip4 offload\n"
+            "  buffer-offload-flags offload-ip-cksum offload-udp-cksum\n"
             "  data {{\n"
-            "    IP4: {src_mac} -> 00:02:03:04:05:06\n"
-            "    UDP: 192.168.20.20 -> 192.168.10.100\n"
+            "    IP4: {src_mac} -> {dst_mac}\n"
+            "    UDP: {src} -> {dst}\n"
             "    UDP: 1234 -> 4321\n"
             "    incrementing 100\n"
             "  }}\n"
@@ -67,11 +73,79 @@ class TestPgStream(VppTestCase):
                 count=count,
                 rate=rate,
                 packet_size=packet_size,
-                src_mac=self.pg0.local_mac,
+                src_mac=self.pg0.remote_mac,
+                dst_mac=self.pg0.local_mac,
+                src=self.pg0.remote_ip4,
+                dst=self.pg1.remote_ip4,
+            ),
+            "packet-generator new {{\n"
+            "  name pg0-pg2-stream\n"
+            "  limit {count}\n"
+            "  node ethernet-input\n"
+            "  source pg0\n"
+            "  rate {rate}\n"
+            "  size {packet_size}+{packet_size}\n"
+            "  buffer-flags ip6 offload\n"
+            "  buffer-offload-flags offload-udp-cksum\n"
+            "  data {{\n"
+            "    IP6: {src_mac} -> {dst_mac}\n"
+            "    UDP: {src} -> {dst}\n"
+            "    UDP: 1234 -> 4321\n"
+            "    incrementing 100\n"
+            "  }}\n"
+            "}}\n".format(
+                count=count,
+                rate=rate,
+                packet_size=packet_size,
+                src_mac=self.pg0.remote_mac,
+                dst_mac=self.pg0.local_mac,
+                src=self.pg0.remote_ip6,
+                dst=self.pg2.remote_ip6,
+            ),
+            "packet-generator new {{\n"
+            "  name pg1-pg0-stream\n"
+            "  limit {count}\n"
+            "  node ip4-input\n"
+            "  source pg1\n"
+            "  rate {rate}\n"
+            "  size {packet_size}+{packet_size}\n"
+            "  buffer-flags ip4 offload\n"
+            "  buffer-offload-flags offload-ip-cksum offload-udp-cksum\n"
+            "  data {{\n"
+            "    UDP: {src} -> {dst}\n"
+            "    UDP: 1234 -> 4321\n"
+            "    incrementing 100\n"
+            "  }}\n"
+            "}}\n".format(
+                count=count,
+                rate=rate,
+                packet_size=packet_size,
+                src=self.pg1.remote_ip4,
+                dst=self.pg0.remote_ip4,
+            ),
+            "packet-generator new {{\n"
+            "  name pg2-pg0-stream\n"
+            "  limit {count}\n"
+            "  node ip6-input\n"
+            "  source pg2\n"
+            "  rate {rate}\n"
+            "  size {packet_size}+{packet_size}\n"
+            "  buffer-flags ip6 offload\n"
+            "  buffer-offload-flags offload-udp-cksum\n"
+            "  data {{\n"
+            "    UDP: {src} -> {dst}\n"
+            "    UDP: 1234 -> 4321\n"
+            "    incrementing 100\n"
+            "  }}\n"
+            "}}\n".format(
+                count=count,
+                rate=rate,
+                packet_size=packet_size,
+                src=self.pg2.remote_ip6,
+                dst=self.pg0.remote_ip6,
             ),
             "packet-generator enable",
-            "packet-generator disable",
-            "packet-generator delete pg0-stream",
+            "show error",
         ]
 
         for cmd in cmds:
@@ -81,6 +155,79 @@ class TestPgStream(VppTestCase):
                     self.logger.info(cmd + " FAIL reply " + r.reply)
                 else:
                     self.logger.info(cmd + " FAIL retval " + str(r.retval))
+            self.assertTrue(r.retval == 0)
+
+        deadline = time.time() + 30
+        while self.vapi.cli("show packet-generator").find("Yes") != -1:
+            self.sleep(0.01)  # yield
+            if time.time() > deadline:
+                self.logger.error("Timeout waiting for pg to stop")
+                break
+
+        r = self.vapi.cli_return_response("show trace")
+        self.assertTrue(r.retval == 0)
+        self.assertTrue(hasattr(r, "reply"))
+        rv = r.reply
+        packets = rv.split("\nPacket ")
+        for packet in enumerate(packets, start=1):
+            match = re.search(r"stream\s+([\w-]+)", packet[1])
+            if match:
+                stream_name = match.group(1)
+            else:
+                continue
+            if stream_name == "pg0-pg1-stream":
+                look_here = packet[1].find("ethernet-input")
+                self.assertNotEqual(look_here, -1)
+                search_string = "ip4 offload-ip-cksum offload-udp-cksum  l2-hdr-offset 0 l3-hdr-offset 14 l4-hdr-offset 34"
+                look_here = packet[1].find(search_string)
+                self.assertNotEqual(look_here, -1)
+                search_string = "ip4 l2-hdr-offset 0 l3-hdr-offset 14 l4-hdr-offset 34"
+                look_here = packet[1].find(search_string)
+                self.assertNotEqual(look_here, -1)
+            elif stream_name == "pg0-pg2-stream":
+                look_here = packet[1].find("ethernet-input")
+                self.assertNotEqual(look_here, -1)
+                search_string = "ip6 offload-udp-cksum  l2-hdr-offset 0 l3-hdr-offset 14 l4-hdr-offset 54"
+                look_here = packet[1].find(search_string)
+                self.assertNotEqual(look_here, -1)
+                search_string = "ip6 l2-hdr-offset 0 l3-hdr-offset 14 l4-hdr-offset 54"
+                look_here = packet[1].find(search_string)
+                self.assertNotEqual(look_here, -1)
+            elif stream_name == "pg1-pg0-stream":
+                look_here = packet[1].find("ethernet-input")
+                self.assertEqual(look_here, -1)
+                look_here = packet[1].find("ip4-input")
+                self.assertNotEqual(look_here, -1)
+                search_string = "ip4 offload-ip-cksum offload-udp-cksum  l2-hdr-offset 0 l3-hdr-offset 0 l4-hdr-offset 20"
+                look_here = packet[1].find(search_string)
+                self.assertNotEqual(look_here, -1)
+                search_string = "ip4 l2-hdr-offset 0 l3-hdr-offset 0 l4-hdr-offset 20"
+                look_here = packet[1].find(search_string)
+                self.assertNotEqual(look_here, -1)
+            elif stream_name == "pg2-pg0-stream":
+                look_here = packet[1].find("ethernet-input")
+                self.assertEqual(look_here, -1)
+                look_here = packet[1].find("ip6-input")
+                self.assertNotEqual(look_here, -1)
+                search_string = "ip6 offload-udp-cksum  l2-hdr-offset 0 l3-hdr-offset 0 l4-hdr-offset 40"
+                look_here = packet[1].find(search_string)
+                self.assertNotEqual(look_here, -1)
+                search_string = "ip6 l2-hdr-offset 0 l3-hdr-offset 0 l4-hdr-offset 40"
+                look_here = packet[1].find(search_string)
+                self.assertNotEqual(look_here, -1)
+
+        self.logger.info(self.vapi.cli("packet-generator disable"))
+        self.logger.info(self.vapi.cli("packet-generator delete pg0-pg1-stream"))
+        self.logger.info(self.vapi.cli("packet-generator delete pg0-pg2-stream"))
+        self.logger.info(self.vapi.cli("packet-generator delete pg1-pg0-stream"))
+        self.logger.info(self.vapi.cli("packet-generator delete pg2-pg0-stream"))
+
+        r = self.vapi.cli_return_response("show buffers")
+        self.assertTrue(r.retval == 0)
+        self.assertTrue(hasattr(r, "reply"))
+        rv = r.reply
+        used = int(rv.strip().split("\n")[-1].split()[-1])
+        self.assertEqual(used, 0)
 
     def test_pg_stream(self):
         """PG Stream testing"""