docs/usecases/contiv/VPP_PACKET_TRACING_K8S.rst

   1 How to do VPP Packet Tracing in Kubernetes
   2 ==========================================
   3
   4 This document describes the steps to do *manual* packet tracing
   5 (capture) using VPP in Kubernetes. Contiv/VPP also ships with a simple
   6 bash script
   7 `vpptrace.sh <https://github.com/contiv/vpp/blob/master/scripts/vpptrace.sh>`__,
   8 which allows to *continuously* trace and *filter* packets incoming
   9 through a given set of interface types. Documentation for vpptrace.sh is
  10 available
  11 `here <https://github.com/contiv/vpp/blob/master/docs/VPPTRACE.md>`__.
  12
  13 More information about VPP packet tracing is in:
  14
  15 -  https://wiki.fd.io/view/VPP/Command-line_Interface_(CLI)_Guide#packet_tracer
  16 -  https://wiki.fd.io/view/VPP/How_To_Use_The_Packet_Generator_and_Packet_Tracer
  17 -  https://wiki.fd.io/view/VPP/Tutorial_Routing_and_Switching
  18
  19 SSH into the Node
  20 -----------------
  21
  22 Perform the following commands to SSH into the node:
  23
  24 ::
  25
  26    cd vpp/vagrant/vagrant-scripts/
  27    vagrant ssh k8s-worker1
  28
  29 Check the VPP Graph Nodes (Input and Output Queues)
  30 ---------------------------------------------------
  31
  32 The following content shows what is running on VPP, via the ``show run``
  33 command
  34
  35 ::
  36
  37     vagrant@k8s-worker1:~$ sudo vppctl
  38         _______    _        _   _____  ___
  39       __/ __/ _ \  (_)__    | | / / _ \/ _ \
  40       _/ _// // / / / _ \   | |/ / ___/ ___/
  41       /_/ /____(_)_/\___/   |___/_/  /_/
  42
  43     vpp# show run
  44     Time 1026791.9, average vectors/node 1.12, last 128 main loops 0.00 per node 0.00
  45       vector rates in 1.6459e-4, out 1.5485e-4, drop 1.3635e-5, punt 0.0000e0
  46                   Name                 State         Calls          Vectors        Suspends         Clocks       Vectors/Call
  47     GigabitEthernet0/8/0-output      active                 56              69               0          1.34e3            1.23
  48     GigabitEthernet0/8/0-tx          active                 54              67               0          8.09e5            1.24
  49     acl-plugin-fa-cleaner-process  event wait                0               0               1          2.84e4            0.00
  50     admin-up-down-process          event wait                0               0               1          4.59e3            0.00
  51     api-rx-from-ring                any wait                 0               0         3316292          1.24e5            0.00
  52     arp-input                        active                  3               3               0          2.53e5            1.00
  53     bfd-process                    event wait                0               0               1          5.94e3            0.00
  54     cdp-process                     any wait                 0               0          145916          1.36e4            0.00
  55     dhcp-client-process             any wait                 0               0           10268          3.65e4            0.00
  56     dns-resolver-process            any wait                 0               0            1027          5.86e4            0.00
  57     dpdk-input                       polling     8211032318951              93               0         1.48e13            0.00
  58     dpdk-ipsec-process                done                   1               0               0          2.10e5            0.00
  59     dpdk-process                    any wait                 0               0          342233          9.86e6            0.00
  60     error-drop                       active                 12              14               0          6.67e3            1.17
  61     ethernet-input                   active                 60              74               0          5.81e3            1.23
  62     fib-walk                        any wait                 0               0          513322          1.59e4            0.00
  63     flow-report-process             any wait                 0               0               1          1.45e3            0.00
  64     flowprobe-timer-process         any wait                 0               0               1          6.34e3            0.00
  65     ikev2-manager-process           any wait                 0               0         1026484          1.18e4            0.00
  66     interface-output                 active                  2               2               0          3.23e3            1.00
  67     ioam-export-process             any wait                 0               0               1          1.98e3            0.00
  68     ip-route-resolver-process       any wait                 0               0           10268          3.02e4            0.00
  69     ip4-arp                          active                  1               1               0          1.49e4            1.00
  70     ip4-input                        active                223             248               0          3.39e3            1.11
  71     ip4-load-balance                 active                106             132               0          5.34e3            1.25
  72     ip4-local                        active                 86              92               0          2.46e3            1.07
  73     ip4-local-end-of-arc             active                 86              92               0          1.00e3            1.07
  74     ip4-lookup                       active                223             248               0          3.31e3            1.11
  75     ip4-rewrite                      active                190             222               0          1.92e3            1.17
  76     ip4-udp-lookup                   active                 86              92               0          3.76e3            1.07
  77     ip6-drop                         active                  6               7               0          2.29e3            1.17
  78     ip6-icmp-neighbor-discovery-ev  any wait                 0               0         1026484          1.13e4            0.00
  79     ip6-input                        active                  6               7               0          3.33e3            1.17
  80     l2-flood                         active                  2               2               0          4.42e3            1.00
  81     l2-fwd                           active                138             157               0          2.13e3            1.14
  82     l2-input                         active                140             159               0          2.41e3            1.14
  83     l2-learn                         active                 86              92               0          3.64e4            1.07
  84     l2-output                        active                 54              67               0          3.05e3            1.24
  85     l2fib-mac-age-scanner-process  event wait                0               0              85          5.01e4            0.00
  86     lisp-retry-service              any wait                 0               0          513322          1.62e4            0.00
  87     lldp-process                   event wait                0               0               1          5.02e4            0.00
  88     loop0-output                     active                 54              67               0          1.66e3            1.24
  89     loop0-tx                         active                 54               0               0          2.49e3            0.00
  90     memif-process                  event wait                0               0               1          1.70e4            0.00
  91     nat-det-expire-walk               done                   1               0               0          3.79e3            0.00
  92     nat44-classify                   active                171             183               0          2.49e3            1.07
  93     nat44-hairpinning                active                 86              92               0          1.80e3            1.07
  94     nat44-in2out                     active                171             183               0          4.45e3            1.07
  95     nat44-in2out-slowpath            active                171             183               0          3.98e3            1.07
  96     nat44-out2in                     active                 52              65               0          1.28e4            1.25
  97     nat64-expire-walk               any wait                 0               0          102677          5.95e4            0.00
  98     nat64-expire-worker-walk      interrupt wa          102676               0               0          7.39e3            0.00
  99     send-garp-na-process           event wait                0               0               1          1.28e3            0.00
 100     startup-config-process            done                   1               0               1          4.19e3            0.00
 101     tapcli-0-output                  active                  1               1               0          6.97e3            1.00
 102     tapcli-0-tx                      active                  1               1               0          7.32e4            1.00
 103     tapcli-1-output                  active                 57              63               0          1.66e3            1.11
 104     tapcli-1-tx                      active                 57              63               0          1.35e5            1.11
 105     tapcli-2-output                  active                 28              28               0          3.26e3            1.00
 106     tapcli-2-tx                      active                 28              28               0          4.06e5            1.00
 107     tapcli-rx                     interrupt wa              62              76               0          6.58e4            1.23
 108     udp-ping-process                any wait                 0               0               1          1.79e4            0.00
 109     unix-cli-127.0.0.1:43282         active                  2               0             455         1.26e15            0.00
 110     unix-epoll-input                 polling        8010763239               0               0          8.17e2            0.00
 111     vhost-user-process              any wait                 0               0               1          1.96e3            0.00
 112     vhost-user-send-interrupt-proc  any wait                 0               0               1          3.85e3            0.00
 113     vpe-link-state-process         event wait                0               0               8          9.79e4            0.00
 114     vpe-oam-process                 any wait                 0               0          503263          1.21e4            0.00
 115     vxlan-gpe-ioam-export-process   any wait                 0               0               1          2.91e3            0.00
 116     vxlan4-encap                     active                 54              67               0          3.55e3            1.24
 117     vxlan4-input                     active                 86              92               0          3.79e3            1.07
 118     wildcard-ip4-arp-publisher-pro event wait                0               0               1          6.44e3            0.00
 119
 120 ``tapcli-rx`` above is the node-level input queue for incoming packets
 121 into all the pods on the node. There is one ``tapcli-rx`` input queue
 122 for every node.
 123
 124 The following are the input and output queues for each pod and the node:
 125
 126 ::
 127
 128    tapcli-0-output
 129    tapcli-0-tx
 130    tapcli-1-output
 131    tapcli-1-tx
 132    tapcli-2-output
 133    tapcli-2-tx
 134
 135 Each pod and node has two queues, one for rx (``tapcli-X-output``), and
 136 one for tx (``tapcli-X-tx``). The above output is with two ``nginx``
 137 pods in kubernetes.
 138
 139 Clear Existing VPP Packet Trace
 140 -------------------------------
 141
 142 Enter the following command:
 143
 144 ::
 145
 146    vpp# clear trace
 147
 148 How to Turn on VPP Packet Tracing
 149 ---------------------------------
 150
 151 Enter the following commands:
 152
 153 ::
 154
 155    vpp# trace add <input or output queue name> <number of packets to capture>
 156
 157    vpp# trace add dpdk-input 1000
 158
 159    vpp# trace add tapcli-rx 1000
 160
 161 Send Traffic to the Pods
 162 ------------------------
 163
 164 Open another terminal, SSH into the master node, refer the documentation
 165 in ``vpp/vagrant/README.md`` and send traffic to the two ``nginx`` pods
 166 using ``wget``.
 167
 168 ::
 169
 170     cd vpp/vagrant/vagrant-scripts/
 171     vagrant ssh k8s-master
 172
 173     vagrant@k8s-master:~$ kubectl get pods -o wide
 174     NAME                   READY     STATUS    RESTARTS   AGE       IP         NODE
 175     nginx-8586cf59-768qw   1/1       Running   0          11d       10.1.2.3   k8s-worker1
 176     nginx-8586cf59-d27h2   1/1       Running   0          11d       10.1.2.2   k8s-worker1
 177
 178     vagrant@k8s-master:~$ wget 10.1.2.2
 179     --2018-02-08 16:46:01--  http://10.1.2.2/
 180     Connecting to 10.1.2.2:80... connected.
 181     HTTP request sent, awaiting response... 200 OK
 182     Length: 612 [text/html]
 183     Saving to: ‘index.html’
 184     index.html                       100%[=========================================================>]     612  --.-KB/s    in 0.004s
 185     2018-02-08 16:46:01 (162 KB/s) - ‘index.html’ saved [612/612]
 186
 187     vagrant@k8s-master:~$ wget 10.1.2.3
 188     --2018-02-08 16:46:02--  http://10.1.2.3/
 189     Connecting to 10.1.2.3:80... connected.
 190     HTTP request sent, awaiting response... 200 OK
 191     Length: 612 [text/html]
 192     Saving to: ‘index.html.1’
 193     index.html.1                     100%[=========================================================>]     612  --.-KB/s    in 0.004s
 194     2018-02-08 16:46:02 (143 KB/s) - ‘index.html.1’ saved [612/612]
 195
 196 Check the Packets Captured by VPP
 197 ---------------------------------
 198
 199 Back in the first terminal, check the packets captured by VPP.
 200
 201 ::
 202
 203     vpp# show trace
 204     ...
 205     ...
 206     Packet 33
 207
 208     21:34:51:476110: tapcli-rx
 209       tapcli-2
 210     21:34:51:476115: ethernet-input
 211       IP4: 00:00:00:00:00:02 -> 02:fe:72:95:66:c7
 212     21:34:51:476117: ip4-input
 213       TCP: 10.1.2.3 -> 172.30.1.2
 214         tos 0x00, ttl 64, length 52, checksum 0x6fb4
 215         fragment id 0x11ec, flags DONT_FRAGMENT
 216       TCP: 80 -> 58430
 217         seq. 0x5db741c8 ack 0x709defa7
 218         flags 0x11 FIN ACK, tcp header: 32 bytes
 219         window 235, checksum 0x55c3
 220     21:34:51:476118: nat44-out2in
 221       NAT44_OUT2IN: sw_if_index 6, next index 1, session index -1
 222     21:34:51:476120: ip4-lookup
 223       fib 0 dpo-idx 23 flow hash: 0x00000000
 224       TCP: 10.1.2.3 -> 172.30.1.2
 225         tos 0x00, ttl 64, length 52, checksum 0x6fb4
 226         fragment id 0x11ec, flags DONT_FRAGMENT
 227       TCP: 80 -> 58430
 228         seq. 0x5db741c8 ack 0x709defa7
 229         flags 0x11 FIN ACK, tcp header: 32 bytes
 230         window 235, checksum 0x55c3
 231     21:34:51:476121: ip4-load-balance
 232       fib 0 dpo-idx 23 flow hash: 0x00000000
 233       TCP: 10.1.2.3 -> 172.30.1.2
 234         tos 0x00, ttl 64, length 52, checksum 0x6fb4
 235         fragment id 0x11ec, flags DONT_FRAGMENT
 236       TCP: 80 -> 58430
 237         seq. 0x5db741c8 ack 0x709defa7
 238         flags 0x11 FIN ACK, tcp header: 32 bytes
 239         window 235, checksum 0x55c3
 240     21:34:51:476122: ip4-rewrite
 241       tx_sw_if_index 3 dpo-idx 5 : ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800 flow hash: 0x00000000
 242       00000000: 1a2b3c4d5e011a2b3c4d5e0208004500003411ec40003f0670b40a010203ac1e
 243       00000020: 01020050e43e5db741c8709defa7801100eb55c300000101080a0f4b
 244     21:34:51:476123: loop0-output
 245       loop0
 246       IP4: 1a:2b:3c:4d:5e:02 -> 1a:2b:3c:4d:5e:01
 247       TCP: 10.1.2.3 -> 172.30.1.2
 248         tos 0x00, ttl 63, length 52, checksum 0x70b4
 249         fragment id 0x11ec, flags DONT_FRAGMENT
 250       TCP: 80 -> 58430
 251         seq. 0x5db741c8 ack 0x709defa7
 252         flags 0x11 FIN ACK, tcp header: 32 bytes
 253         window 235, checksum 0x55c3
 254     21:34:51:476124: l2-input
 255       l2-input: sw_if_index 3 dst 1a:2b:3c:4d:5e:01 src 1a:2b:3c:4d:5e:02
 256     21:34:51:476125: l2-fwd
 257       l2-fwd:   sw_if_index 3 dst 1a:2b:3c:4d:5e:01 src 1a:2b:3c:4d:5e:02 bd_index 1
 258     21:34:51:476125: l2-output
 259       l2-output: sw_if_index 4 dst 1a:2b:3c:4d:5e:01 src 1a:2b:3c:4d:5e:02 data 08 00 45 00 00 34 11 ec 40 00 3f 06
 260     21:34:51:476126: vxlan4-encap
 261       VXLAN encap to vxlan_tunnel0 vni 10
 262     21:34:51:476126: ip4-load-balance
 263       fib 4 dpo-idx 22 flow hash: 0x00000103
 264       UDP: 192.168.16.2 -> 192.168.16.1
 265         tos 0x00, ttl 254, length 102, checksum 0x1b33
 266         fragment id 0x0000
 267       UDP: 24320 -> 4789
 268         length 82, checksum 0x0000
 269     21:34:51:476127: ip4-rewrite
 270       tx_sw_if_index 1 dpo-idx 4 : ipv4 via 192.168.16.1 GigabitEthernet0/8/0: 080027b2610908002733fb6f0800 flow hash: 0x00000103
 271       00000000: 080027b2610908002733fb6f08004500006600000000fd111c33c0a81002c0a8
 272       00000020: 10015f0012b5005200000800000000000a001a2b3c4d5e011a2b3c4d
 273     21:34:51:476127: GigabitEthernet0/8/0-output
 274       GigabitEthernet0/8/0
 275       IP4: 08:00:27:33:fb:6f -> 08:00:27:b2:61:09
 276       UDP: 192.168.16.2 -> 192.168.16.1
 277         tos 0x00, ttl 253, length 102, checksum 0x1c33
 278         fragment id 0x0000
 279       UDP: 24320 -> 4789
 280         length 82, checksum 0x0000
 281     21:34:51:476128: GigabitEthernet0/8/0-tx
 282       GigabitEthernet0/8/0 tx queue 0
 283       buffer 0xfa7f: current data -50, length 116, free-list 0, clone-count 0, totlen-nifb 0, trace 0x20
 284                       l2-hdr-offset 0 l3-hdr-offset 14
 285       PKT MBUF: port 255, nb_segs 1, pkt_len 116
 286         buf_len 2176, data_len 116, ol_flags 0x0, data_off 78, phys_addr 0x569ea040
 287         packet_type 0x0 l2_len 0 l3_len 0 outer_l2_len 0 outer_l3_len 0
 288       IP4: 08:00:27:33:fb:6f -> 08:00:27:b2:61:09
 289       UDP: 192.168.16.2 -> 192.168.16.1
 290         tos 0x00, ttl 253, length 102, checksum 0x1c33
 291         fragment id 0x0000
 292       UDP: 24320 -> 4789
 293         length 82, checksum 0x0000
 294
 295 In the above captured packet, we can see:
 296
 297 -  Input queue name ``tapcli-rx``
 298 -  Pod’s IP address ``10.1.2.3``
 299 -  IP address of the master node ``172.30.1.2``, which sent the ``wget``
 300    traffic to the two pods
 301 -  HTTP port ``80``, destination port and TCP protocol
 302    (``TCP: 80 -> 58430``)
 303 -  NAT queue name ``nat44-out2in``
 304 -  VXLAN VNI ID ``VXLAN encap to vxlan_tunnel0 vni 10``
 305 -  VXLAN UDP port ``4789``
 306 -  IP address of ``GigabitEthernet0/8/0`` interface (``192.168.16.2``)
 307 -  Packet on the outgoing queue ``GigabitEthernet0/8/0-tx``
 308
 309 Find IP Addresses of GigabitEthernet and the Tap Interfaces
 310 -----------------------------------------------------------
 311
 312 Enter the following commands to find the IP addresses and Tap
 313 interfaces:
 314
 315 ::
 316
 317    vpp# show int address
 318    GigabitEthernet0/8/0 (up):
 319      L3 192.168.16.2/24
 320    local0 (dn):
 321    loop0 (up):
 322      L2 bridge bd-id 1 idx 1 shg 0 bvi
 323      L3 192.168.30.2/24
 324    tapcli-0 (up):
 325      L3 172.30.2.1/24
 326    tapcli-1 (up):
 327      L3 10.2.1.2/32
 328    tapcli-2 (up):
 329      L3 10.2.1.3/32
 330    vxlan_tunnel0 (up):
 331      L2 bridge bd-id 1 idx 1 shg 0
 332
 333 Other Useful VPP CLIs
 334 ---------------------
 335
 336 Enter the following commands to see additional information about VPP:
 337
 338 ::
 339
 340    vpp# show int
 341                  Name               Idx       State          Counter          Count
 342    GigabitEthernet0/8/0              1         up       rx packets                   138
 343                                                         rx bytes                   18681
 344                                                         tx packets                   100
 345                                                         tx bytes                   29658
 346                                                         drops                          1
 347                                                         ip4                          137
 348                                                         tx-error                       2
 349    local0                            0        down      drops                          1
 350    loop0                             3         up       rx packets                   137
 351                                                         rx bytes                    9853
 352                                                         tx packets                   200
 353                                                         tx bytes                   49380
 354                                                         drops                          1
 355                                                         ip4                          136
 356    tapcli-0                          2         up       rx packets                     8
 357                                                         rx bytes                     600
 358                                                         tx packets                     1
 359                                                         tx bytes                      42
 360                                                         drops                          9
 361                                                         ip6                            7
 362    tapcli-1                          5         up       rx packets                    56
 363                                                         rx bytes                   13746
 364                                                         tx packets                    78
 365                                                         tx bytes                    6733
 366                                                         drops                          1
 367                                                         ip4                           56
 368    tapcli-2                          6         up       rx packets                    42
 369                                                         rx bytes                   10860
 370                                                         tx packets                    58
 371                                                         tx bytes                    4996
 372                                                         drops                          1
 373                                                         ip4                           42
 374    vxlan_tunnel0                     4         up       rx packets                   137
 375                                                         rx bytes                   11771
 376                                                         tx packets                   100
 377                                                         tx bytes                   28290
 378
 379    vpp# show hardware
 380                  Name                Idx   Link  Hardware
 381    GigabitEthernet0/8/0               1     up   GigabitEthernet0/8/0
 382      Ethernet address 08:00:27:33:fb:6f
 383      Intel 82540EM (e1000)
 384        carrier up full duplex speed 1000 mtu 9216
 385        rx queues 1, rx desc 1024, tx queues 1, tx desc 1024
 386        cpu socket 0
 387
 388        tx frames ok                                         100
 389        tx bytes ok                                        29658
 390        rx frames ok                                         138
 391        rx bytes ok                                        19233
 392        extended stats:
 393          rx good packets                                    138
 394          tx good packets                                    100
 395          rx good bytes                                    19233
 396          tx good bytes                                    29658
 397    local0                             0    down  local0
 398      local
 399    loop0                              3     up   loop0
 400      Ethernet address 1a:2b:3c:4d:5e:02
 401    tapcli-0                           2     up   tapcli-0
 402      Ethernet address 02:fe:95:07:df:9c
 403    tapcli-1                           5     up   tapcli-1
 404      Ethernet address 02:fe:3f:5f:0f:9a
 405    tapcli-2                           6     up   tapcli-2
 406      Ethernet address 02:fe:72:95:66:c7
 407    vxlan_tunnel0                      4     up   vxlan_tunnel0
 408      VXLAN
 409
 410    vpp# show bridge-domain
 411      BD-ID   Index   BSN  Age(min)  Learning  U-Forwrd  UU-Flood  Flooding  ARP-Term  BVI-Intf
 412        1       1      1     off        on        on        on        on       off      loop0
 413
 414    vpp# show bridge-domain 1 detail
 415      BD-ID   Index   BSN  Age(min)  Learning  U-Forwrd  UU-Flood  Flooding  ARP-Term  BVI-Intf
 416        1       1      1     off        on        on        on        on       off      loop0
 417
 418               Interface           If-idx ISN  SHG  BVI  TxFlood        VLAN-Tag-Rewrite
 419                 loop0               3     3    0    *      *                 none
 420             vxlan_tunnel0           4     1    0    -      *                 none
 421
 422    vpp# show l2fib verbose
 423        Mac-Address     BD-Idx If-Idx BSN-ISN Age(min) static filter bvi         Interface-Name
 424     1a:2b:3c:4d:5e:02    1      3      0/0      -       *      -     *               loop0
 425     1a:2b:3c:4d:5e:01    1      4      1/1      -       -      -     -           vxlan_tunnel0
 426    L2FIB total/learned entries: 2/1  Last scan time: 0.0000e0sec  Learn limit: 4194304
 427
 428    vpp# show ip fib
 429    ipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport proto ] locks:[src:(nil):2, src:adjacency:3, src:default-route:1, ]
 430    0.0.0.0/0
 431      unicast-ip4-chain
 432      [@0]: dpo-load-balance: [proto:ip4 index:1 buckets:1 uRPF:21 to:[0:0]]
 433        [0] [@5]: ipv4 via 172.30.2.2 tapcli-0: def35b93961902fe9507df9c0800
 434    0.0.0.0/32
 435      unicast-ip4-chain
 436      [@0]: dpo-load-balance: [proto:ip4 index:2 buckets:1 uRPF:1 to:[0:0]]
 437        [0] [@0]: dpo-drop ip4
 438    10.1.1.0/24
 439      unicast-ip4-chain
 440      [@0]: dpo-load-balance: [proto:ip4 index:24 buckets:1 uRPF:29 to:[0:0]]
 441        [0] [@10]: dpo-load-balance: [proto:ip4 index:23 buckets:1 uRPF:28 to:[0:0] via:[98:23234]]
 442              [0] [@5]: ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800
 443    10.1.2.2/32
 444      unicast-ip4-chain
 445      [@0]: dpo-load-balance: [proto:ip4 index:27 buckets:1 uRPF:12 to:[78:5641]]
 446        [0] [@5]: ipv4 via 10.1.2.2 tapcli-1: 00000000000202fe3f5f0f9a0800
 447    10.1.2.3/32
 448      unicast-ip4-chain
 449      [@0]: dpo-load-balance: [proto:ip4 index:29 buckets:1 uRPF:32 to:[58:4184]]
 450        [0] [@5]: ipv4 via 10.1.2.3 tapcli-2: 00000000000202fe729566c70800
 451    10.2.1.2/32
 452      unicast-ip4-chain
 453      [@0]: dpo-load-balance: [proto:ip4 index:26 buckets:1 uRPF:31 to:[0:0]]
 454        [0] [@2]: dpo-receive: 10.2.1.2 on tapcli-1
 455    10.2.1.3/32
 456      unicast-ip4-chain
 457      [@0]: dpo-load-balance: [proto:ip4 index:28 buckets:1 uRPF:33 to:[0:0]]
 458        [0] [@2]: dpo-receive: 10.2.1.3 on tapcli-2
 459    172.30.1.0/24
 460      unicast-ip4-chain
 461      [@0]: dpo-load-balance: [proto:ip4 index:25 buckets:1 uRPF:29 to:[98:23234]]
 462        [0] [@10]: dpo-load-balance: [proto:ip4 index:23 buckets:1 uRPF:28 to:[0:0] via:[98:23234]]
 463              [0] [@5]: ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800
 464    172.30.2.0/32
 465      unicast-ip4-chain
 466      [@0]: dpo-load-balance: [proto:ip4 index:14 buckets:1 uRPF:15 to:[0:0]]
 467        [0] [@0]: dpo-drop ip4
 468    172.30.2.0/24
 469      unicast-ip4-chain
 470      [@0]: dpo-load-balance: [proto:ip4 index:13 buckets:1 uRPF:14 to:[0:0]]
 471        [0] [@4]: ipv4-glean: tapcli-0
 472    172.30.2.1/32
 473      unicast-ip4-chain
 474      [@0]: dpo-load-balance: [proto:ip4 index:16 buckets:1 uRPF:19 to:[0:0]]
 475        [0] [@2]: dpo-receive: 172.30.2.1 on tapcli-0
 476    172.30.2.2/32
 477      unicast-ip4-chain
 478      [@0]: dpo-load-balance: [proto:ip4 index:17 buckets:1 uRPF:18 to:[0:0]]
 479        [0] [@5]: ipv4 via 172.30.2.2 tapcli-0: def35b93961902fe9507df9c0800
 480    172.30.2.255/32
 481      unicast-ip4-chain
 482      [@0]: dpo-load-balance: [proto:ip4 index:15 buckets:1 uRPF:17 to:[0:0]]
 483        [0] [@0]: dpo-drop ip4
 484    192.168.16.0/32
 485      unicast-ip4-chain
 486      [@0]: dpo-load-balance: [proto:ip4 index:10 buckets:1 uRPF:9 to:[0:0]]
 487        [0] [@0]: dpo-drop ip4
 488    192.168.16.1/32
 489      unicast-ip4-chain
 490      [@0]: dpo-load-balance: [proto:ip4 index:22 buckets:1 uRPF:34 to:[0:0] via:[100:28290]]
 491        [0] [@5]: ipv4 via 192.168.16.1 GigabitEthernet0/8/0: 080027b2610908002733fb6f0800
 492    192.168.16.0/24
 493      unicast-ip4-chain
 494      [@0]: dpo-load-balance: [proto:ip4 index:9 buckets:1 uRPF:30 to:[0:0]]
 495        [0] [@4]: ipv4-glean: GigabitEthernet0/8/0
 496    192.168.16.2/32
 497      unicast-ip4-chain
 498      [@0]: dpo-load-balance: [proto:ip4 index:12 buckets:1 uRPF:13 to:[137:16703]]
 499        [0] [@2]: dpo-receive: 192.168.16.2 on GigabitEthernet0/8/0
 500    192.168.16.255/32
 501      unicast-ip4-chain
 502      [@0]: dpo-load-balance: [proto:ip4 index:11 buckets:1 uRPF:11 to:[0:0]]
 503        [0] [@0]: dpo-drop ip4
 504    192.168.30.0/32
 505      unicast-ip4-chain
 506      [@0]: dpo-load-balance: [proto:ip4 index:19 buckets:1 uRPF:23 to:[0:0]]
 507        [0] [@0]: dpo-drop ip4
 508    192.168.30.1/32
 509      unicast-ip4-chain
 510      [@0]: dpo-load-balance: [proto:ip4 index:23 buckets:1 uRPF:28 to:[0:0] via:[98:23234]]
 511        [0] [@5]: ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800
 512    192.168.30.0/24
 513      unicast-ip4-chain
 514      [@0]: dpo-load-balance: [proto:ip4 index:18 buckets:1 uRPF:22 to:[0:0]]
 515        [0] [@4]: ipv4-glean: loop0
 516    192.168.30.2/32
 517      unicast-ip4-chain
 518      [@0]: dpo-load-balance: [proto:ip4 index:21 buckets:1 uRPF:27 to:[0:0]]
 519        [0] [@2]: dpo-receive: 192.168.30.2 on loop0
 520    192.168.30.255/32
 521      unicast-ip4-chain
 522      [@0]: dpo-load-balance: [proto:ip4 index:20 buckets:1 uRPF:25 to:[0:0]]
 523        [0] [@0]: dpo-drop ip4
 524    224.0.0.0/4
 525      unicast-ip4-chain
 526      [@0]: dpo-load-balance: [proto:ip4 index:4 buckets:1 uRPF:3 to:[0:0]]
 527        [0] [@0]: dpo-drop ip4
 528    240.0.0.0/4
 529      unicast-ip4-chain
 530      [@0]: dpo-load-balance: [proto:ip4 index:3 buckets:1 uRPF:2 to:[0:0]]
 531        [0] [@0]: dpo-drop ip4
 532    255.255.255.255/32
 533      unicast-ip4-chain
 534      [@0]: dpo-load-balance: [proto:ip4 index:5 buckets:1 uRPF:4 to:[0:0]]
 535        [0] [@0]: dpo-drop ip4