reassembly: fix buffer usage counter
[vpp.git] / test / test_reassembly.py
1 #!/usr/bin/env python
2
3 import six
4 import unittest
5 from random import shuffle
6
7 from framework import VppTestCase, VppTestRunner
8
9 from scapy.packet import Raw
10 from scapy.layers.l2 import Ether, GRE
11 from scapy.layers.inet import IP, UDP, ICMP
12 from util import ppp, fragment_rfc791, fragment_rfc8200
13 from scapy.layers.inet6 import IPv6, IPv6ExtHdrFragment, ICMPv6ParamProblem,\
14     ICMPv6TimeExceeded
15 from vpp_gre_interface import VppGreInterface, VppGre6Interface
16 from vpp_ip import DpoProto
17 from vpp_ip_route import VppIpRoute, VppRoutePath
18
19 # 35 is enough to have >257 400-byte fragments
20 test_packet_count = 35
21
22
23 class TestIPv4Reassembly(VppTestCase):
24     """ IPv4 Reassembly """
25
26     @classmethod
27     def setUpClass(cls):
28         super(TestIPv4Reassembly, cls).setUpClass()
29
30         cls.create_pg_interfaces([0, 1])
31         cls.src_if = cls.pg0
32         cls.dst_if = cls.pg1
33
34         # setup all interfaces
35         for i in cls.pg_interfaces:
36             i.admin_up()
37             i.config_ip4()
38             i.resolve_arp()
39
40         # packet sizes
41         cls.packet_sizes = [64, 512, 1518, 9018]
42         cls.padding = " abcdefghijklmn"
43         cls.create_stream(cls.packet_sizes)
44         cls.create_fragments()
45
46     def setUp(self):
47         """ Test setup - force timeout on existing reassemblies """
48         super(TestIPv4Reassembly, self).setUp()
49         self.vapi.ip_reassembly_enable_disable(
50             sw_if_index=self.src_if.sw_if_index, enable_ip4=True)
51         self.vapi.ip_reassembly_set(timeout_ms=0, max_reassemblies=1000,
52                                     expire_walk_interval_ms=10)
53         self.sleep(.25)
54         self.vapi.ip_reassembly_set(timeout_ms=1000000, max_reassemblies=1000,
55                                     expire_walk_interval_ms=10000)
56
57     def tearDown(self):
58         super(TestIPv4Reassembly, self).tearDown()
59         self.logger.debug(self.vapi.ppcli("show ip4-reassembly details"))
60         self.logger.debug(self.vapi.ppcli("show buffers"))
61
62     @classmethod
63     def create_stream(cls, packet_sizes, packet_count=test_packet_count):
64         """Create input packet stream
65
66         :param list packet_sizes: Required packet sizes.
67         """
68         for i in range(0, packet_count):
69             info = cls.create_packet_info(cls.src_if, cls.src_if)
70             payload = cls.info_to_payload(info)
71             p = (Ether(dst=cls.src_if.local_mac, src=cls.src_if.remote_mac) /
72                  IP(id=info.index, src=cls.src_if.remote_ip4,
73                     dst=cls.dst_if.remote_ip4) /
74                  UDP(sport=1234, dport=5678) /
75                  Raw(payload))
76             size = packet_sizes[(i // 2) % len(packet_sizes)]
77             cls.extend_packet(p, size, cls.padding)
78             info.data = p
79
80     @classmethod
81     def create_fragments(cls):
82         infos = cls._packet_infos
83         cls.pkt_infos = []
84         for index, info in six.iteritems(infos):
85             p = info.data
86             # cls.logger.debug(ppp("Packet:", p.__class__(str(p))))
87             fragments_400 = fragment_rfc791(p, 400)
88             fragments_300 = fragment_rfc791(p, 300)
89             fragments_200 = [
90                 x for f in fragments_400 for x in fragment_rfc791(f, 200)]
91             cls.pkt_infos.append(
92                 (index, fragments_400, fragments_300, fragments_200))
93         cls.fragments_400 = [
94             x for (_, frags, _, _) in cls.pkt_infos for x in frags]
95         cls.fragments_300 = [
96             x for (_, _, frags, _) in cls.pkt_infos for x in frags]
97         cls.fragments_200 = [
98             x for (_, _, _, frags) in cls.pkt_infos for x in frags]
99         cls.logger.debug("Fragmented %s packets into %s 400-byte fragments, "
100                          "%s 300-byte fragments and %s 200-byte fragments" %
101                          (len(infos), len(cls.fragments_400),
102                              len(cls.fragments_300), len(cls.fragments_200)))
103
104     def verify_capture(self, capture, dropped_packet_indexes=[]):
105         """Verify captured packet stream.
106
107         :param list capture: Captured packet stream.
108         """
109         info = None
110         seen = set()
111         for packet in capture:
112             try:
113                 self.logger.debug(ppp("Got packet:", packet))
114                 ip = packet[IP]
115                 udp = packet[UDP]
116                 payload_info = self.payload_to_info(str(packet[Raw]))
117                 packet_index = payload_info.index
118                 self.assertTrue(
119                     packet_index not in dropped_packet_indexes,
120                     ppp("Packet received, but should be dropped:", packet))
121                 if packet_index in seen:
122                     raise Exception(ppp("Duplicate packet received", packet))
123                 seen.add(packet_index)
124                 self.assertEqual(payload_info.dst, self.src_if.sw_if_index)
125                 info = self._packet_infos[packet_index]
126                 self.assertTrue(info is not None)
127                 self.assertEqual(packet_index, info.index)
128                 saved_packet = info.data
129                 self.assertEqual(ip.src, saved_packet[IP].src)
130                 self.assertEqual(ip.dst, saved_packet[IP].dst)
131                 self.assertEqual(udp.payload, saved_packet[UDP].payload)
132             except Exception:
133                 self.logger.error(ppp("Unexpected or invalid packet:", packet))
134                 raise
135         for index in self._packet_infos:
136             self.assertTrue(index in seen or index in dropped_packet_indexes,
137                             "Packet with packet_index %d not received" % index)
138
139     def test_reassembly(self):
140         """ basic reassembly """
141
142         self.pg_enable_capture()
143         self.src_if.add_stream(self.fragments_200)
144         self.pg_start()
145
146         packets = self.dst_if.get_capture(len(self.pkt_infos))
147         self.verify_capture(packets)
148         self.src_if.assert_nothing_captured()
149
150         # run it all again to verify correctness
151         self.pg_enable_capture()
152         self.src_if.add_stream(self.fragments_200)
153         self.pg_start()
154
155         packets = self.dst_if.get_capture(len(self.pkt_infos))
156         self.verify_capture(packets)
157         self.src_if.assert_nothing_captured()
158
159     def test_reversed(self):
160         """ reverse order reassembly """
161
162         fragments = list(self.fragments_200)
163         fragments.reverse()
164
165         self.pg_enable_capture()
166         self.src_if.add_stream(fragments)
167         self.pg_start()
168
169         packets = self.dst_if.get_capture(len(self.packet_infos))
170         self.verify_capture(packets)
171         self.src_if.assert_nothing_captured()
172
173         # run it all again to verify correctness
174         self.pg_enable_capture()
175         self.src_if.add_stream(fragments)
176         self.pg_start()
177
178         packets = self.dst_if.get_capture(len(self.packet_infos))
179         self.verify_capture(packets)
180         self.src_if.assert_nothing_captured()
181
182     def test_5737(self):
183         """ fragment length + ip header size > 65535 """
184         self.vapi.cli("clear errors")
185         raw = ('E\x00\x00\x88,\xf8\x1f\xfe@\x01\x98\x00\xc0\xa8\n-\xc0\xa8\n'
186                '\x01\x08\x00\xf0J\xed\xcb\xf1\xf5Test-group: IPv4.IPv4.ipv4-'
187                'message.Ethernet-Payload.IPv4-Packet.IPv4-Header.Fragment-Of'
188                'fset; Test-case: 5737')
189
190         malformed_packet = (Ether(dst=self.src_if.local_mac,
191                                   src=self.src_if.remote_mac) /
192                             IP(raw))
193         p = (Ether(dst=self.src_if.local_mac, src=self.src_if.remote_mac) /
194              IP(id=1000, src=self.src_if.remote_ip4,
195                 dst=self.dst_if.remote_ip4) /
196              UDP(sport=1234, dport=5678) /
197              Raw("X" * 1000))
198         valid_fragments = fragment_rfc791(p, 400)
199
200         self.pg_enable_capture()
201         self.src_if.add_stream([malformed_packet] + valid_fragments)
202         self.pg_start()
203
204         self.dst_if.get_capture(1)
205         self.assert_packet_counter_equal("ip4-reassembly-feature", 1)
206         # TODO remove above, uncomment below once clearing of counters
207         # is supported
208         # self.assert_packet_counter_equal(
209         #     "/err/ip4-reassembly-feature/malformed packets", 1)
210
211     def test_44924(self):
212         """ compress tiny fragments """
213         packets = [(Ether(dst=self.src_if.local_mac,
214                           src=self.src_if.remote_mac) /
215                     IP(id=24339, flags="MF", frag=0, ttl=64,
216                        src=self.src_if.remote_ip4,
217                        dst=self.dst_if.remote_ip4) /
218                     ICMP(type="echo-request", code=0, id=0x1fe6, seq=0x2407) /
219                     Raw(load='Test-group: IPv4')),
220                    (Ether(dst=self.src_if.local_mac,
221                           src=self.src_if.remote_mac) /
222                     IP(id=24339, flags="MF", frag=3, ttl=64,
223                        src=self.src_if.remote_ip4,
224                        dst=self.dst_if.remote_ip4) /
225                     ICMP(type="echo-request", code=0, id=0x1fe6, seq=0x2407) /
226                     Raw(load='.IPv4.Fragmentation.vali')),
227                    (Ether(dst=self.src_if.local_mac,
228                           src=self.src_if.remote_mac) /
229                     IP(id=24339, frag=6, ttl=64,
230                        src=self.src_if.remote_ip4,
231                        dst=self.dst_if.remote_ip4) /
232                     ICMP(type="echo-request", code=0, id=0x1fe6, seq=0x2407) /
233                     Raw(load='d; Test-case: 44924'))
234                    ]
235
236         self.pg_enable_capture()
237         self.src_if.add_stream(packets)
238         self.pg_start()
239
240         self.dst_if.get_capture(1)
241
242     def test_frag_1(self):
243         """ fragment of size 1 """
244         self.vapi.cli("clear errors")
245         malformed_packets = [(Ether(dst=self.src_if.local_mac,
246                                     src=self.src_if.remote_mac) /
247                               IP(id=7, len=21, flags="MF", frag=0, ttl=64,
248                                  src=self.src_if.remote_ip4,
249                                  dst=self.dst_if.remote_ip4) /
250                               ICMP(type="echo-request")),
251                              (Ether(dst=self.src_if.local_mac,
252                                     src=self.src_if.remote_mac) /
253                               IP(id=7, len=21, frag=1, ttl=64,
254                                  src=self.src_if.remote_ip4,
255                                  dst=self.dst_if.remote_ip4) /
256                               Raw(load='\x08')),
257                              ]
258
259         p = (Ether(dst=self.src_if.local_mac, src=self.src_if.remote_mac) /
260              IP(id=1000, src=self.src_if.remote_ip4,
261                 dst=self.dst_if.remote_ip4) /
262              UDP(sport=1234, dport=5678) /
263              Raw("X" * 1000))
264         valid_fragments = fragment_rfc791(p, 400)
265
266         self.pg_enable_capture()
267         self.src_if.add_stream(malformed_packets + valid_fragments)
268         self.pg_start()
269
270         self.dst_if.get_capture(1)
271
272         self.assert_packet_counter_equal("ip4-reassembly-feature", 1)
273         # TODO remove above, uncomment below once clearing of counters
274         # is supported
275         # self.assert_packet_counter_equal(
276         #     "/err/ip4-reassembly-feature/malformed packets", 1)
277
278     def test_random(self):
279         """ random order reassembly """
280
281         fragments = list(self.fragments_200)
282         shuffle(fragments)
283
284         self.pg_enable_capture()
285         self.src_if.add_stream(fragments)
286         self.pg_start()
287
288         packets = self.dst_if.get_capture(len(self.packet_infos))
289         self.verify_capture(packets)
290         self.src_if.assert_nothing_captured()
291
292         # run it all again to verify correctness
293         self.pg_enable_capture()
294         self.src_if.add_stream(fragments)
295         self.pg_start()
296
297         packets = self.dst_if.get_capture(len(self.packet_infos))
298         self.verify_capture(packets)
299         self.src_if.assert_nothing_captured()
300
301     def test_duplicates(self):
302         """ duplicate fragments """
303
304         fragments = [
305             x for (_, frags, _, _) in self.pkt_infos
306             for x in frags
307             for _ in range(0, min(2, len(frags)))
308         ]
309
310         self.pg_enable_capture()
311         self.src_if.add_stream(fragments)
312         self.pg_start()
313
314         packets = self.dst_if.get_capture(len(self.pkt_infos))
315         self.verify_capture(packets)
316         self.src_if.assert_nothing_captured()
317
318     def test_overlap1(self):
319         """ overlapping fragments case #1 """
320
321         fragments = []
322         for _, _, frags_300, frags_200 in self.pkt_infos:
323             if len(frags_300) == 1:
324                 fragments.extend(frags_300)
325             else:
326                 for i, j in zip(frags_200, frags_300):
327                     fragments.extend(i)
328                     fragments.extend(j)
329
330         self.pg_enable_capture()
331         self.src_if.add_stream(fragments)
332         self.pg_start()
333
334         packets = self.dst_if.get_capture(len(self.pkt_infos))
335         self.verify_capture(packets)
336         self.src_if.assert_nothing_captured()
337
338         # run it all to verify correctness
339         self.pg_enable_capture()
340         self.src_if.add_stream(fragments)
341         self.pg_start()
342
343         packets = self.dst_if.get_capture(len(self.pkt_infos))
344         self.verify_capture(packets)
345         self.src_if.assert_nothing_captured()
346
347     def test_overlap2(self):
348         """ overlapping fragments case #2 """
349
350         fragments = []
351         for _, _, frags_300, frags_200 in self.pkt_infos:
352             if len(frags_300) == 1:
353                 fragments.extend(frags_300)
354             else:
355                 # care must be taken here so that there are no fragments
356                 # received by vpp after reassembly is finished, otherwise
357                 # new reassemblies will be started and packet generator will
358                 # freak out when it detects unfreed buffers
359                 zipped = zip(frags_300, frags_200)
360                 for i, j in zipped[:-1]:
361                     fragments.extend(i)
362                     fragments.extend(j)
363                 fragments.append(zipped[-1][0])
364
365         self.pg_enable_capture()
366         self.src_if.add_stream(fragments)
367         self.pg_start()
368
369         packets = self.dst_if.get_capture(len(self.pkt_infos))
370         self.verify_capture(packets)
371         self.src_if.assert_nothing_captured()
372
373         # run it all to verify correctness
374         self.pg_enable_capture()
375         self.src_if.add_stream(fragments)
376         self.pg_start()
377
378         packets = self.dst_if.get_capture(len(self.pkt_infos))
379         self.verify_capture(packets)
380         self.src_if.assert_nothing_captured()
381
382     def test_timeout_inline(self):
383         """ timeout (inline) """
384
385         dropped_packet_indexes = set(
386             index for (index, frags, _, _) in self.pkt_infos if len(frags) > 1
387         )
388
389         self.vapi.ip_reassembly_set(timeout_ms=0, max_reassemblies=1000,
390                                     expire_walk_interval_ms=10000)
391
392         self.pg_enable_capture()
393         self.src_if.add_stream(self.fragments_400)
394         self.pg_start()
395
396         packets = self.dst_if.get_capture(
397             len(self.pkt_infos) - len(dropped_packet_indexes))
398         self.verify_capture(packets, dropped_packet_indexes)
399         self.src_if.assert_nothing_captured()
400
401     def test_timeout_cleanup(self):
402         """ timeout (cleanup) """
403
404         # whole packets + fragmented packets sans last fragment
405         fragments = [
406             x for (_, frags_400, _, _) in self.pkt_infos
407             for x in frags_400[:-1 if len(frags_400) > 1 else None]
408         ]
409
410         # last fragments for fragmented packets
411         fragments2 = [frags_400[-1]
412                       for (_, frags_400, _, _) in self.pkt_infos
413                       if len(frags_400) > 1]
414
415         dropped_packet_indexes = set(
416             index for (index, frags_400, _, _) in self.pkt_infos
417             if len(frags_400) > 1)
418
419         self.vapi.ip_reassembly_set(timeout_ms=100, max_reassemblies=1000,
420                                     expire_walk_interval_ms=50)
421
422         self.pg_enable_capture()
423         self.src_if.add_stream(fragments)
424         self.pg_start()
425
426         self.sleep(.25, "wait before sending rest of fragments")
427
428         self.src_if.add_stream(fragments2)
429         self.pg_start()
430
431         packets = self.dst_if.get_capture(
432             len(self.pkt_infos) - len(dropped_packet_indexes))
433         self.verify_capture(packets, dropped_packet_indexes)
434         self.src_if.assert_nothing_captured()
435
436     def test_disabled(self):
437         """ reassembly disabled """
438
439         dropped_packet_indexes = set(
440             index for (index, frags_400, _, _) in self.pkt_infos
441             if len(frags_400) > 1)
442
443         self.vapi.ip_reassembly_set(timeout_ms=1000, max_reassemblies=0,
444                                     expire_walk_interval_ms=10000)
445
446         self.pg_enable_capture()
447         self.src_if.add_stream(self.fragments_400)
448         self.pg_start()
449
450         packets = self.dst_if.get_capture(
451             len(self.pkt_infos) - len(dropped_packet_indexes))
452         self.verify_capture(packets, dropped_packet_indexes)
453         self.src_if.assert_nothing_captured()
454
455
456 class TestIPv6Reassembly(VppTestCase):
457     """ IPv6 Reassembly """
458
459     @classmethod
460     def setUpClass(cls):
461         super(TestIPv6Reassembly, cls).setUpClass()
462
463         cls.create_pg_interfaces([0, 1])
464         cls.src_if = cls.pg0
465         cls.dst_if = cls.pg1
466
467         # setup all interfaces
468         for i in cls.pg_interfaces:
469             i.admin_up()
470             i.config_ip6()
471             i.resolve_ndp()
472
473         # packet sizes
474         cls.packet_sizes = [64, 512, 1518, 9018]
475         cls.padding = " abcdefghijklmn"
476         cls.create_stream(cls.packet_sizes)
477         cls.create_fragments()
478
479     def setUp(self):
480         """ Test setup - force timeout on existing reassemblies """
481         super(TestIPv6Reassembly, self).setUp()
482         self.vapi.ip_reassembly_enable_disable(
483             sw_if_index=self.src_if.sw_if_index, enable_ip6=True)
484         self.vapi.ip_reassembly_set(timeout_ms=0, max_reassemblies=1000,
485                                     expire_walk_interval_ms=10, is_ip6=1)
486         self.sleep(.25)
487         self.vapi.ip_reassembly_set(timeout_ms=1000000, max_reassemblies=1000,
488                                     expire_walk_interval_ms=10000, is_ip6=1)
489         self.logger.debug(self.vapi.ppcli("show ip6-reassembly details"))
490         self.logger.debug(self.vapi.ppcli("show buffers"))
491
492     def tearDown(self):
493         super(TestIPv6Reassembly, self).tearDown()
494         self.logger.debug(self.vapi.ppcli("show ip6-reassembly details"))
495         self.logger.debug(self.vapi.ppcli("show buffers"))
496
497     @classmethod
498     def create_stream(cls, packet_sizes, packet_count=test_packet_count):
499         """Create input packet stream for defined interface.
500
501         :param list packet_sizes: Required packet sizes.
502         """
503         for i in range(0, packet_count):
504             info = cls.create_packet_info(cls.src_if, cls.src_if)
505             payload = cls.info_to_payload(info)
506             p = (Ether(dst=cls.src_if.local_mac, src=cls.src_if.remote_mac) /
507                  IPv6(src=cls.src_if.remote_ip6,
508                       dst=cls.dst_if.remote_ip6) /
509                  UDP(sport=1234, dport=5678) /
510                  Raw(payload))
511             size = packet_sizes[(i // 2) % len(packet_sizes)]
512             cls.extend_packet(p, size, cls.padding)
513             info.data = p
514
515     @classmethod
516     def create_fragments(cls):
517         infos = cls._packet_infos
518         cls.pkt_infos = []
519         for index, info in six.iteritems(infos):
520             p = info.data
521             # cls.logger.debug(ppp("Packet:", p.__class__(str(p))))
522             fragments_400 = fragment_rfc8200(p, info.index, 400)
523             fragments_300 = fragment_rfc8200(p, info.index, 300)
524             cls.pkt_infos.append((index, fragments_400, fragments_300))
525         cls.fragments_400 = [
526             x for _, frags, _ in cls.pkt_infos for x in frags]
527         cls.fragments_300 = [
528             x for _, _, frags in cls.pkt_infos for x in frags]
529         cls.logger.debug("Fragmented %s packets into %s 400-byte fragments, "
530                          "and %s 300-byte fragments" %
531                          (len(infos), len(cls.fragments_400),
532                              len(cls.fragments_300)))
533
534     def verify_capture(self, capture, dropped_packet_indexes=[]):
535         """Verify captured packet strea .
536
537         :param list capture: Captured packet stream.
538         """
539         info = None
540         seen = set()
541         for packet in capture:
542             try:
543                 self.logger.debug(ppp("Got packet:", packet))
544                 ip = packet[IPv6]
545                 udp = packet[UDP]
546                 payload_info = self.payload_to_info(str(packet[Raw]))
547                 packet_index = payload_info.index
548                 self.assertTrue(
549                     packet_index not in dropped_packet_indexes,
550                     ppp("Packet received, but should be dropped:", packet))
551                 if packet_index in seen:
552                     raise Exception(ppp("Duplicate packet received", packet))
553                 seen.add(packet_index)
554                 self.assertEqual(payload_info.dst, self.src_if.sw_if_index)
555                 info = self._packet_infos[packet_index]
556                 self.assertTrue(info is not None)
557                 self.assertEqual(packet_index, info.index)
558                 saved_packet = info.data
559                 self.assertEqual(ip.src, saved_packet[IPv6].src)
560                 self.assertEqual(ip.dst, saved_packet[IPv6].dst)
561                 self.assertEqual(udp.payload, saved_packet[UDP].payload)
562             except Exception:
563                 self.logger.error(ppp("Unexpected or invalid packet:", packet))
564                 raise
565         for index in self._packet_infos:
566             self.assertTrue(index in seen or index in dropped_packet_indexes,
567                             "Packet with packet_index %d not received" % index)
568
569     def test_reassembly(self):
570         """ basic reassembly """
571
572         self.pg_enable_capture()
573         self.src_if.add_stream(self.fragments_400)
574         self.pg_start()
575
576         packets = self.dst_if.get_capture(len(self.pkt_infos))
577         self.verify_capture(packets)
578         self.src_if.assert_nothing_captured()
579
580         # run it all again to verify correctness
581         self.pg_enable_capture()
582         self.src_if.add_stream(self.fragments_400)
583         self.pg_start()
584
585         packets = self.dst_if.get_capture(len(self.pkt_infos))
586         self.verify_capture(packets)
587         self.src_if.assert_nothing_captured()
588
589     def test_reversed(self):
590         """ reverse order reassembly """
591
592         fragments = list(self.fragments_400)
593         fragments.reverse()
594
595         self.pg_enable_capture()
596         self.src_if.add_stream(fragments)
597         self.pg_start()
598
599         packets = self.dst_if.get_capture(len(self.pkt_infos))
600         self.verify_capture(packets)
601         self.src_if.assert_nothing_captured()
602
603         # run it all again to verify correctness
604         self.pg_enable_capture()
605         self.src_if.add_stream(fragments)
606         self.pg_start()
607
608         packets = self.dst_if.get_capture(len(self.pkt_infos))
609         self.verify_capture(packets)
610         self.src_if.assert_nothing_captured()
611
612     def test_random(self):
613         """ random order reassembly """
614
615         fragments = list(self.fragments_400)
616         shuffle(fragments)
617
618         self.pg_enable_capture()
619         self.src_if.add_stream(fragments)
620         self.pg_start()
621
622         packets = self.dst_if.get_capture(len(self.pkt_infos))
623         self.verify_capture(packets)
624         self.src_if.assert_nothing_captured()
625
626         # run it all again to verify correctness
627         self.pg_enable_capture()
628         self.src_if.add_stream(fragments)
629         self.pg_start()
630
631         packets = self.dst_if.get_capture(len(self.pkt_infos))
632         self.verify_capture(packets)
633         self.src_if.assert_nothing_captured()
634
635     def test_duplicates(self):
636         """ duplicate fragments """
637
638         fragments = [
639             x for (_, frags, _) in self.pkt_infos
640             for x in frags
641             for _ in range(0, min(2, len(frags)))
642         ]
643
644         self.pg_enable_capture()
645         self.src_if.add_stream(fragments)
646         self.pg_start()
647
648         packets = self.dst_if.get_capture(len(self.pkt_infos))
649         self.verify_capture(packets)
650         self.src_if.assert_nothing_captured()
651
652     def test_overlap1(self):
653         """ overlapping fragments case #1 """
654
655         fragments = []
656         for _, frags_400, frags_300 in self.pkt_infos:
657             if len(frags_300) == 1:
658                 fragments.extend(frags_400)
659             else:
660                 for i, j in zip(frags_300, frags_400):
661                     fragments.extend(i)
662                     fragments.extend(j)
663
664         dropped_packet_indexes = set(
665             index for (index, _, frags) in self.pkt_infos if len(frags) > 1
666         )
667
668         self.pg_enable_capture()
669         self.src_if.add_stream(fragments)
670         self.pg_start()
671
672         packets = self.dst_if.get_capture(
673             len(self.pkt_infos) - len(dropped_packet_indexes))
674         self.verify_capture(packets, dropped_packet_indexes)
675         self.src_if.assert_nothing_captured()
676
677     def test_overlap2(self):
678         """ overlapping fragments case #2 """
679
680         fragments = []
681         for _, frags_400, frags_300 in self.pkt_infos:
682             if len(frags_400) == 1:
683                 fragments.extend(frags_400)
684             else:
685                 # care must be taken here so that there are no fragments
686                 # received by vpp after reassembly is finished, otherwise
687                 # new reassemblies will be started and packet generator will
688                 # freak out when it detects unfreed buffers
689                 zipped = zip(frags_400, frags_300)
690                 for i, j in zipped[:-1]:
691                     fragments.extend(i)
692                     fragments.extend(j)
693                 fragments.append(zipped[-1][0])
694
695         dropped_packet_indexes = set(
696             index for (index, _, frags) in self.pkt_infos if len(frags) > 1
697         )
698
699         self.pg_enable_capture()
700         self.src_if.add_stream(fragments)
701         self.pg_start()
702
703         packets = self.dst_if.get_capture(
704             len(self.pkt_infos) - len(dropped_packet_indexes))
705         self.verify_capture(packets, dropped_packet_indexes)
706         self.src_if.assert_nothing_captured()
707
708     def test_timeout_inline(self):
709         """ timeout (inline) """
710
711         dropped_packet_indexes = set(
712             index for (index, frags, _) in self.pkt_infos if len(frags) > 1
713         )
714
715         self.vapi.ip_reassembly_set(timeout_ms=0, max_reassemblies=1000,
716                                     expire_walk_interval_ms=10000, is_ip6=1)
717
718         self.pg_enable_capture()
719         self.src_if.add_stream(self.fragments_400)
720         self.pg_start()
721
722         packets = self.dst_if.get_capture(
723             len(self.pkt_infos) - len(dropped_packet_indexes))
724         self.verify_capture(packets, dropped_packet_indexes)
725         pkts = self.src_if.get_capture(
726             expected_count=len(dropped_packet_indexes))
727         for icmp in pkts:
728             self.assertIn(ICMPv6TimeExceeded, icmp)
729             self.assertIn(IPv6ExtHdrFragment, icmp)
730             self.assertIn(icmp[IPv6ExtHdrFragment].id, dropped_packet_indexes)
731             dropped_packet_indexes.remove(icmp[IPv6ExtHdrFragment].id)
732
733     def test_timeout_cleanup(self):
734         """ timeout (cleanup) """
735
736         # whole packets + fragmented packets sans last fragment
737         fragments = [
738             x for (_, frags_400, _) in self.pkt_infos
739             for x in frags_400[:-1 if len(frags_400) > 1 else None]
740         ]
741
742         # last fragments for fragmented packets
743         fragments2 = [frags_400[-1]
744                       for (_, frags_400, _) in self.pkt_infos
745                       if len(frags_400) > 1]
746
747         dropped_packet_indexes = set(
748             index for (index, frags_400, _) in self.pkt_infos
749             if len(frags_400) > 1)
750
751         self.vapi.ip_reassembly_set(timeout_ms=100, max_reassemblies=1000,
752                                     expire_walk_interval_ms=50)
753
754         self.vapi.ip_reassembly_set(timeout_ms=100, max_reassemblies=1000,
755                                     expire_walk_interval_ms=50, is_ip6=1)
756
757         self.pg_enable_capture()
758         self.src_if.add_stream(fragments)
759         self.pg_start()
760
761         self.sleep(.25, "wait before sending rest of fragments")
762
763         self.src_if.add_stream(fragments2)
764         self.pg_start()
765
766         packets = self.dst_if.get_capture(
767             len(self.pkt_infos) - len(dropped_packet_indexes))
768         self.verify_capture(packets, dropped_packet_indexes)
769         pkts = self.src_if.get_capture(
770             expected_count=len(dropped_packet_indexes))
771         for icmp in pkts:
772             self.assertIn(ICMPv6TimeExceeded, icmp)
773             self.assertIn(IPv6ExtHdrFragment, icmp)
774             self.assertIn(icmp[IPv6ExtHdrFragment].id, dropped_packet_indexes)
775             dropped_packet_indexes.remove(icmp[IPv6ExtHdrFragment].id)
776
777     def test_disabled(self):
778         """ reassembly disabled """
779
780         dropped_packet_indexes = set(
781             index for (index, frags_400, _) in self.pkt_infos
782             if len(frags_400) > 1)
783
784         self.vapi.ip_reassembly_set(timeout_ms=1000, max_reassemblies=0,
785                                     expire_walk_interval_ms=10000, is_ip6=1)
786
787         self.pg_enable_capture()
788         self.src_if.add_stream(self.fragments_400)
789         self.pg_start()
790
791         packets = self.dst_if.get_capture(
792             len(self.pkt_infos) - len(dropped_packet_indexes))
793         self.verify_capture(packets, dropped_packet_indexes)
794         self.src_if.assert_nothing_captured()
795
796     def test_missing_upper(self):
797         """ missing upper layer """
798         p = (Ether(dst=self.src_if.local_mac, src=self.src_if.remote_mac) /
799              IPv6(src=self.src_if.remote_ip6,
800                   dst=self.src_if.local_ip6) /
801              UDP(sport=1234, dport=5678) /
802              Raw())
803         self.extend_packet(p, 1000, self.padding)
804         fragments = fragment_rfc8200(p, 1, 500)
805         bad_fragment = p.__class__(str(fragments[1]))
806         bad_fragment[IPv6ExtHdrFragment].nh = 59
807         bad_fragment[IPv6ExtHdrFragment].offset = 0
808         self.pg_enable_capture()
809         self.src_if.add_stream([bad_fragment])
810         self.pg_start()
811         pkts = self.src_if.get_capture(expected_count=1)
812         icmp = pkts[0]
813         self.assertIn(ICMPv6ParamProblem, icmp)
814         self.assert_equal(icmp[ICMPv6ParamProblem].code, 3, "ICMP code")
815
816     def test_invalid_frag_size(self):
817         """ fragment size not a multiple of 8 """
818         p = (Ether(dst=self.src_if.local_mac, src=self.src_if.remote_mac) /
819              IPv6(src=self.src_if.remote_ip6,
820                   dst=self.src_if.local_ip6) /
821              UDP(sport=1234, dport=5678) /
822              Raw())
823         self.extend_packet(p, 1000, self.padding)
824         fragments = fragment_rfc8200(p, 1, 500)
825         bad_fragment = fragments[0]
826         self.extend_packet(bad_fragment, len(bad_fragment) + 5)
827         self.pg_enable_capture()
828         self.src_if.add_stream([bad_fragment])
829         self.pg_start()
830         pkts = self.src_if.get_capture(expected_count=1)
831         icmp = pkts[0]
832         self.assertIn(ICMPv6ParamProblem, icmp)
833         self.assert_equal(icmp[ICMPv6ParamProblem].code, 0, "ICMP code")
834
835     def test_invalid_packet_size(self):
836         """ total packet size > 65535 """
837         p = (Ether(dst=self.src_if.local_mac, src=self.src_if.remote_mac) /
838              IPv6(src=self.src_if.remote_ip6,
839                   dst=self.src_if.local_ip6) /
840              UDP(sport=1234, dport=5678) /
841              Raw())
842         self.extend_packet(p, 1000, self.padding)
843         fragments = fragment_rfc8200(p, 1, 500)
844         bad_fragment = fragments[1]
845         bad_fragment[IPv6ExtHdrFragment].offset = 65500
846         self.pg_enable_capture()
847         self.src_if.add_stream([bad_fragment])
848         self.pg_start()
849         pkts = self.src_if.get_capture(expected_count=1)
850         icmp = pkts[0]
851         self.assertIn(ICMPv6ParamProblem, icmp)
852         self.assert_equal(icmp[ICMPv6ParamProblem].code, 0, "ICMP code")
853
854
855 class TestIPv4ReassemblyLocalNode(VppTestCase):
856     """ IPv4 Reassembly for packets coming to ip4-local node """
857
858     @classmethod
859     def setUpClass(cls):
860         super(TestIPv4ReassemblyLocalNode, cls).setUpClass()
861
862         cls.create_pg_interfaces([0])
863         cls.src_dst_if = cls.pg0
864
865         # setup all interfaces
866         for i in cls.pg_interfaces:
867             i.admin_up()
868             i.config_ip4()
869             i.resolve_arp()
870
871         cls.padding = " abcdefghijklmn"
872         cls.create_stream()
873         cls.create_fragments()
874
875     def setUp(self):
876         """ Test setup - force timeout on existing reassemblies """
877         super(TestIPv4ReassemblyLocalNode, self).setUp()
878         self.vapi.ip_reassembly_set(timeout_ms=0, max_reassemblies=1000,
879                                     expire_walk_interval_ms=10)
880         self.sleep(.25)
881         self.vapi.ip_reassembly_set(timeout_ms=1000000, max_reassemblies=1000,
882                                     expire_walk_interval_ms=10000)
883
884     def tearDown(self):
885         super(TestIPv4ReassemblyLocalNode, self).tearDown()
886         self.logger.debug(self.vapi.ppcli("show ip4-reassembly details"))
887         self.logger.debug(self.vapi.ppcli("show buffers"))
888
889     @classmethod
890     def create_stream(cls, packet_count=test_packet_count):
891         """Create input packet stream for defined interface.
892
893         :param list packet_sizes: Required packet sizes.
894         """
895         for i in range(0, packet_count):
896             info = cls.create_packet_info(cls.src_dst_if, cls.src_dst_if)
897             payload = cls.info_to_payload(info)
898             p = (Ether(dst=cls.src_dst_if.local_mac,
899                        src=cls.src_dst_if.remote_mac) /
900                  IP(id=info.index, src=cls.src_dst_if.remote_ip4,
901                     dst=cls.src_dst_if.local_ip4) /
902                  ICMP(type='echo-request', id=1234) /
903                  Raw(payload))
904             cls.extend_packet(p, 1518, cls.padding)
905             info.data = p
906
907     @classmethod
908     def create_fragments(cls):
909         infos = cls._packet_infos
910         cls.pkt_infos = []
911         for index, info in six.iteritems(infos):
912             p = info.data
913             # cls.logger.debug(ppp("Packet:", p.__class__(str(p))))
914             fragments_300 = fragment_rfc791(p, 300)
915             cls.pkt_infos.append((index, fragments_300))
916         cls.fragments_300 = [x for (_, frags) in cls.pkt_infos for x in frags]
917         cls.logger.debug("Fragmented %s packets into %s 300-byte fragments" %
918                          (len(infos), len(cls.fragments_300)))
919
920     def verify_capture(self, capture):
921         """Verify captured packet stream.
922
923         :param list capture: Captured packet stream.
924         """
925         info = None
926         seen = set()
927         for packet in capture:
928             try:
929                 self.logger.debug(ppp("Got packet:", packet))
930                 ip = packet[IP]
931                 icmp = packet[ICMP]
932                 payload_info = self.payload_to_info(str(packet[Raw]))
933                 packet_index = payload_info.index
934                 if packet_index in seen:
935                     raise Exception(ppp("Duplicate packet received", packet))
936                 seen.add(packet_index)
937                 self.assertEqual(payload_info.dst, self.src_dst_if.sw_if_index)
938                 info = self._packet_infos[packet_index]
939                 self.assertIsNotNone(info)
940                 self.assertEqual(packet_index, info.index)
941                 saved_packet = info.data
942                 self.assertEqual(ip.src, saved_packet[IP].dst)
943                 self.assertEqual(ip.dst, saved_packet[IP].src)
944                 self.assertEqual(icmp.type, 0)  # echo reply
945                 self.assertEqual(icmp.id, saved_packet[ICMP].id)
946                 self.assertEqual(icmp.payload, saved_packet[ICMP].payload)
947             except Exception:
948                 self.logger.error(ppp("Unexpected or invalid packet:", packet))
949                 raise
950         for index in self._packet_infos:
951             self.assertIn(index, seen,
952                           "Packet with packet_index %d not received" % index)
953
954     def test_reassembly(self):
955         """ basic reassembly """
956
957         self.pg_enable_capture()
958         self.src_dst_if.add_stream(self.fragments_300)
959         self.pg_start()
960
961         packets = self.src_dst_if.get_capture(len(self.pkt_infos))
962         self.verify_capture(packets)
963
964         # run it all again to verify correctness
965         self.pg_enable_capture()
966         self.src_dst_if.add_stream(self.fragments_300)
967         self.pg_start()
968
969         packets = self.src_dst_if.get_capture(len(self.pkt_infos))
970         self.verify_capture(packets)
971
972
973 class TestFIFReassembly(VppTestCase):
974     """ Fragments in fragments reassembly """
975
976     @classmethod
977     def setUpClass(cls):
978         super(TestFIFReassembly, cls).setUpClass()
979
980         cls.create_pg_interfaces([0, 1])
981         cls.src_if = cls.pg0
982         cls.dst_if = cls.pg1
983         for i in cls.pg_interfaces:
984             i.admin_up()
985             i.config_ip4()
986             i.resolve_arp()
987             i.config_ip6()
988             i.resolve_ndp()
989
990         cls.packet_sizes = [64, 512, 1518, 9018]
991         cls.padding = " abcdefghijklmn"
992
993     def setUp(self):
994         """ Test setup - force timeout on existing reassemblies """
995         super(TestFIFReassembly, self).setUp()
996         self.vapi.ip_reassembly_enable_disable(
997             sw_if_index=self.src_if.sw_if_index, enable_ip4=True,
998             enable_ip6=True)
999         self.vapi.ip_reassembly_enable_disable(
1000             sw_if_index=self.dst_if.sw_if_index, enable_ip4=True,
1001             enable_ip6=True)
1002         self.vapi.ip_reassembly_set(timeout_ms=0, max_reassemblies=1000,
1003                                     expire_walk_interval_ms=10)
1004         self.vapi.ip_reassembly_set(timeout_ms=0, max_reassemblies=1000,
1005                                     expire_walk_interval_ms=10, is_ip6=1)
1006         self.sleep(.25)
1007         self.vapi.ip_reassembly_set(timeout_ms=1000000, max_reassemblies=1000,
1008                                     expire_walk_interval_ms=10000)
1009         self.vapi.ip_reassembly_set(timeout_ms=1000000, max_reassemblies=1000,
1010                                     expire_walk_interval_ms=10000, is_ip6=1)
1011
1012     def tearDown(self):
1013         self.logger.debug(self.vapi.ppcli("show ip4-reassembly details"))
1014         self.logger.debug(self.vapi.ppcli("show ip6-reassembly details"))
1015         self.logger.debug(self.vapi.ppcli("show buffers"))
1016         super(TestFIFReassembly, self).tearDown()
1017
1018     def verify_capture(self, capture, ip_class, dropped_packet_indexes=[]):
1019         """Verify captured packet stream.
1020
1021         :param list capture: Captured packet stream.
1022         """
1023         info = None
1024         seen = set()
1025         for packet in capture:
1026             try:
1027                 self.logger.debug(ppp("Got packet:", packet))
1028                 ip = packet[ip_class]
1029                 udp = packet[UDP]
1030                 payload_info = self.payload_to_info(str(packet[Raw]))
1031                 packet_index = payload_info.index
1032                 self.assertTrue(
1033                     packet_index not in dropped_packet_indexes,
1034                     ppp("Packet received, but should be dropped:", packet))
1035                 if packet_index in seen:
1036                     raise Exception(ppp("Duplicate packet received", packet))
1037                 seen.add(packet_index)
1038                 self.assertEqual(payload_info.dst, self.dst_if.sw_if_index)
1039                 info = self._packet_infos[packet_index]
1040                 self.assertTrue(info is not None)
1041                 self.assertEqual(packet_index, info.index)
1042                 saved_packet = info.data
1043                 self.assertEqual(ip.src, saved_packet[ip_class].src)
1044                 self.assertEqual(ip.dst, saved_packet[ip_class].dst)
1045                 self.assertEqual(udp.payload, saved_packet[UDP].payload)
1046             except Exception:
1047                 self.logger.error(ppp("Unexpected or invalid packet:", packet))
1048                 raise
1049         for index in self._packet_infos:
1050             self.assertTrue(index in seen or index in dropped_packet_indexes,
1051                             "Packet with packet_index %d not received" % index)
1052
1053     def test_fif4(self):
1054         """ Fragments in fragments (4o4) """
1055
1056         # TODO this should be ideally in setUpClass, but then we hit a bug
1057         # with VppIpRoute incorrectly reporting it's present when it's not
1058         # so we need to manually remove the vpp config, thus we cannot have
1059         # it shared for multiple test cases
1060         self.tun_ip4 = "1.1.1.2"
1061
1062         self.gre4 = VppGreInterface(self, self.src_if.local_ip4, self.tun_ip4)
1063         self.gre4.add_vpp_config()
1064         self.gre4.admin_up()
1065         self.gre4.config_ip4()
1066
1067         self.vapi.ip_reassembly_enable_disable(
1068             sw_if_index=self.gre4.sw_if_index, enable_ip4=True)
1069
1070         self.route4 = VppIpRoute(self, self.tun_ip4, 32,
1071                                  [VppRoutePath(self.src_if.remote_ip4,
1072                                                self.src_if.sw_if_index)])
1073         self.route4.add_vpp_config()
1074
1075         self.reset_packet_infos()
1076         for i in range(test_packet_count):
1077             info = self.create_packet_info(self.src_if, self.dst_if)
1078             payload = self.info_to_payload(info)
1079             # Ethernet header here is only for size calculation, thus it
1080             # doesn't matter how it's initialized. This is to ensure that
1081             # reassembled packet is not > 9000 bytes, so that it's not dropped
1082             p = (Ether() /
1083                  IP(id=i, src=self.src_if.remote_ip4,
1084                     dst=self.dst_if.remote_ip4) /
1085                  UDP(sport=1234, dport=5678) /
1086                  Raw(payload))
1087             size = self.packet_sizes[(i // 2) % len(self.packet_sizes)]
1088             self.extend_packet(p, size, self.padding)
1089             info.data = p[IP]  # use only IP part, without ethernet header
1090
1091         fragments = [x for _, p in six.iteritems(self._packet_infos)
1092                      for x in fragment_rfc791(p.data, 400)]
1093
1094         encapped_fragments = \
1095             [Ether(dst=self.src_if.local_mac, src=self.src_if.remote_mac) /
1096              IP(src=self.tun_ip4, dst=self.src_if.local_ip4) /
1097                 GRE() /
1098                 p
1099                 for p in fragments]
1100
1101         fragmented_encapped_fragments = \
1102             [x for p in encapped_fragments
1103              for x in fragment_rfc791(p, 200)]
1104
1105         self.src_if.add_stream(fragmented_encapped_fragments)
1106
1107         self.pg_enable_capture(self.pg_interfaces)
1108         self.pg_start()
1109
1110         self.src_if.assert_nothing_captured()
1111         packets = self.dst_if.get_capture(len(self._packet_infos))
1112         self.verify_capture(packets, IP)
1113
1114         # TODO remove gre vpp config by hand until VppIpRoute gets fixed
1115         # so that it's query_vpp_config() works as it should
1116         self.gre4.remove_vpp_config()
1117         self.logger.debug(self.vapi.ppcli("show interface"))
1118
1119     def test_fif6(self):
1120         """ Fragments in fragments (6o6) """
1121         # TODO this should be ideally in setUpClass, but then we hit a bug
1122         # with VppIpRoute incorrectly reporting it's present when it's not
1123         # so we need to manually remove the vpp config, thus we cannot have
1124         # it shared for multiple test cases
1125         self.tun_ip6 = "1002::1"
1126
1127         self.gre6 = VppGre6Interface(self, self.src_if.local_ip6, self.tun_ip6)
1128         self.gre6.add_vpp_config()
1129         self.gre6.admin_up()
1130         self.gre6.config_ip6()
1131
1132         self.vapi.ip_reassembly_enable_disable(
1133             sw_if_index=self.gre6.sw_if_index, enable_ip6=True)
1134
1135         self.route6 = VppIpRoute(self, self.tun_ip6, 128,
1136                                  [VppRoutePath(self.src_if.remote_ip6,
1137                                                self.src_if.sw_if_index,
1138                                                proto=DpoProto.DPO_PROTO_IP6)],
1139                                  is_ip6=1)
1140         self.route6.add_vpp_config()
1141
1142         self.reset_packet_infos()
1143         for i in range(test_packet_count):
1144             info = self.create_packet_info(self.src_if, self.dst_if)
1145             payload = self.info_to_payload(info)
1146             # Ethernet header here is only for size calculation, thus it
1147             # doesn't matter how it's initialized. This is to ensure that
1148             # reassembled packet is not > 9000 bytes, so that it's not dropped
1149             p = (Ether() /
1150                  IPv6(src=self.src_if.remote_ip6, dst=self.dst_if.remote_ip6) /
1151                  UDP(sport=1234, dport=5678) /
1152                  Raw(payload))
1153             size = self.packet_sizes[(i // 2) % len(self.packet_sizes)]
1154             self.extend_packet(p, size, self.padding)
1155             info.data = p[IPv6]  # use only IPv6 part, without ethernet header
1156
1157         fragments = [x for _, i in six.iteritems(self._packet_infos)
1158                      for x in fragment_rfc8200(
1159                          i.data, i.index, 400)]
1160
1161         encapped_fragments = \
1162             [Ether(dst=self.src_if.local_mac, src=self.src_if.remote_mac) /
1163              IPv6(src=self.tun_ip6, dst=self.src_if.local_ip6) /
1164                 GRE() /
1165                 p
1166                 for p in fragments]
1167
1168         fragmented_encapped_fragments = \
1169             [x for p in encapped_fragments for x in (
1170                 fragment_rfc8200(
1171                     p,
1172                     2 * len(self._packet_infos) + p[IPv6ExtHdrFragment].id,
1173                     200)
1174                 if IPv6ExtHdrFragment in p else [p]
1175             )
1176             ]
1177
1178         self.src_if.add_stream(fragmented_encapped_fragments)
1179
1180         self.pg_enable_capture(self.pg_interfaces)
1181         self.pg_start()
1182
1183         self.src_if.assert_nothing_captured()
1184         packets = self.dst_if.get_capture(len(self._packet_infos))
1185         self.verify_capture(packets, IPv6)
1186
1187         # TODO remove gre vpp config by hand until VppIpRoute gets fixed
1188         # so that it's query_vpp_config() works as it should
1189         self.gre6.remove_vpp_config()
1190
1191
1192 if __name__ == '__main__':
1193     unittest.main(testRunner=VppTestRunner)