make test: create virtualenv under /test/
[vpp.git] / test / ipfix.py
1 #!/usr/bin/env python
2 # IPFIX support for Scapy (RFC7011)
3
4 from scapy.all import *
5
6
7 # IPFIX Information Elements http://www.iana.org/assignments/ipfix/ipfix.xhtml
8 information_elements = {
9     1:   "octetDeltaCount",
10     2:   "packetDeltaCount",
11     3:   "deltaFlowCount",
12     4:   "protocolIdentifier",
13     5:   "ipClassOfService",
14     6:   "tcpControlBits",
15     7:   "sourceTransportPort",
16     8:   "sourceIPv4Address",
17     9:   "sourceIPv4PrefixLength",
18     10:  "ingressInterface",
19     11:  "destinationTransportPort",
20     12:  "destinationIPv4Address",
21     13:  "destinationIPv4PrefixLength",
22     14:  "egressInterface",
23     15:  "ipNextHopIPv4Address",
24     16:  "bgpSourceAsNumber",
25     17:  "bgpDestinationAsNumber",
26     18:  "bgpNextHopIPv4Address",
27     19:  "postMCastPacketDeltaCount",
28     20:  "postMCastOctetDeltaCount",
29     21:  "flowEndSysUpTime",
30     22:  "flowStartSysUpTime",
31     23:  "postOctetDeltaCount",
32     24:  "postPacketDeltaCount",
33     25:  "minimumIpTotalLength",
34     26:  "maximumIpTotalLength",
35     27:  "sourceIPv6Address",
36     28:  "destinationIPv6Address",
37     29:  "sourceIPv6PrefixLength",
38     30:  "destinationIPv6PrefixLength",
39     31:  "flowLabelIPv6",
40     32:  "icmpTypeCodeIPv4",
41     33:  "igmpType",
42     34:  "samplingInterval",
43     35:  "samplingAlgorithm",
44     36:  "flowActiveTimeout",
45     37:  "flowIdleTimeout",
46     38:  "engineType",
47     39:  "engineId",
48     40:  "exportedOctetTotalCount",
49     41:  "exportedMessageTotalCount",
50     42:  "exportedFlowRecordTotalCount",
51     43:  "ipv4RouterSc",
52     44:  "sourceIPv4Prefix",
53     45:  "destinationIPv4Prefix",
54     46:  "mplsTopLabelType",
55     47:  "mplsTopLabelIPv4Address",
56     48:  "samplerId",
57     49:  "samplerMode",
58     50:  "samplerRandomInterval",
59     51:  "classId",
60     52:  "minimumTTL",
61     53:  "maximumTTL",
62     54:  "fragmentIdentification",
63     55:  "postIpClassOfService",
64     56:  "sourceMacAddress",
65     57:  "postDestinationMacAddress",
66     58:  "vlanId",
67     59:  "postVlanId",
68     60:  "ipVersion",
69     61:  "flowDirection",
70     62:  "ipNextHopIPv6Address",
71     63:  "bgpNextHopIPv6Address",
72     64:  "ipv6ExtensionHeaders",
73     70:  "mplsTopLabelStackSection",
74     71:  "mplsLabelStackSection2",
75     72:  "mplsLabelStackSection3",
76     73:  "mplsLabelStackSection4",
77     74:  "mplsLabelStackSection5",
78     75:  "mplsLabelStackSection6",
79     76:  "mplsLabelStackSection7",
80     77:  "mplsLabelStackSection8",
81     78:  "mplsLabelStackSection9",
82     79:  "mplsLabelStackSection10",
83     80:  "destinationMacAddress",
84     81:  "postSourceMacAddress",
85     82:  "interfaceName",
86     83:  "interfaceDescription",
87     84:  "samplerName",
88     85:  "octetTotalCount",
89     86:  "packetTotalCount",
90     87:  "flagsAndSamplerId",
91     88:  "fragmentOffset",
92     89:  "forwardingStatus",
93     90:  "mplsVpnRouteDistinguisher",
94     91:  "mplsTopLabelPrefixLength",
95     92:  "srcTrafficIndex",
96     93:  "dstTrafficIndex",
97     94:  "applicationDescription",
98     95:  "applicationId",
99     96:  "applicationName",
100     98:  "postIpDiffServCodePoint",
101     99:  "multicastReplicationFactor",
102     100: "className",
103     101: "classificationEngineId",
104     102: "layer2packetSectionOffset",
105     103: "layer2packetSectionSize",
106     104: "layer2packetSectionData",
107     128: "bgpNextAdjacentAsNumber",
108     129: "bgpPrevAdjacentAsNumber",
109     130: "exporterIPv4Address",
110     131: "exporterIPv6Address",
111     132: "droppedOctetDeltaCount",
112     133: "droppedPacketDeltaCount",
113     134: "droppedOctetTotalCount",
114     135: "droppedPacketTotalCount",
115     136: "flowEndReason",
116     137: "commonPropertiesId",
117     138: "observationPointId",
118     139: "icmpTypeCodeIPv6",
119     140: "mplsTopLabelIPv6Address",
120     141: "lineCardId",
121     142: "portId",
122     143: "meteringProcessId",
123     144: "exportingProcessId",
124     145: "templateId",
125     146: "wlanChannelId",
126     147: "wlanSSID",
127     148: "flowId",
128     149: "observationDomainId",
129     150: "flowStartSeconds",
130     151: "flowEndSeconds",
131     152: "flowStartMilliseconds",
132     153: "flowEndMilliseconds",
133     154: "flowStartMicroseconds",
134     155: "flowEndMicroseconds",
135     156: "flowStartNanoseconds",
136     157: "flowEndNanoseconds",
137     158: "flowStartDeltaMicroseconds",
138     159: "flowEndDeltaMicroseconds",
139     160: "systemInitTimeMilliseconds",
140     161: "flowDurationMilliseconds",
141     162: "flowDurationMicroseconds",
142     163: "observedFlowTotalCount",
143     164: "ignoredPacketTotalCount",
144     165: "ignoredOctetTotalCount",
145     166: "notSentFlowTotalCount",
146     167: "notSentPacketTotalCount",
147     168: "notSentOctetTotalCount",
148     169: "destinationIPv6Prefix",
149     170: "sourceIPv6Prefix",
150     171: "postOctetTotalCount",
151     172: "postPacketTotalCount",
152     173: "flowKeyIndicator",
153     174: "postMCastPacketTotalCount",
154     175: "postMCastOctetTotalCount",
155     176: "icmpTypeIPv4",
156     177: "icmpCodeIPv4",
157     178: "icmpTypeIPv6",
158     179: "icmpCodeIPv6",
159     180: "udpSourcePort",
160     181: "udpDestinationPort",
161     182: "tcpSourcePort",
162     183: "tcpDestinationPort",
163     184: "tcpSequenceNumber",
164     185: "tcpAcknowledgementNumber",
165     186: "tcpWindowSize",
166     187: "tcpUrgentPointer",
167     188: "tcpHeaderLength",
168     189: "ipHeaderLength",
169     190: "totalLengthIPv4",
170     191: "payloadLengthIPv6",
171     192: "ipTTL",
172     193: "nextHeaderIPv6",
173     194: "mplsPayloadLength",
174     195: "ipDiffServCodePoint",
175     196: "ipPrecedence",
176     197: "fragmentFlags",
177     198: "octetDeltaSumOfSquares",
178     199: "octetTotalSumOfSquares",
179     200: "mplsTopLabelTTL",
180     201: "mplsLabelStackLength",
181     202: "mplsLabelStackDepth",
182     203: "mplsTopLabelExp",
183     204: "ipPayloadLength",
184     205: "udpMessageLength",
185     206: "isMulticast",
186     207: "ipv4IHL",
187     208: "ipv4Options",
188     209: "tcpOptions",
189     210: "paddingOctets",
190     211: "collectorIPv4Address",
191     212: "collectorIPv6Address",
192     213: "exportInterface",
193     214: "exportProtocolVersion",
194     215: "exportTransportProtocol",
195     216: "collectorTransportPort",
196     217: "exporterTransportPort",
197     218: "tcpSynTotalCount",
198     219: "tcpFinTotalCount",
199     220: "tcpRstTotalCount",
200     221: "tcpPshTotalCount",
201     222: "tcpAckTotalCount",
202     223: "tcpUrgTotalCount",
203     224: "ipTotalLength",
204     225: "postNATSourceIPv4Address",
205     226: "postNATDestinationIPv4Address",
206     227: "postNAPTSourceTransportPort",
207     228: "postNAPTDestinationTransportPort",
208     229: "natOriginatingAddressRealm",
209     230: "natEvent",
210     231: "initiatorOctets",
211     232: "responderOctets",
212     233: "firewallEvent",
213     234: "ingressVRFID",
214     235: "egressVRFID",
215     236: "VRFname",
216     237: "postMplsTopLabelExp",
217     238: "tcpWindowScale",
218     239: "biflowDirection",
219     240: "ethernetHeaderLength",
220     241: "ethernetPayloadLength",
221     242: "ethernetTotalLength",
222     243: "dot1qVlanId",
223     244: "dot1qPriority",
224     245: "dot1qCustomerVlanId",
225     246: "dot1qCustomerPriority",
226     247: "metroEvcId",
227     248: "metroEvcType",
228     249: "pseudoWireId",
229     250: "pseudoWireType",
230     251: "pseudoWireControlWord",
231     252: "ingressPhysicalInterface",
232     253: "egressPhysicalInterface",
233     254: "postDot1qVlanId",
234     255: "postDot1qCustomerVlanId",
235     256: "ethernetType",
236     257: "postIpPrecedence",
237     258: "collectionTimeMilliseconds",
238     259: "exportSctpStreamId",
239     260: "maxExportSeconds",
240     261: "maxFlowEndSeconds",
241     262: "messageMD5Checksum",
242     263: "messageScope",
243     264: "minExportSeconds",
244     265: "minFlowStartSeconds",
245     266: "opaqueOctets",
246     267: "sessionScope",
247     268: "maxFlowEndMicroseconds",
248     269: "maxFlowEndMilliseconds",
249     270: "maxFlowEndNanoseconds",
250     271: "minFlowStartMicroseconds",
251     272: "minFlowStartMilliseconds",
252     273: "minFlowStartNanoseconds",
253     274: "collectorCertificate",
254     275: "exporterCertificate",
255     276: "dataRecordsReliability",
256     277: "observationPointType",
257     278: "newConnectionDeltaCount",
258     279: "connectionSumDurationSeconds",
259     280: "connectionTransactionId",
260     281: "postNATSourceIPv6Address",
261     282: "postNATDestinationIPv6Address",
262     283: "natPoolId",
263     284: "natPoolName",
264     285: "anonymizationFlags",
265     286: "anonymizationTechnique",
266     287: "informationElementIndex",
267     288: "p2pTechnology",
268     289: "tunnelTechnology",
269     290: "encryptedTechnology",
270     291: "basicList",
271     292: "subTemplateList",
272     293: "subTemplateMultiList",
273     294: "bgpValidityState",
274     295: "IPSecSPI",
275     296: "greKey",
276     297: "natType",
277     298: "initiatorPackets",
278     299: "responderPackets",
279     300: "observationDomainName",
280     301: "selectionSequenceId",
281     302: "selectorId",
282     303: "informationElementId",
283     304: "selectorAlgorithm",
284     305: "samplingPacketInterval",
285     306: "samplingPacketSpace",
286     307: "samplingTimeInterval",
287     308: "samplingTimeSpace",
288     309: "samplingSize",
289     310: "samplingPopulation",
290     311: "samplingProbability",
291     312: "dataLinkFrameSize",
292     313: "ipHeaderPacketSection",
293     314: "ipPayloadPacketSection",
294     315: "dataLinkFrameSection",
295     316: "mplsLabelStackSection",
296     317: "mplsPayloadPacketSection",
297     318: "selectorIdTotalPktsObserved",
298     319: "selectorIdTotalPktsSelected",
299     320: "absoluteError",
300     321: "relativeError",
301     322: "observationTimeSeconds",
302     323: "observationTimeMilliseconds",
303     324: "observationTimeMicroseconds",
304     325: "observationTimeNanoseconds",
305     326: "digestHashValue",
306     327: "hashIPPayloadOffset",
307     328: "hashIPPayloadSize",
308     329: "hashOutputRangeMin",
309     330: "hashOutputRangeMax",
310     331: "hashSelectedRangeMin",
311     332: "hashSelectedRangeMax",
312     333: "hashDigestOutput",
313     334: "hashInitialiserValue",
314     335: "selectorName",
315     336: "upperCILimit",
316     337: "lowerCILimit",
317     338: "confidenceLevel",
318     339: "informationElementDataType",
319     340: "informationElementDescription",
320     341: "informationElementName",
321     342: "informationElementRangeBegin",
322     343: "informationElementRangeEnd",
323     344: "informationElementSemantics",
324     345: "informationElementUnits",
325     346: "privateEnterpriseNumber",
326     347: "virtualStationInterfaceId",
327     348: "virtualStationInterfaceName",
328     349: "virtualStationUUID",
329     350: "virtualStationName",
330     351: "layer2SegmentId",
331     352: "layer2OctetDeltaCount",
332     353: "layer2OctetTotalCount",
333     354: "ingressUnicastPacketTotalCount",
334     355: "ingressMulticastPacketTotalCount",
335     356: "ingressBroadcastPacketTotalCount",
336     357: "egressUnicastPacketTotalCount",
337     358: "egressBroadcastPacketTotalCount",
338     359: "monitoringIntervalStartMilliSeconds",
339     360: "monitoringIntervalEndMilliSeconds",
340     361: "portRangeStart",
341     362: "portRangeEnd",
342     363: "portRangeStepSize",
343     364: "portRangeNumPorts",
344     365: "staMacAddress",
345     366: "staIPv4Address",
346     367: "wtpMacAddress",
347     368: "ingressInterfaceType",
348     369: "egressInterfaceType",
349     370: "rtpSequenceNumber",
350     371: "userName",
351     372: "applicationCategoryName",
352     373: "applicationSubCategoryName",
353     374: "applicationGroupName",
354     375: "originalFlowsPresent",
355     376: "originalFlowsInitiated",
356     377: "originalFlowsCompleted",
357     378: "distinctCountOfSourceIPAddress",
358     379: "distinctCountOfDestinationIPAddress",
359     380: "distinctCountOfSourceIPv4Address",
360     381: "distinctCountOfDestinationIPv4Address",
361     382: "distinctCountOfSourceIPv6Address",
362     383: "distinctCountOfDestinationIPv6Address",
363     384: "valueDistributionMethod",
364     385: "rfc3550JitterMilliseconds",
365     386: "rfc3550JitterMicroseconds",
366     387: "rfc3550JitterNanoseconds",
367     388: "dot1qDEI",
368     389: "dot1qCustomerDEI",
369     390: "flowSelectorAlgorithm",
370     391: "flowSelectedOctetDeltaCount",
371     392: "flowSelectedPacketDeltaCount",
372     393: "flowSelectedFlowDeltaCount",
373     394: "selectorIDTotalFlowsObserved",
374     395: "selectorIDTotalFlowsSelected",
375     396: "samplingFlowInterval",
376     397: "samplingFlowSpacing",
377     398: "flowSamplingTimeInterval",
378     399: "flowSamplingTimeSpacing",
379     400: "hashFlowDomain",
380     401: "transportOctetDeltaCount",
381     402: "transportPacketDeltaCount",
382     403: "originalExporterIPv4Address",
383     404: "originalExporterIPv6Address",
384     405: "originalObservationDomainId",
385     406: "intermediateProcessId",
386     407: "ignoredDataRecordTotalCount",
387     408: "dataLinkFrameType",
388     409: "sectionOffset",
389     410: "sectionExportedOctets",
390     411: "dot1qServiceInstanceTag",
391     412: "dot1qServiceInstanceId",
392     413: "dot1qServiceInstancePriority",
393     414: "dot1qCustomerSourceMacAddress",
394     415: "dot1qCustomerDestinationMacAddress",
395     417: "postLayer2OctetDeltaCount",
396     418: "postMCastLayer2OctetDeltaCount",
397     420: "postLayer2OctetTotalCount",
398     421: "postMCastLayer2OctetTotalCount",
399     422: "minimumLayer2TotalLength",
400     423: "maximumLayer2TotalLength",
401     424: "droppedLayer2OctetDeltaCount",
402     425: "droppedLayer2OctetTotalCount",
403     426: "ignoredLayer2OctetTotalCount",
404     427: "notSentLayer2OctetTotalCount",
405     428: "layer2OctetDeltaSumOfSquares",
406     429: "layer2OctetTotalSumOfSquares",
407     430: "layer2FrameDeltaCount",
408     431: "layer2FrameTotalCount",
409     432: "pseudoWireDestinationIPv4Address",
410     433: "ignoredLayer2FrameTotalCount",
411     434: "mibObjectValueInteger",
412     435: "mibObjectValueOctetString",
413     436: "mibObjectValueOID",
414     437: "mibObjectValueBits",
415     438: "mibObjectValueIPAddress",
416     439: "mibObjectValueCounter",
417     440: "mibObjectValueGauge",
418     441: "mibObjectValueTimeTicks",
419     442: "mibObjectValueUnsigned",
420     443: "mibObjectValueTable",
421     444: "mibObjectValueRow",
422     445: "mibObjectIdentifier",
423     446: "mibSubIdentifier",
424     447: "mibIndexIndicator",
425     448: "mibCaptureTimeSemantics",
426     449: "mibContextEngineID",
427     450: "mibContextName",
428     451: "mibObjectName",
429     452: "mibObjectDescription",
430     453: "mibObjectSyntax",
431     454: "mibModuleName",
432     455: "mobileIMSI",
433     456: "mobileMSISDN",
434     457: "httpStatusCode",
435     458: "sourceTransportPortsLimit",
436     459: "httpRequestMethod",
437     460: "httpRequestHost",
438     461: "httpRequestTarget",
439     462: "httpMessageVersion",
440     466: "natQuotaExceededEvent",
441     471: "maxSessionEntries",
442     472: "maxBIBEntries",
443     473: "maxEntriesPerUser",
444     475: "maxFragmentsPendingReassembly"
445 }
446
447
448 class IPFIX(Packet):
449     name = "IPFIX"
450     fields_desc = [ShortField("version", 10),
451                    ShortField("length", None),
452                    IntField("exportTime", None),
453                    IntField("sequenceNumber", 1),
454                    IntField("observationDomainID", 1)]
455
456
457 class FieldSpecifier(Packet):
458     name = "Field Specifier"
459     fields_desc = [ShortEnumField(
460         "informationElement", None, information_elements),
461         ShortField("fieldLength", None)]
462
463     def extract_padding(self, s):
464         return "", s
465
466
467 class Template(Packet):
468     name = "Template"
469     fields_desc = [ShortField("templateID", 256),
470                    FieldLenField("fieldCount", None, count_of="fields"),
471                    PacketListField("templateFields", [], FieldSpecifier,
472                                    count_from=lambda p: p.fieldCount)]
473
474
475 class Data(Packet):
476     name = "Data"
477     fields_desc = [
478         StrLenField("data", "", length_from=lambda p: p.underlayer.length - 4)]
479
480     def extract_padding(self, s):
481         return "", s
482
483
484 class Set(Packet):
485     name = "Set"
486     fields_desc = [ShortField("setID", 256),
487                    ShortField("length", None)]
488
489     def guess_payload_class(self, payload):
490         if self.setID == 2:
491             return Template
492         elif self.setID > 255:
493             return Data
494         else:
495             return Packet.guess_payload_class(self, payload)
496
497
498 bind_layers(IPFIX, Set)
499 bind_layers(UDP, IPFIX, dport=4739)
500
501
502 class IPFIXDecoder(object):
503     """ IPFIX data set decoder """
504
505     def __init__(self):
506         self._templates = []
507
508     def add_template(self, template):
509         """
510         Add IPFIX tempalte
511
512         :param template: IPFIX template
513         """
514         templateID = template.templateID
515         fields = []
516         rec_len = 0
517         for field in template.templateFields:
518             fields.append(
519                 {'name': field.informationElement, 'len': field.fieldLength})
520             rec_len += field.fieldLength
521         self._templates.append(
522             {'id': templateID, 'fields': fields, 'rec_len': rec_len})
523
524     def decode_data_set(self, data_set):
525         """
526         Decode IPFIX data
527
528         :param data_set: IPFIX data set
529         :returns: List of decoded data records.
530         """
531         data = []
532         for template in self._templates:
533             if template['id'] == data_set.setID:
534                 offset = 0
535                 d = data_set[Data].data
536                 for i in range(len(d) / template['rec_len']):
537                     record = {}
538                     for field in template['fields']:
539                         f = d[offset:offset + field['len']]
540                         offset += field['len']
541                         record.update({field['name']: f})
542                     data.append(record)
543                 break
544         return data