test/ipfix.py

   1 #!/usr/bin/env python3
   2 # IPFIX support for Scapy (RFC7011)
   3
   4 from scapy.all import (
   5     bind_layers,
   6     FieldLenField,
   7     IntField,
   8     Packet,
   9     PacketListField,
  10     ShortEnumField,
  11     ShortField,
  12     StrLenField,
  13 )
  14 from scapy.layers.inet import UDP
  15
  16
  17 # IPFIX Information Elements http://www.iana.org/assignments/ipfix/ipfix.xhtml
  18 information_elements = {
  19     1: "octetDeltaCount",
  20     2: "packetDeltaCount",
  21     3: "deltaFlowCount",
  22     4: "protocolIdentifier",
  23     5: "ipClassOfService",
  24     6: "tcpControlBits",
  25     7: "sourceTransportPort",
  26     8: "sourceIPv4Address",
  27     9: "sourceIPv4PrefixLength",
  28     10: "ingressInterface",
  29     11: "destinationTransportPort",
  30     12: "destinationIPv4Address",
  31     13: "destinationIPv4PrefixLength",
  32     14: "egressInterface",
  33     15: "ipNextHopIPv4Address",
  34     16: "bgpSourceAsNumber",
  35     17: "bgpDestinationAsNumber",
  36     18: "bgpNextHopIPv4Address",
  37     19: "postMCastPacketDeltaCount",
  38     20: "postMCastOctetDeltaCount",
  39     21: "flowEndSysUpTime",
  40     22: "flowStartSysUpTime",
  41     23: "postOctetDeltaCount",
  42     24: "postPacketDeltaCount",
  43     25: "minimumIpTotalLength",
  44     26: "maximumIpTotalLength",
  45     27: "sourceIPv6Address",
  46     28: "destinationIPv6Address",
  47     29: "sourceIPv6PrefixLength",
  48     30: "destinationIPv6PrefixLength",
  49     31: "flowLabelIPv6",
  50     32: "icmpTypeCodeIPv4",
  51     33: "igmpType",
  52     34: "samplingInterval",
  53     35: "samplingAlgorithm",
  54     36: "flowActiveTimeout",
  55     37: "flowIdleTimeout",
  56     38: "engineType",
  57     39: "engineId",
  58     40: "exportedOctetTotalCount",
  59     41: "exportedMessageTotalCount",
  60     42: "exportedFlowRecordTotalCount",
  61     43: "ipv4RouterSc",
  62     44: "sourceIPv4Prefix",
  63     45: "destinationIPv4Prefix",
  64     46: "mplsTopLabelType",
  65     47: "mplsTopLabelIPv4Address",
  66     48: "samplerId",
  67     49: "samplerMode",
  68     50: "samplerRandomInterval",
  69     51: "classId",
  70     52: "minimumTTL",
  71     53: "maximumTTL",
  72     54: "fragmentIdentification",
  73     55: "postIpClassOfService",
  74     56: "sourceMacAddress",
  75     57: "postDestinationMacAddress",
  76     58: "vlanId",
  77     59: "postVlanId",
  78     60: "ipVersion",
  79     61: "flowDirection",
  80     62: "ipNextHopIPv6Address",
  81     63: "bgpNextHopIPv6Address",
  82     64: "ipv6ExtensionHeaders",
  83     70: "mplsTopLabelStackSection",
  84     71: "mplsLabelStackSection2",
  85     72: "mplsLabelStackSection3",
  86     73: "mplsLabelStackSection4",
  87     74: "mplsLabelStackSection5",
  88     75: "mplsLabelStackSection6",
  89     76: "mplsLabelStackSection7",
  90     77: "mplsLabelStackSection8",
  91     78: "mplsLabelStackSection9",
  92     79: "mplsLabelStackSection10",
  93     80: "destinationMacAddress",
  94     81: "postSourceMacAddress",
  95     82: "interfaceName",
  96     83: "interfaceDescription",
  97     84: "samplerName",
  98     85: "octetTotalCount",
  99     86: "packetTotalCount",
 100     87: "flagsAndSamplerId",
 101     88: "fragmentOffset",
 102     89: "forwardingStatus",
 103     90: "mplsVpnRouteDistinguisher",
 104     91: "mplsTopLabelPrefixLength",
 105     92: "srcTrafficIndex",
 106     93: "dstTrafficIndex",
 107     94: "applicationDescription",
 108     95: "applicationId",
 109     96: "applicationName",
 110     98: "postIpDiffServCodePoint",
 111     99: "multicastReplicationFactor",
 112     100: "className",
 113     101: "classificationEngineId",
 114     102: "layer2packetSectionOffset",
 115     103: "layer2packetSectionSize",
 116     104: "layer2packetSectionData",
 117     128: "bgpNextAdjacentAsNumber",
 118     129: "bgpPrevAdjacentAsNumber",
 119     130: "exporterIPv4Address",
 120     131: "exporterIPv6Address",
 121     132: "droppedOctetDeltaCount",
 122     133: "droppedPacketDeltaCount",
 123     134: "droppedOctetTotalCount",
 124     135: "droppedPacketTotalCount",
 125     136: "flowEndReason",
 126     137: "commonPropertiesId",
 127     138: "observationPointId",
 128     139: "icmpTypeCodeIPv6",
 129     140: "mplsTopLabelIPv6Address",
 130     141: "lineCardId",
 131     142: "portId",
 132     143: "meteringProcessId",
 133     144: "exportingProcessId",
 134     145: "templateId",
 135     146: "wlanChannelId",
 136     147: "wlanSSID",
 137     148: "flowId",
 138     149: "observationDomainId",
 139     150: "flowStartSeconds",
 140     151: "flowEndSeconds",
 141     152: "flowStartMilliseconds",
 142     153: "flowEndMilliseconds",
 143     154: "flowStartMicroseconds",
 144     155: "flowEndMicroseconds",
 145     156: "flowStartNanoseconds",
 146     157: "flowEndNanoseconds",
 147     158: "flowStartDeltaMicroseconds",
 148     159: "flowEndDeltaMicroseconds",
 149     160: "systemInitTimeMilliseconds",
 150     161: "flowDurationMilliseconds",
 151     162: "flowDurationMicroseconds",
 152     163: "observedFlowTotalCount",
 153     164: "ignoredPacketTotalCount",
 154     165: "ignoredOctetTotalCount",
 155     166: "notSentFlowTotalCount",
 156     167: "notSentPacketTotalCount",
 157     168: "notSentOctetTotalCount",
 158     169: "destinationIPv6Prefix",
 159     170: "sourceIPv6Prefix",
 160     171: "postOctetTotalCount",
 161     172: "postPacketTotalCount",
 162     173: "flowKeyIndicator",
 163     174: "postMCastPacketTotalCount",
 164     175: "postMCastOctetTotalCount",
 165     176: "icmpTypeIPv4",
 166     177: "icmpCodeIPv4",
 167     178: "icmpTypeIPv6",
 168     179: "icmpCodeIPv6",
 169     180: "udpSourcePort",
 170     181: "udpDestinationPort",
 171     182: "tcpSourcePort",
 172     183: "tcpDestinationPort",
 173     184: "tcpSequenceNumber",
 174     185: "tcpAcknowledgementNumber",
 175     186: "tcpWindowSize",
 176     187: "tcpUrgentPointer",
 177     188: "tcpHeaderLength",
 178     189: "ipHeaderLength",
 179     190: "totalLengthIPv4",
 180     191: "payloadLengthIPv6",
 181     192: "ipTTL",
 182     193: "nextHeaderIPv6",
 183     194: "mplsPayloadLength",
 184     195: "ipDiffServCodePoint",
 185     196: "ipPrecedence",
 186     197: "fragmentFlags",
 187     198: "octetDeltaSumOfSquares",
 188     199: "octetTotalSumOfSquares",
 189     200: "mplsTopLabelTTL",
 190     201: "mplsLabelStackLength",
 191     202: "mplsLabelStackDepth",
 192     203: "mplsTopLabelExp",
 193     204: "ipPayloadLength",
 194     205: "udpMessageLength",
 195     206: "isMulticast",
 196     207: "ipv4IHL",
 197     208: "ipv4Options",
 198     209: "tcpOptions",
 199     210: "paddingOctets",
 200     211: "collectorIPv4Address",
 201     212: "collectorIPv6Address",
 202     213: "exportInterface",
 203     214: "exportProtocolVersion",
 204     215: "exportTransportProtocol",
 205     216: "collectorTransportPort",
 206     217: "exporterTransportPort",
 207     218: "tcpSynTotalCount",
 208     219: "tcpFinTotalCount",
 209     220: "tcpRstTotalCount",
 210     221: "tcpPshTotalCount",
 211     222: "tcpAckTotalCount",
 212     223: "tcpUrgTotalCount",
 213     224: "ipTotalLength",
 214     225: "postNATSourceIPv4Address",
 215     226: "postNATDestinationIPv4Address",
 216     227: "postNAPTSourceTransportPort",
 217     228: "postNAPTDestinationTransportPort",
 218     229: "natOriginatingAddressRealm",
 219     230: "natEvent",
 220     231: "initiatorOctets",
 221     232: "responderOctets",
 222     233: "firewallEvent",
 223     234: "ingressVRFID",
 224     235: "egressVRFID",
 225     236: "VRFname",
 226     237: "postMplsTopLabelExp",
 227     238: "tcpWindowScale",
 228     239: "biflowDirection",
 229     240: "ethernetHeaderLength",
 230     241: "ethernetPayloadLength",
 231     242: "ethernetTotalLength",
 232     243: "dot1qVlanId",
 233     244: "dot1qPriority",
 234     245: "dot1qCustomerVlanId",
 235     246: "dot1qCustomerPriority",
 236     247: "metroEvcId",
 237     248: "metroEvcType",
 238     249: "pseudoWireId",
 239     250: "pseudoWireType",
 240     251: "pseudoWireControlWord",
 241     252: "ingressPhysicalInterface",
 242     253: "egressPhysicalInterface",
 243     254: "postDot1qVlanId",
 244     255: "postDot1qCustomerVlanId",
 245     256: "ethernetType",
 246     257: "postIpPrecedence",
 247     258: "collectionTimeMilliseconds",
 248     259: "exportSctpStreamId",
 249     260: "maxExportSeconds",
 250     261: "maxFlowEndSeconds",
 251     262: "messageMD5Checksum",
 252     263: "messageScope",
 253     264: "minExportSeconds",
 254     265: "minFlowStartSeconds",
 255     266: "opaqueOctets",
 256     267: "sessionScope",
 257     268: "maxFlowEndMicroseconds",
 258     269: "maxFlowEndMilliseconds",
 259     270: "maxFlowEndNanoseconds",
 260     271: "minFlowStartMicroseconds",
 261     272: "minFlowStartMilliseconds",
 262     273: "minFlowStartNanoseconds",
 263     274: "collectorCertificate",
 264     275: "exporterCertificate",
 265     276: "dataRecordsReliability",
 266     277: "observationPointType",
 267     278: "newConnectionDeltaCount",
 268     279: "connectionSumDurationSeconds",
 269     280: "connectionTransactionId",
 270     281: "postNATSourceIPv6Address",
 271     282: "postNATDestinationIPv6Address",
 272     283: "natPoolId",
 273     284: "natPoolName",
 274     285: "anonymizationFlags",
 275     286: "anonymizationTechnique",
 276     287: "informationElementIndex",
 277     288: "p2pTechnology",
 278     289: "tunnelTechnology",
 279     290: "encryptedTechnology",
 280     291: "basicList",
 281     292: "subTemplateList",
 282     293: "subTemplateMultiList",
 283     294: "bgpValidityState",
 284     295: "IPSecSPI",
 285     296: "greKey",
 286     297: "natType",
 287     298: "initiatorPackets",
 288     299: "responderPackets",
 289     300: "observationDomainName",
 290     301: "selectionSequenceId",
 291     302: "selectorId",
 292     303: "informationElementId",
 293     304: "selectorAlgorithm",
 294     305: "samplingPacketInterval",
 295     306: "samplingPacketSpace",
 296     307: "samplingTimeInterval",
 297     308: "samplingTimeSpace",
 298     309: "samplingSize",
 299     310: "samplingPopulation",
 300     311: "samplingProbability",
 301     312: "dataLinkFrameSize",
 302     313: "ipHeaderPacketSection",
 303     314: "ipPayloadPacketSection",
 304     315: "dataLinkFrameSection",
 305     316: "mplsLabelStackSection",
 306     317: "mplsPayloadPacketSection",
 307     318: "selectorIdTotalPktsObserved",
 308     319: "selectorIdTotalPktsSelected",
 309     320: "absoluteError",
 310     321: "relativeError",
 311     322: "observationTimeSeconds",
 312     323: "observationTimeMilliseconds",
 313     324: "observationTimeMicroseconds",
 314     325: "observationTimeNanoseconds",
 315     326: "digestHashValue",
 316     327: "hashIPPayloadOffset",
 317     328: "hashIPPayloadSize",
 318     329: "hashOutputRangeMin",
 319     330: "hashOutputRangeMax",
 320     331: "hashSelectedRangeMin",
 321     332: "hashSelectedRangeMax",
 322     333: "hashDigestOutput",
 323     334: "hashInitialiserValue",
 324     335: "selectorName",
 325     336: "upperCILimit",
 326     337: "lowerCILimit",
 327     338: "confidenceLevel",
 328     339: "informationElementDataType",
 329     340: "informationElementDescription",
 330     341: "informationElementName",
 331     342: "informationElementRangeBegin",
 332     343: "informationElementRangeEnd",
 333     344: "informationElementSemantics",
 334     345: "informationElementUnits",
 335     346: "privateEnterpriseNumber",
 336     347: "virtualStationInterfaceId",
 337     348: "virtualStationInterfaceName",
 338     349: "virtualStationUUID",
 339     350: "virtualStationName",
 340     351: "layer2SegmentId",
 341     352: "layer2OctetDeltaCount",
 342     353: "layer2OctetTotalCount",
 343     354: "ingressUnicastPacketTotalCount",
 344     355: "ingressMulticastPacketTotalCount",
 345     356: "ingressBroadcastPacketTotalCount",
 346     357: "egressUnicastPacketTotalCount",
 347     358: "egressBroadcastPacketTotalCount",
 348     359: "monitoringIntervalStartMilliSeconds",
 349     360: "monitoringIntervalEndMilliSeconds",
 350     361: "portRangeStart",
 351     362: "portRangeEnd",
 352     363: "portRangeStepSize",
 353     364: "portRangeNumPorts",
 354     365: "staMacAddress",
 355     366: "staIPv4Address",
 356     367: "wtpMacAddress",
 357     368: "ingressInterfaceType",
 358     369: "egressInterfaceType",
 359     370: "rtpSequenceNumber",
 360     371: "userName",
 361     372: "applicationCategoryName",
 362     373: "applicationSubCategoryName",
 363     374: "applicationGroupName",
 364     375: "originalFlowsPresent",
 365     376: "originalFlowsInitiated",
 366     377: "originalFlowsCompleted",
 367     378: "distinctCountOfSourceIPAddress",
 368     379: "distinctCountOfDestinationIPAddress",
 369     380: "distinctCountOfSourceIPv4Address",
 370     381: "distinctCountOfDestinationIPv4Address",
 371     382: "distinctCountOfSourceIPv6Address",
 372     383: "distinctCountOfDestinationIPv6Address",
 373     384: "valueDistributionMethod",
 374     385: "rfc3550JitterMilliseconds",
 375     386: "rfc3550JitterMicroseconds",
 376     387: "rfc3550JitterNanoseconds",
 377     388: "dot1qDEI",
 378     389: "dot1qCustomerDEI",
 379     390: "flowSelectorAlgorithm",
 380     391: "flowSelectedOctetDeltaCount",
 381     392: "flowSelectedPacketDeltaCount",
 382     393: "flowSelectedFlowDeltaCount",
 383     394: "selectorIDTotalFlowsObserved",
 384     395: "selectorIDTotalFlowsSelected",
 385     396: "samplingFlowInterval",
 386     397: "samplingFlowSpacing",
 387     398: "flowSamplingTimeInterval",
 388     399: "flowSamplingTimeSpacing",
 389     400: "hashFlowDomain",
 390     401: "transportOctetDeltaCount",
 391     402: "transportPacketDeltaCount",
 392     403: "originalExporterIPv4Address",
 393     404: "originalExporterIPv6Address",
 394     405: "originalObservationDomainId",
 395     406: "intermediateProcessId",
 396     407: "ignoredDataRecordTotalCount",
 397     408: "dataLinkFrameType",
 398     409: "sectionOffset",
 399     410: "sectionExportedOctets",
 400     411: "dot1qServiceInstanceTag",
 401     412: "dot1qServiceInstanceId",
 402     413: "dot1qServiceInstancePriority",
 403     414: "dot1qCustomerSourceMacAddress",
 404     415: "dot1qCustomerDestinationMacAddress",
 405     417: "postLayer2OctetDeltaCount",
 406     418: "postMCastLayer2OctetDeltaCount",
 407     420: "postLayer2OctetTotalCount",
 408     421: "postMCastLayer2OctetTotalCount",
 409     422: "minimumLayer2TotalLength",
 410     423: "maximumLayer2TotalLength",
 411     424: "droppedLayer2OctetDeltaCount",
 412     425: "droppedLayer2OctetTotalCount",
 413     426: "ignoredLayer2OctetTotalCount",
 414     427: "notSentLayer2OctetTotalCount",
 415     428: "layer2OctetDeltaSumOfSquares",
 416     429: "layer2OctetTotalSumOfSquares",
 417     430: "layer2FrameDeltaCount",
 418     431: "layer2FrameTotalCount",
 419     432: "pseudoWireDestinationIPv4Address",
 420     433: "ignoredLayer2FrameTotalCount",
 421     434: "mibObjectValueInteger",
 422     435: "mibObjectValueOctetString",
 423     436: "mibObjectValueOID",
 424     437: "mibObjectValueBits",
 425     438: "mibObjectValueIPAddress",
 426     439: "mibObjectValueCounter",
 427     440: "mibObjectValueGauge",
 428     441: "mibObjectValueTimeTicks",
 429     442: "mibObjectValueUnsigned",
 430     443: "mibObjectValueTable",
 431     444: "mibObjectValueRow",
 432     445: "mibObjectIdentifier",
 433     446: "mibSubIdentifier",
 434     447: "mibIndexIndicator",
 435     448: "mibCaptureTimeSemantics",
 436     449: "mibContextEngineID",
 437     450: "mibContextName",
 438     451: "mibObjectName",
 439     452: "mibObjectDescription",
 440     453: "mibObjectSyntax",
 441     454: "mibModuleName",
 442     455: "mobileIMSI",
 443     456: "mobileMSISDN",
 444     457: "httpStatusCode",
 445     458: "sourceTransportPortsLimit",
 446     459: "httpRequestMethod",
 447     460: "httpRequestHost",
 448     461: "httpRequestTarget",
 449     462: "httpMessageVersion",
 450     466: "natQuotaExceededEvent",
 451     471: "maxSessionEntries",
 452     472: "maxBIBEntries",
 453     473: "maxEntriesPerUser",
 454     475: "maxFragmentsPendingReassembly",
 455 }
 456
 457
 458 class IPFIX(Packet):
 459     name = "IPFIX"
 460     fields_desc = [
 461         ShortField("version", 10),
 462         ShortField("length", None),
 463         IntField("exportTime", None),
 464         IntField("sequenceNumber", 1),
 465         IntField("observationDomainID", 1),
 466     ]
 467
 468
 469 class FieldSpecifier(Packet):
 470     name = "Field Specifier"
 471     fields_desc = [
 472         ShortEnumField("informationElement", None, information_elements),
 473         ShortField("fieldLength", None),
 474     ]
 475
 476     def extract_padding(self, s):
 477         return "", s
 478
 479
 480 class Template(Packet):
 481     name = "Template"
 482     fields_desc = [
 483         ShortField("templateID", 256),
 484         FieldLenField("fieldCount", None, count_of="fields"),
 485         PacketListField(
 486             "templateFields", [], FieldSpecifier, count_from=lambda p: p.fieldCount
 487         ),
 488     ]
 489
 490
 491 class Data(Packet):
 492     name = "Data"
 493     fields_desc = [
 494         StrLenField("data", "", length_from=lambda p: p.underlayer.length - 4)
 495     ]
 496
 497     def extract_padding(self, s):
 498         return "", s
 499
 500
 501 class Set(Packet):
 502     name = "Set"
 503     fields_desc = [ShortField("setID", 256), ShortField("length", None)]
 504
 505     def guess_payload_class(self, payload):
 506         if self.setID == 2:
 507             return Template
 508         elif self.setID > 255:
 509             return Data
 510         else:
 511             return Packet.guess_payload_class(self, payload)
 512
 513
 514 bind_layers(IPFIX, Set)
 515 bind_layers(UDP, IPFIX, dport=4739)
 516
 517
 518 class IPFIXDecoder:
 519     """IPFIX data set decoder"""
 520
 521     def __init__(self):
 522         self._templates = []
 523
 524     def add_template(self, template):
 525         """
 526         Add IPFIX template
 527
 528         :param template: IPFIX template
 529         """
 530         templateID = template.templateID
 531         fields = []
 532         rec_len = 0
 533         for field in template.templateFields:
 534             fields.append({"name": field.informationElement, "len": field.fieldLength})
 535             rec_len += field.fieldLength
 536         self._templates.append({"id": templateID, "fields": fields, "rec_len": rec_len})
 537
 538     def decode_data_set(self, data_set):
 539         """
 540         Decode IPFIX data
 541
 542         :param data_set: IPFIX data set
 543         :returns: List of decoded data records.
 544         """
 545         data = []
 546         for template in self._templates:
 547             if template["id"] == data_set.setID:
 548                 offset = 0
 549                 d = data_set[Data].data
 550                 for i in range(len(d) // template["rec_len"]):
 551                     record = {}
 552                     for field in template["fields"]:
 553                         f = d[offset : offset + field["len"]]
 554                         offset += field["len"]
 555                         record.update({field["name"]: f})
 556                     data.append(record)
 557                 break
 558         return data