test/ipfix.py

   1 #!/usr/bin/env python
   2 # IPFIX support for Scapy (RFC7011)
   3
   4 from scapy.all import *
   5
   6
   7 # IPFIX Information Elements http://www.iana.org/assignments/ipfix/ipfix.xhtml
   8 information_elements = {
   9     1:   "octetDeltaCount",
  10     2:   "packetDeltaCount",
  11     3:   "deltaFlowCount",
  12     4:   "protocolIdentifier",
  13     5:   "ipClassOfService",
  14     6:   "tcpControlBits",
  15     7:   "sourceTransportPort",
  16     8:   "sourceIPv4Address",
  17     9:   "sourceIPv4PrefixLength",
  18     10:  "ingressInterface",
  19     11:  "destinationTransportPort",
  20     12:  "destinationIPv4Address",
  21     13:  "destinationIPv4PrefixLength",
  22     14:  "egressInterface",
  23     15:  "ipNextHopIPv4Address",
  24     16:  "bgpSourceAsNumber",
  25     17:  "bgpDestinationAsNumber",
  26     18:  "bgpNextHopIPv4Address",
  27     19:  "postMCastPacketDeltaCount",
  28     20:  "postMCastOctetDeltaCount",
  29     21:  "flowEndSysUpTime",
  30     22:  "flowStartSysUpTime",
  31     23:  "postOctetDeltaCount",
  32     24:  "postPacketDeltaCount",
  33     25:  "minimumIpTotalLength",
  34     26:  "maximumIpTotalLength",
  35     27:  "sourceIPv6Address",
  36     28:  "destinationIPv6Address",
  37     29:  "sourceIPv6PrefixLength",
  38     30:  "destinationIPv6PrefixLength",
  39     31:  "flowLabelIPv6",
  40     32:  "icmpTypeCodeIPv4",
  41     33:  "igmpType",
  42     34:  "samplingInterval",
  43     35:  "samplingAlgorithm",
  44     36:  "flowActiveTimeout",
  45     37:  "flowIdleTimeout",
  46     38:  "engineType",
  47     39:  "engineId",
  48     40:  "exportedOctetTotalCount",
  49     41:  "exportedMessageTotalCount",
  50     42:  "exportedFlowRecordTotalCount",
  51     43:  "ipv4RouterSc",
  52     44:  "sourceIPv4Prefix",
  53     45:  "destinationIPv4Prefix",
  54     46:  "mplsTopLabelType",
  55     47:  "mplsTopLabelIPv4Address",
  56     48:  "samplerId",
  57     49:  "samplerMode",
  58     50:  "samplerRandomInterval",
  59     51:  "classId",
  60     52:  "minimumTTL",
  61     53:  "maximumTTL",
  62     54:  "fragmentIdentification",
  63     55:  "postIpClassOfService",
  64     56:  "sourceMacAddress",
  65     57:  "postDestinationMacAddress",
  66     58:  "vlanId",
  67     59:  "postVlanId",
  68     60:  "ipVersion",
  69     61:  "flowDirection",
  70     62:  "ipNextHopIPv6Address",
  71     63:  "bgpNextHopIPv6Address",
  72     64:  "ipv6ExtensionHeaders",
  73     70:  "mplsTopLabelStackSection",
  74     71:  "mplsLabelStackSection2",
  75     72:  "mplsLabelStackSection3",
  76     73:  "mplsLabelStackSection4",
  77     74:  "mplsLabelStackSection5",
  78     75:  "mplsLabelStackSection6",
  79     76:  "mplsLabelStackSection7",
  80     77:  "mplsLabelStackSection8",
  81     78:  "mplsLabelStackSection9",
  82     79:  "mplsLabelStackSection10",
  83     80:  "destinationMacAddress",
  84     81:  "postSourceMacAddress",
  85     82:  "interfaceName",
  86     83:  "interfaceDescription",
  87     84:  "samplerName",
  88     85:  "octetTotalCount",
  89     86:  "packetTotalCount",
  90     87:  "flagsAndSamplerId",
  91     88:  "fragmentOffset",
  92     89:  "forwardingStatus",
  93     90:  "mplsVpnRouteDistinguisher",
  94     91:  "mplsTopLabelPrefixLength",
  95     92:  "srcTrafficIndex",
  96     93:  "dstTrafficIndex",
  97     94:  "applicationDescription",
  98     95:  "applicationId",
  99     96:  "applicationName",
 100     98:  "postIpDiffServCodePoint",
 101     99:  "multicastReplicationFactor",
 102     100: "className",
 103     101: "classificationEngineId",
 104     102: "layer2packetSectionOffset",
 105     103: "layer2packetSectionSize",
 106     104: "layer2packetSectionData",
 107     128: "bgpNextAdjacentAsNumber",
 108     129: "bgpPrevAdjacentAsNumber",
 109     130: "exporterIPv4Address",
 110     131: "exporterIPv6Address",
 111     132: "droppedOctetDeltaCount",
 112     133: "droppedPacketDeltaCount",
 113     134: "droppedOctetTotalCount",
 114     135: "droppedPacketTotalCount",
 115     136: "flowEndReason",
 116     137: "commonPropertiesId",
 117     138: "observationPointId",
 118     139: "icmpTypeCodeIPv6",
 119     140: "mplsTopLabelIPv6Address",
 120     141: "lineCardId",
 121     142: "portId",
 122     143: "meteringProcessId",
 123     144: "exportingProcessId",
 124     145: "templateId",
 125     146: "wlanChannelId",
 126     147: "wlanSSID",
 127     148: "flowId",
 128     149: "observationDomainId",
 129     150: "flowStartSeconds",
 130     151: "flowEndSeconds",
 131     152: "flowStartMilliseconds",
 132     153: "flowEndMilliseconds",
 133     154: "flowStartMicroseconds",
 134     155: "flowEndMicroseconds",
 135     156: "flowStartNanoseconds",
 136     157: "flowEndNanoseconds",
 137     158: "flowStartDeltaMicroseconds",
 138     159: "flowEndDeltaMicroseconds",
 139     160: "systemInitTimeMilliseconds",
 140     161: "flowDurationMilliseconds",
 141     162: "flowDurationMicroseconds",
 142     163: "observedFlowTotalCount",
 143     164: "ignoredPacketTotalCount",
 144     165: "ignoredOctetTotalCount",
 145     166: "notSentFlowTotalCount",
 146     167: "notSentPacketTotalCount",
 147     168: "notSentOctetTotalCount",
 148     169: "destinationIPv6Prefix",
 149     170: "sourceIPv6Prefix",
 150     171: "postOctetTotalCount",
 151     172: "postPacketTotalCount",
 152     173: "flowKeyIndicator",
 153     174: "postMCastPacketTotalCount",
 154     175: "postMCastOctetTotalCount",
 155     176: "icmpTypeIPv4",
 156     177: "icmpCodeIPv4",
 157     178: "icmpTypeIPv6",
 158     179: "icmpCodeIPv6",
 159     180: "udpSourcePort",
 160     181: "udpDestinationPort",
 161     182: "tcpSourcePort",
 162     183: "tcpDestinationPort",
 163     184: "tcpSequenceNumber",
 164     185: "tcpAcknowledgementNumber",
 165     186: "tcpWindowSize",
 166     187: "tcpUrgentPointer",
 167     188: "tcpHeaderLength",
 168     189: "ipHeaderLength",
 169     190: "totalLengthIPv4",
 170     191: "payloadLengthIPv6",
 171     192: "ipTTL",
 172     193: "nextHeaderIPv6",
 173     194: "mplsPayloadLength",
 174     195: "ipDiffServCodePoint",
 175     196: "ipPrecedence",
 176     197: "fragmentFlags",
 177     198: "octetDeltaSumOfSquares",
 178     199: "octetTotalSumOfSquares",
 179     200: "mplsTopLabelTTL",
 180     201: "mplsLabelStackLength",
 181     202: "mplsLabelStackDepth",
 182     203: "mplsTopLabelExp",
 183     204: "ipPayloadLength",
 184     205: "udpMessageLength",
 185     206: "isMulticast",
 186     207: "ipv4IHL",
 187     208: "ipv4Options",
 188     209: "tcpOptions",
 189     210: "paddingOctets",
 190     211: "collectorIPv4Address",
 191     212: "collectorIPv6Address",
 192     213: "exportInterface",
 193     214: "exportProtocolVersion",
 194     215: "exportTransportProtocol",
 195     216: "collectorTransportPort",
 196     217: "exporterTransportPort",
 197     218: "tcpSynTotalCount",
 198     219: "tcpFinTotalCount",
 199     220: "tcpRstTotalCount",
 200     221: "tcpPshTotalCount",
 201     222: "tcpAckTotalCount",
 202     223: "tcpUrgTotalCount",
 203     224: "ipTotalLength",
 204     225: "postNATSourceIPv4Address",
 205     226: "postNATDestinationIPv4Address",
 206     227: "postNAPTSourceTransportPort",
 207     228: "postNAPTDestinationTransportPort",
 208     229: "natOriginatingAddressRealm",
 209     230: "natEvent",
 210     231: "initiatorOctets",
 211     232: "responderOctets",
 212     233: "firewallEvent",
 213     234: "ingressVRFID",
 214     235: "egressVRFID",
 215     236: "VRFname",
 216     237: "postMplsTopLabelExp",
 217     238: "tcpWindowScale",
 218     239: "biflowDirection",
 219     240: "ethernetHeaderLength",
 220     241: "ethernetPayloadLength",
 221     242: "ethernetTotalLength",
 222     243: "dot1qVlanId",
 223     244: "dot1qPriority",
 224     245: "dot1qCustomerVlanId",
 225     246: "dot1qCustomerPriority",
 226     247: "metroEvcId",
 227     248: "metroEvcType",
 228     249: "pseudoWireId",
 229     250: "pseudoWireType",
 230     251: "pseudoWireControlWord",
 231     252: "ingressPhysicalInterface",
 232     253: "egressPhysicalInterface",
 233     254: "postDot1qVlanId",
 234     255: "postDot1qCustomerVlanId",
 235     256: "ethernetType",
 236     257: "postIpPrecedence",
 237     258: "collectionTimeMilliseconds",
 238     259: "exportSctpStreamId",
 239     260: "maxExportSeconds",
 240     261: "maxFlowEndSeconds",
 241     262: "messageMD5Checksum",
 242     263: "messageScope",
 243     264: "minExportSeconds",
 244     265: "minFlowStartSeconds",
 245     266: "opaqueOctets",
 246     267: "sessionScope",
 247     268: "maxFlowEndMicroseconds",
 248     269: "maxFlowEndMilliseconds",
 249     270: "maxFlowEndNanoseconds",
 250     271: "minFlowStartMicroseconds",
 251     272: "minFlowStartMilliseconds",
 252     273: "minFlowStartNanoseconds",
 253     274: "collectorCertificate",
 254     275: "exporterCertificate",
 255     276: "dataRecordsReliability",
 256     277: "observationPointType",
 257     278: "newConnectionDeltaCount",
 258     279: "connectionSumDurationSeconds",
 259     280: "connectionTransactionId",
 260     281: "postNATSourceIPv6Address",
 261     282: "postNATDestinationIPv6Address",
 262     283: "natPoolId",
 263     284: "natPoolName",
 264     285: "anonymizationFlags",
 265     286: "anonymizationTechnique",
 266     287: "informationElementIndex",
 267     288: "p2pTechnology",
 268     289: "tunnelTechnology",
 269     290: "encryptedTechnology",
 270     291: "basicList",
 271     292: "subTemplateList",
 272     293: "subTemplateMultiList",
 273     294: "bgpValidityState",
 274     295: "IPSecSPI",
 275     296: "greKey",
 276     297: "natType",
 277     298: "initiatorPackets",
 278     299: "responderPackets",
 279     300: "observationDomainName",
 280     301: "selectionSequenceId",
 281     302: "selectorId",
 282     303: "informationElementId",
 283     304: "selectorAlgorithm",
 284     305: "samplingPacketInterval",
 285     306: "samplingPacketSpace",
 286     307: "samplingTimeInterval",
 287     308: "samplingTimeSpace",
 288     309: "samplingSize",
 289     310: "samplingPopulation",
 290     311: "samplingProbability",
 291     312: "dataLinkFrameSize",
 292     313: "ipHeaderPacketSection",
 293     314: "ipPayloadPacketSection",
 294     315: "dataLinkFrameSection",
 295     316: "mplsLabelStackSection",
 296     317: "mplsPayloadPacketSection",
 297     318: "selectorIdTotalPktsObserved",
 298     319: "selectorIdTotalPktsSelected",
 299     320: "absoluteError",
 300     321: "relativeError",
 301     322: "observationTimeSeconds",
 302     323: "observationTimeMilliseconds",
 303     324: "observationTimeMicroseconds",
 304     325: "observationTimeNanoseconds",
 305     326: "digestHashValue",
 306     327: "hashIPPayloadOffset",
 307     328: "hashIPPayloadSize",
 308     329: "hashOutputRangeMin",
 309     330: "hashOutputRangeMax",
 310     331: "hashSelectedRangeMin",
 311     332: "hashSelectedRangeMax",
 312     333: "hashDigestOutput",
 313     334: "hashInitialiserValue",
 314     335: "selectorName",
 315     336: "upperCILimit",
 316     337: "lowerCILimit",
 317     338: "confidenceLevel",
 318     339: "informationElementDataType",
 319     340: "informationElementDescription",
 320     341: "informationElementName",
 321     342: "informationElementRangeBegin",
 322     343: "informationElementRangeEnd",
 323     344: "informationElementSemantics",
 324     345: "informationElementUnits",
 325     346: "privateEnterpriseNumber",
 326     347: "virtualStationInterfaceId",
 327     348: "virtualStationInterfaceName",
 328     349: "virtualStationUUID",
 329     350: "virtualStationName",
 330     351: "layer2SegmentId",
 331     352: "layer2OctetDeltaCount",
 332     353: "layer2OctetTotalCount",
 333     354: "ingressUnicastPacketTotalCount",
 334     355: "ingressMulticastPacketTotalCount",
 335     356: "ingressBroadcastPacketTotalCount",
 336     357: "egressUnicastPacketTotalCount",
 337     358: "egressBroadcastPacketTotalCount",
 338     359: "monitoringIntervalStartMilliSeconds",
 339     360: "monitoringIntervalEndMilliSeconds",
 340     361: "portRangeStart",
 341     362: "portRangeEnd",
 342     363: "portRangeStepSize",
 343     364: "portRangeNumPorts",
 344     365: "staMacAddress",
 345     366: "staIPv4Address",
 346     367: "wtpMacAddress",
 347     368: "ingressInterfaceType",
 348     369: "egressInterfaceType",
 349     370: "rtpSequenceNumber",
 350     371: "userName",
 351     372: "applicationCategoryName",
 352     373: "applicationSubCategoryName",
 353     374: "applicationGroupName",
 354     375: "originalFlowsPresent",
 355     376: "originalFlowsInitiated",
 356     377: "originalFlowsCompleted",
 357     378: "distinctCountOfSourceIPAddress",
 358     379: "distinctCountOfDestinationIPAddress",
 359     380: "distinctCountOfSourceIPv4Address",
 360     381: "distinctCountOfDestinationIPv4Address",
 361     382: "distinctCountOfSourceIPv6Address",
 362     383: "distinctCountOfDestinationIPv6Address",
 363     384: "valueDistributionMethod",
 364     385: "rfc3550JitterMilliseconds",
 365     386: "rfc3550JitterMicroseconds",
 366     387: "rfc3550JitterNanoseconds",
 367     388: "dot1qDEI",
 368     389: "dot1qCustomerDEI",
 369     390: "flowSelectorAlgorithm",
 370     391: "flowSelectedOctetDeltaCount",
 371     392: "flowSelectedPacketDeltaCount",
 372     393: "flowSelectedFlowDeltaCount",
 373     394: "selectorIDTotalFlowsObserved",
 374     395: "selectorIDTotalFlowsSelected",
 375     396: "samplingFlowInterval",
 376     397: "samplingFlowSpacing",
 377     398: "flowSamplingTimeInterval",
 378     399: "flowSamplingTimeSpacing",
 379     400: "hashFlowDomain",
 380     401: "transportOctetDeltaCount",
 381     402: "transportPacketDeltaCount",
 382     403: "originalExporterIPv4Address",
 383     404: "originalExporterIPv6Address",
 384     405: "originalObservationDomainId",
 385     406: "intermediateProcessId",
 386     407: "ignoredDataRecordTotalCount",
 387     408: "dataLinkFrameType",
 388     409: "sectionOffset",
 389     410: "sectionExportedOctets",
 390     411: "dot1qServiceInstanceTag",
 391     412: "dot1qServiceInstanceId",
 392     413: "dot1qServiceInstancePriority",
 393     414: "dot1qCustomerSourceMacAddress",
 394     415: "dot1qCustomerDestinationMacAddress",
 395     417: "postLayer2OctetDeltaCount",
 396     418: "postMCastLayer2OctetDeltaCount",
 397     420: "postLayer2OctetTotalCount",
 398     421: "postMCastLayer2OctetTotalCount",
 399     422: "minimumLayer2TotalLength",
 400     423: "maximumLayer2TotalLength",
 401     424: "droppedLayer2OctetDeltaCount",
 402     425: "droppedLayer2OctetTotalCount",
 403     426: "ignoredLayer2OctetTotalCount",
 404     427: "notSentLayer2OctetTotalCount",
 405     428: "layer2OctetDeltaSumOfSquares",
 406     429: "layer2OctetTotalSumOfSquares",
 407     430: "layer2FrameDeltaCount",
 408     431: "layer2FrameTotalCount",
 409     432: "pseudoWireDestinationIPv4Address",
 410     433: "ignoredLayer2FrameTotalCount",
 411     434: "mibObjectValueInteger",
 412     435: "mibObjectValueOctetString",
 413     436: "mibObjectValueOID",
 414     437: "mibObjectValueBits",
 415     438: "mibObjectValueIPAddress",
 416     439: "mibObjectValueCounter",
 417     440: "mibObjectValueGauge",
 418     441: "mibObjectValueTimeTicks",
 419     442: "mibObjectValueUnsigned",
 420     443: "mibObjectValueTable",
 421     444: "mibObjectValueRow",
 422     445: "mibObjectIdentifier",
 423     446: "mibSubIdentifier",
 424     447: "mibIndexIndicator",
 425     448: "mibCaptureTimeSemantics",
 426     449: "mibContextEngineID",
 427     450: "mibContextName",
 428     451: "mibObjectName",
 429     452: "mibObjectDescription",
 430     453: "mibObjectSyntax",
 431     454: "mibModuleName",
 432     455: "mobileIMSI",
 433     456: "mobileMSISDN",
 434     457: "httpStatusCode",
 435     458: "sourceTransportPortsLimit",
 436     459: "httpRequestMethod",
 437     460: "httpRequestHost",
 438     461: "httpRequestTarget",
 439     462: "httpMessageVersion",
 440     466: "natQuotaExceededEvent",
 441     471: "maxSessionEntries",
 442     472: "maxBIBEntries",
 443     473: "maxEntriesPerUser",
 444     475: "maxFragmentsPendingReassembly"
 445 }
 446
 447
 448 class IPFIX(Packet):
 449     name = "IPFIX"
 450     fields_desc = [ShortField("version", 10),
 451                    ShortField("length", None),
 452                    IntField("exportTime", None),
 453                    IntField("sequenceNumber", 1),
 454                    IntField("observationDomainID", 1)]
 455
 456
 457 class FieldSpecifier(Packet):
 458     name = "Field Specifier"
 459     fields_desc = [ShortEnumField(
 460         "informationElement", None, information_elements),
 461         ShortField("fieldLength", None)]
 462
 463     def extract_padding(self, s):
 464         return "", s
 465
 466
 467 class Template(Packet):
 468     name = "Template"
 469     fields_desc = [ShortField("templateID", 256),
 470                    FieldLenField("fieldCount", None, count_of="fields"),
 471                    PacketListField("templateFields", [], FieldSpecifier,
 472                                    count_from=lambda p: p.fieldCount)]
 473
 474
 475 class Data(Packet):
 476     name = "Data"
 477     fields_desc = [
 478         StrLenField("data", "", length_from=lambda p: p.underlayer.length - 4)]
 479
 480     def extract_padding(self, s):
 481         return "", s
 482
 483
 484 class Set(Packet):
 485     name = "Set"
 486     fields_desc = [ShortField("setID", 256),
 487                    ShortField("length", None)]
 488
 489     def guess_payload_class(self, payload):
 490         if self.setID == 2:
 491             return Template
 492         elif self.setID > 255:
 493             return Data
 494         else:
 495             return Packet.guess_payload_class(self, payload)
 496
 497
 498 bind_layers(IPFIX, Set)
 499 bind_layers(UDP, IPFIX, dport=4739)
 500
 501
 502 class IPFIXDecoder(object):
 503     """ IPFIX data set decoder """
 504
 505     def __init__(self):
 506         self._templates = []
 507
 508     def add_template(self, template):
 509         """
 510         Add IPFIX tempalte
 511
 512         :param template: IPFIX template
 513         """
 514         templateID = template.templateID
 515         fields = []
 516         rec_len = 0
 517         for field in template.templateFields:
 518             fields.append(
 519                 {'name': field.informationElement, 'len': field.fieldLength})
 520             rec_len += field.fieldLength
 521         self._templates.append(
 522             {'id': templateID, 'fields': fields, 'rec_len': rec_len})
 523
 524     def decode_data_set(self, data_set):
 525         """
 526         Decode IPFIX data
 527
 528         :param data_set: IPFIX data set
 529         :returns: List of decoded data records.
 530         """
 531         data = []
 532         for template in self._templates:
 533             if template['id'] == data_set.setID:
 534                 offset = 0
 535                 d = data_set[Data].data
 536                 for i in range(len(d) / template['rec_len']):
 537                     record = {}
 538                     for field in template['fields']:
 539                         f = d[offset:offset + field['len']]
 540                         offset += field['len']
 541                         record.update({field['name']: f})
 542                     data.append(record)
 543                 break
 544         return data