test/ipfix.py

   1 #!/usr/bin/env python
   2 # IPFIX support for Scapy (RFC7011)
   3
   4 from scapy.all import bind_layers, FieldLenField, IntField, Packet, \
   5     PacketListField, ShortEnumField, ShortField, StrLenField
   6 from scapy.layers.inet import UDP
   7
   8
   9 # IPFIX Information Elements http://www.iana.org/assignments/ipfix/ipfix.xhtml
  10 information_elements = {
  11     1:   "octetDeltaCount",
  12     2:   "packetDeltaCount",
  13     3:   "deltaFlowCount",
  14     4:   "protocolIdentifier",
  15     5:   "ipClassOfService",
  16     6:   "tcpControlBits",
  17     7:   "sourceTransportPort",
  18     8:   "sourceIPv4Address",
  19     9:   "sourceIPv4PrefixLength",
  20     10:  "ingressInterface",
  21     11:  "destinationTransportPort",
  22     12:  "destinationIPv4Address",
  23     13:  "destinationIPv4PrefixLength",
  24     14:  "egressInterface",
  25     15:  "ipNextHopIPv4Address",
  26     16:  "bgpSourceAsNumber",
  27     17:  "bgpDestinationAsNumber",
  28     18:  "bgpNextHopIPv4Address",
  29     19:  "postMCastPacketDeltaCount",
  30     20:  "postMCastOctetDeltaCount",
  31     21:  "flowEndSysUpTime",
  32     22:  "flowStartSysUpTime",
  33     23:  "postOctetDeltaCount",
  34     24:  "postPacketDeltaCount",
  35     25:  "minimumIpTotalLength",
  36     26:  "maximumIpTotalLength",
  37     27:  "sourceIPv6Address",
  38     28:  "destinationIPv6Address",
  39     29:  "sourceIPv6PrefixLength",
  40     30:  "destinationIPv6PrefixLength",
  41     31:  "flowLabelIPv6",
  42     32:  "icmpTypeCodeIPv4",
  43     33:  "igmpType",
  44     34:  "samplingInterval",
  45     35:  "samplingAlgorithm",
  46     36:  "flowActiveTimeout",
  47     37:  "flowIdleTimeout",
  48     38:  "engineType",
  49     39:  "engineId",
  50     40:  "exportedOctetTotalCount",
  51     41:  "exportedMessageTotalCount",
  52     42:  "exportedFlowRecordTotalCount",
  53     43:  "ipv4RouterSc",
  54     44:  "sourceIPv4Prefix",
  55     45:  "destinationIPv4Prefix",
  56     46:  "mplsTopLabelType",
  57     47:  "mplsTopLabelIPv4Address",
  58     48:  "samplerId",
  59     49:  "samplerMode",
  60     50:  "samplerRandomInterval",
  61     51:  "classId",
  62     52:  "minimumTTL",
  63     53:  "maximumTTL",
  64     54:  "fragmentIdentification",
  65     55:  "postIpClassOfService",
  66     56:  "sourceMacAddress",
  67     57:  "postDestinationMacAddress",
  68     58:  "vlanId",
  69     59:  "postVlanId",
  70     60:  "ipVersion",
  71     61:  "flowDirection",
  72     62:  "ipNextHopIPv6Address",
  73     63:  "bgpNextHopIPv6Address",
  74     64:  "ipv6ExtensionHeaders",
  75     70:  "mplsTopLabelStackSection",
  76     71:  "mplsLabelStackSection2",
  77     72:  "mplsLabelStackSection3",
  78     73:  "mplsLabelStackSection4",
  79     74:  "mplsLabelStackSection5",
  80     75:  "mplsLabelStackSection6",
  81     76:  "mplsLabelStackSection7",
  82     77:  "mplsLabelStackSection8",
  83     78:  "mplsLabelStackSection9",
  84     79:  "mplsLabelStackSection10",
  85     80:  "destinationMacAddress",
  86     81:  "postSourceMacAddress",
  87     82:  "interfaceName",
  88     83:  "interfaceDescription",
  89     84:  "samplerName",
  90     85:  "octetTotalCount",
  91     86:  "packetTotalCount",
  92     87:  "flagsAndSamplerId",
  93     88:  "fragmentOffset",
  94     89:  "forwardingStatus",
  95     90:  "mplsVpnRouteDistinguisher",
  96     91:  "mplsTopLabelPrefixLength",
  97     92:  "srcTrafficIndex",
  98     93:  "dstTrafficIndex",
  99     94:  "applicationDescription",
 100     95:  "applicationId",
 101     96:  "applicationName",
 102     98:  "postIpDiffServCodePoint",
 103     99:  "multicastReplicationFactor",
 104     100: "className",
 105     101: "classificationEngineId",
 106     102: "layer2packetSectionOffset",
 107     103: "layer2packetSectionSize",
 108     104: "layer2packetSectionData",
 109     128: "bgpNextAdjacentAsNumber",
 110     129: "bgpPrevAdjacentAsNumber",
 111     130: "exporterIPv4Address",
 112     131: "exporterIPv6Address",
 113     132: "droppedOctetDeltaCount",
 114     133: "droppedPacketDeltaCount",
 115     134: "droppedOctetTotalCount",
 116     135: "droppedPacketTotalCount",
 117     136: "flowEndReason",
 118     137: "commonPropertiesId",
 119     138: "observationPointId",
 120     139: "icmpTypeCodeIPv6",
 121     140: "mplsTopLabelIPv6Address",
 122     141: "lineCardId",
 123     142: "portId",
 124     143: "meteringProcessId",
 125     144: "exportingProcessId",
 126     145: "templateId",
 127     146: "wlanChannelId",
 128     147: "wlanSSID",
 129     148: "flowId",
 130     149: "observationDomainId",
 131     150: "flowStartSeconds",
 132     151: "flowEndSeconds",
 133     152: "flowStartMilliseconds",
 134     153: "flowEndMilliseconds",
 135     154: "flowStartMicroseconds",
 136     155: "flowEndMicroseconds",
 137     156: "flowStartNanoseconds",
 138     157: "flowEndNanoseconds",
 139     158: "flowStartDeltaMicroseconds",
 140     159: "flowEndDeltaMicroseconds",
 141     160: "systemInitTimeMilliseconds",
 142     161: "flowDurationMilliseconds",
 143     162: "flowDurationMicroseconds",
 144     163: "observedFlowTotalCount",
 145     164: "ignoredPacketTotalCount",
 146     165: "ignoredOctetTotalCount",
 147     166: "notSentFlowTotalCount",
 148     167: "notSentPacketTotalCount",
 149     168: "notSentOctetTotalCount",
 150     169: "destinationIPv6Prefix",
 151     170: "sourceIPv6Prefix",
 152     171: "postOctetTotalCount",
 153     172: "postPacketTotalCount",
 154     173: "flowKeyIndicator",
 155     174: "postMCastPacketTotalCount",
 156     175: "postMCastOctetTotalCount",
 157     176: "icmpTypeIPv4",
 158     177: "icmpCodeIPv4",
 159     178: "icmpTypeIPv6",
 160     179: "icmpCodeIPv6",
 161     180: "udpSourcePort",
 162     181: "udpDestinationPort",
 163     182: "tcpSourcePort",
 164     183: "tcpDestinationPort",
 165     184: "tcpSequenceNumber",
 166     185: "tcpAcknowledgementNumber",
 167     186: "tcpWindowSize",
 168     187: "tcpUrgentPointer",
 169     188: "tcpHeaderLength",
 170     189: "ipHeaderLength",
 171     190: "totalLengthIPv4",
 172     191: "payloadLengthIPv6",
 173     192: "ipTTL",
 174     193: "nextHeaderIPv6",
 175     194: "mplsPayloadLength",
 176     195: "ipDiffServCodePoint",
 177     196: "ipPrecedence",
 178     197: "fragmentFlags",
 179     198: "octetDeltaSumOfSquares",
 180     199: "octetTotalSumOfSquares",
 181     200: "mplsTopLabelTTL",
 182     201: "mplsLabelStackLength",
 183     202: "mplsLabelStackDepth",
 184     203: "mplsTopLabelExp",
 185     204: "ipPayloadLength",
 186     205: "udpMessageLength",
 187     206: "isMulticast",
 188     207: "ipv4IHL",
 189     208: "ipv4Options",
 190     209: "tcpOptions",
 191     210: "paddingOctets",
 192     211: "collectorIPv4Address",
 193     212: "collectorIPv6Address",
 194     213: "exportInterface",
 195     214: "exportProtocolVersion",
 196     215: "exportTransportProtocol",
 197     216: "collectorTransportPort",
 198     217: "exporterTransportPort",
 199     218: "tcpSynTotalCount",
 200     219: "tcpFinTotalCount",
 201     220: "tcpRstTotalCount",
 202     221: "tcpPshTotalCount",
 203     222: "tcpAckTotalCount",
 204     223: "tcpUrgTotalCount",
 205     224: "ipTotalLength",
 206     225: "postNATSourceIPv4Address",
 207     226: "postNATDestinationIPv4Address",
 208     227: "postNAPTSourceTransportPort",
 209     228: "postNAPTDestinationTransportPort",
 210     229: "natOriginatingAddressRealm",
 211     230: "natEvent",
 212     231: "initiatorOctets",
 213     232: "responderOctets",
 214     233: "firewallEvent",
 215     234: "ingressVRFID",
 216     235: "egressVRFID",
 217     236: "VRFname",
 218     237: "postMplsTopLabelExp",
 219     238: "tcpWindowScale",
 220     239: "biflowDirection",
 221     240: "ethernetHeaderLength",
 222     241: "ethernetPayloadLength",
 223     242: "ethernetTotalLength",
 224     243: "dot1qVlanId",
 225     244: "dot1qPriority",
 226     245: "dot1qCustomerVlanId",
 227     246: "dot1qCustomerPriority",
 228     247: "metroEvcId",
 229     248: "metroEvcType",
 230     249: "pseudoWireId",
 231     250: "pseudoWireType",
 232     251: "pseudoWireControlWord",
 233     252: "ingressPhysicalInterface",
 234     253: "egressPhysicalInterface",
 235     254: "postDot1qVlanId",
 236     255: "postDot1qCustomerVlanId",
 237     256: "ethernetType",
 238     257: "postIpPrecedence",
 239     258: "collectionTimeMilliseconds",
 240     259: "exportSctpStreamId",
 241     260: "maxExportSeconds",
 242     261: "maxFlowEndSeconds",
 243     262: "messageMD5Checksum",
 244     263: "messageScope",
 245     264: "minExportSeconds",
 246     265: "minFlowStartSeconds",
 247     266: "opaqueOctets",
 248     267: "sessionScope",
 249     268: "maxFlowEndMicroseconds",
 250     269: "maxFlowEndMilliseconds",
 251     270: "maxFlowEndNanoseconds",
 252     271: "minFlowStartMicroseconds",
 253     272: "minFlowStartMilliseconds",
 254     273: "minFlowStartNanoseconds",
 255     274: "collectorCertificate",
 256     275: "exporterCertificate",
 257     276: "dataRecordsReliability",
 258     277: "observationPointType",
 259     278: "newConnectionDeltaCount",
 260     279: "connectionSumDurationSeconds",
 261     280: "connectionTransactionId",
 262     281: "postNATSourceIPv6Address",
 263     282: "postNATDestinationIPv6Address",
 264     283: "natPoolId",
 265     284: "natPoolName",
 266     285: "anonymizationFlags",
 267     286: "anonymizationTechnique",
 268     287: "informationElementIndex",
 269     288: "p2pTechnology",
 270     289: "tunnelTechnology",
 271     290: "encryptedTechnology",
 272     291: "basicList",
 273     292: "subTemplateList",
 274     293: "subTemplateMultiList",
 275     294: "bgpValidityState",
 276     295: "IPSecSPI",
 277     296: "greKey",
 278     297: "natType",
 279     298: "initiatorPackets",
 280     299: "responderPackets",
 281     300: "observationDomainName",
 282     301: "selectionSequenceId",
 283     302: "selectorId",
 284     303: "informationElementId",
 285     304: "selectorAlgorithm",
 286     305: "samplingPacketInterval",
 287     306: "samplingPacketSpace",
 288     307: "samplingTimeInterval",
 289     308: "samplingTimeSpace",
 290     309: "samplingSize",
 291     310: "samplingPopulation",
 292     311: "samplingProbability",
 293     312: "dataLinkFrameSize",
 294     313: "ipHeaderPacketSection",
 295     314: "ipPayloadPacketSection",
 296     315: "dataLinkFrameSection",
 297     316: "mplsLabelStackSection",
 298     317: "mplsPayloadPacketSection",
 299     318: "selectorIdTotalPktsObserved",
 300     319: "selectorIdTotalPktsSelected",
 301     320: "absoluteError",
 302     321: "relativeError",
 303     322: "observationTimeSeconds",
 304     323: "observationTimeMilliseconds",
 305     324: "observationTimeMicroseconds",
 306     325: "observationTimeNanoseconds",
 307     326: "digestHashValue",
 308     327: "hashIPPayloadOffset",
 309     328: "hashIPPayloadSize",
 310     329: "hashOutputRangeMin",
 311     330: "hashOutputRangeMax",
 312     331: "hashSelectedRangeMin",
 313     332: "hashSelectedRangeMax",
 314     333: "hashDigestOutput",
 315     334: "hashInitialiserValue",
 316     335: "selectorName",
 317     336: "upperCILimit",
 318     337: "lowerCILimit",
 319     338: "confidenceLevel",
 320     339: "informationElementDataType",
 321     340: "informationElementDescription",
 322     341: "informationElementName",
 323     342: "informationElementRangeBegin",
 324     343: "informationElementRangeEnd",
 325     344: "informationElementSemantics",
 326     345: "informationElementUnits",
 327     346: "privateEnterpriseNumber",
 328     347: "virtualStationInterfaceId",
 329     348: "virtualStationInterfaceName",
 330     349: "virtualStationUUID",
 331     350: "virtualStationName",
 332     351: "layer2SegmentId",
 333     352: "layer2OctetDeltaCount",
 334     353: "layer2OctetTotalCount",
 335     354: "ingressUnicastPacketTotalCount",
 336     355: "ingressMulticastPacketTotalCount",
 337     356: "ingressBroadcastPacketTotalCount",
 338     357: "egressUnicastPacketTotalCount",
 339     358: "egressBroadcastPacketTotalCount",
 340     359: "monitoringIntervalStartMilliSeconds",
 341     360: "monitoringIntervalEndMilliSeconds",
 342     361: "portRangeStart",
 343     362: "portRangeEnd",
 344     363: "portRangeStepSize",
 345     364: "portRangeNumPorts",
 346     365: "staMacAddress",
 347     366: "staIPv4Address",
 348     367: "wtpMacAddress",
 349     368: "ingressInterfaceType",
 350     369: "egressInterfaceType",
 351     370: "rtpSequenceNumber",
 352     371: "userName",
 353     372: "applicationCategoryName",
 354     373: "applicationSubCategoryName",
 355     374: "applicationGroupName",
 356     375: "originalFlowsPresent",
 357     376: "originalFlowsInitiated",
 358     377: "originalFlowsCompleted",
 359     378: "distinctCountOfSourceIPAddress",
 360     379: "distinctCountOfDestinationIPAddress",
 361     380: "distinctCountOfSourceIPv4Address",
 362     381: "distinctCountOfDestinationIPv4Address",
 363     382: "distinctCountOfSourceIPv6Address",
 364     383: "distinctCountOfDestinationIPv6Address",
 365     384: "valueDistributionMethod",
 366     385: "rfc3550JitterMilliseconds",
 367     386: "rfc3550JitterMicroseconds",
 368     387: "rfc3550JitterNanoseconds",
 369     388: "dot1qDEI",
 370     389: "dot1qCustomerDEI",
 371     390: "flowSelectorAlgorithm",
 372     391: "flowSelectedOctetDeltaCount",
 373     392: "flowSelectedPacketDeltaCount",
 374     393: "flowSelectedFlowDeltaCount",
 375     394: "selectorIDTotalFlowsObserved",
 376     395: "selectorIDTotalFlowsSelected",
 377     396: "samplingFlowInterval",
 378     397: "samplingFlowSpacing",
 379     398: "flowSamplingTimeInterval",
 380     399: "flowSamplingTimeSpacing",
 381     400: "hashFlowDomain",
 382     401: "transportOctetDeltaCount",
 383     402: "transportPacketDeltaCount",
 384     403: "originalExporterIPv4Address",
 385     404: "originalExporterIPv6Address",
 386     405: "originalObservationDomainId",
 387     406: "intermediateProcessId",
 388     407: "ignoredDataRecordTotalCount",
 389     408: "dataLinkFrameType",
 390     409: "sectionOffset",
 391     410: "sectionExportedOctets",
 392     411: "dot1qServiceInstanceTag",
 393     412: "dot1qServiceInstanceId",
 394     413: "dot1qServiceInstancePriority",
 395     414: "dot1qCustomerSourceMacAddress",
 396     415: "dot1qCustomerDestinationMacAddress",
 397     417: "postLayer2OctetDeltaCount",
 398     418: "postMCastLayer2OctetDeltaCount",
 399     420: "postLayer2OctetTotalCount",
 400     421: "postMCastLayer2OctetTotalCount",
 401     422: "minimumLayer2TotalLength",
 402     423: "maximumLayer2TotalLength",
 403     424: "droppedLayer2OctetDeltaCount",
 404     425: "droppedLayer2OctetTotalCount",
 405     426: "ignoredLayer2OctetTotalCount",
 406     427: "notSentLayer2OctetTotalCount",
 407     428: "layer2OctetDeltaSumOfSquares",
 408     429: "layer2OctetTotalSumOfSquares",
 409     430: "layer2FrameDeltaCount",
 410     431: "layer2FrameTotalCount",
 411     432: "pseudoWireDestinationIPv4Address",
 412     433: "ignoredLayer2FrameTotalCount",
 413     434: "mibObjectValueInteger",
 414     435: "mibObjectValueOctetString",
 415     436: "mibObjectValueOID",
 416     437: "mibObjectValueBits",
 417     438: "mibObjectValueIPAddress",
 418     439: "mibObjectValueCounter",
 419     440: "mibObjectValueGauge",
 420     441: "mibObjectValueTimeTicks",
 421     442: "mibObjectValueUnsigned",
 422     443: "mibObjectValueTable",
 423     444: "mibObjectValueRow",
 424     445: "mibObjectIdentifier",
 425     446: "mibSubIdentifier",
 426     447: "mibIndexIndicator",
 427     448: "mibCaptureTimeSemantics",
 428     449: "mibContextEngineID",
 429     450: "mibContextName",
 430     451: "mibObjectName",
 431     452: "mibObjectDescription",
 432     453: "mibObjectSyntax",
 433     454: "mibModuleName",
 434     455: "mobileIMSI",
 435     456: "mobileMSISDN",
 436     457: "httpStatusCode",
 437     458: "sourceTransportPortsLimit",
 438     459: "httpRequestMethod",
 439     460: "httpRequestHost",
 440     461: "httpRequestTarget",
 441     462: "httpMessageVersion",
 442     466: "natQuotaExceededEvent",
 443     471: "maxSessionEntries",
 444     472: "maxBIBEntries",
 445     473: "maxEntriesPerUser",
 446     475: "maxFragmentsPendingReassembly"
 447 }
 448
 449
 450 class IPFIX(Packet):
 451     name = "IPFIX"
 452     fields_desc = [ShortField("version", 10),
 453                    ShortField("length", None),
 454                    IntField("exportTime", None),
 455                    IntField("sequenceNumber", 1),
 456                    IntField("observationDomainID", 1)]
 457
 458
 459 class FieldSpecifier(Packet):
 460     name = "Field Specifier"
 461     fields_desc = [ShortEnumField(
 462         "informationElement", None, information_elements),
 463         ShortField("fieldLength", None)]
 464
 465     def extract_padding(self, s):
 466         return "", s
 467
 468
 469 class Template(Packet):
 470     name = "Template"
 471     fields_desc = [ShortField("templateID", 256),
 472                    FieldLenField("fieldCount", None, count_of="fields"),
 473                    PacketListField("templateFields", [], FieldSpecifier,
 474                                    count_from=lambda p: p.fieldCount)]
 475
 476
 477 class Data(Packet):
 478     name = "Data"
 479     fields_desc = [
 480         StrLenField("data", "", length_from=lambda p: p.underlayer.length - 4)]
 481
 482     def extract_padding(self, s):
 483         return "", s
 484
 485
 486 class Set(Packet):
 487     name = "Set"
 488     fields_desc = [ShortField("setID", 256),
 489                    ShortField("length", None)]
 490
 491     def guess_payload_class(self, payload):
 492         if self.setID == 2:
 493             return Template
 494         elif self.setID > 255:
 495             return Data
 496         else:
 497             return Packet.guess_payload_class(self, payload)
 498
 499
 500 bind_layers(IPFIX, Set)
 501 bind_layers(UDP, IPFIX, dport=4739)
 502
 503
 504 class IPFIXDecoder(object):
 505     """ IPFIX data set decoder """
 506
 507     def __init__(self):
 508         self._templates = []
 509
 510     def add_template(self, template):
 511         """
 512         Add IPFIX template
 513
 514         :param template: IPFIX template
 515         """
 516         templateID = template.templateID
 517         fields = []
 518         rec_len = 0
 519         for field in template.templateFields:
 520             fields.append(
 521                 {'name': field.informationElement, 'len': field.fieldLength})
 522             rec_len += field.fieldLength
 523         self._templates.append(
 524             {'id': templateID, 'fields': fields, 'rec_len': rec_len})
 525
 526     def decode_data_set(self, data_set):
 527         """
 528         Decode IPFIX data
 529
 530         :param data_set: IPFIX data set
 531         :returns: List of decoded data records.
 532         """
 533         data = []
 534         for template in self._templates:
 535             if template['id'] == data_set.setID:
 536                 offset = 0
 537                 d = data_set[Data].data
 538                 for i in range(len(d) / template['rec_len']):
 539                     record = {}
 540                     for field in template['fields']:
 541                         f = d[offset:offset + field['len']]
 542                         offset += field['len']
 543                         record.update({field['name']: f})
 544                     data.append(record)
 545                 break
 546         return data