GLOBAL_STATS = 'g'
PORT_STATS = 'p'
PORT_STATUS = 'ps'
-ALL_STATS_OPTS = {GLOBAL_STATS, PORT_STATS, PORT_STATUS}
+STREAMS_STATS = 's'
+
+ALL_STATS_OPTS = {GLOBAL_STATS, PORT_STATS, PORT_STATUS, STREAMS_STATS}
COMPACT = {GLOBAL_STATS, PORT_STATS}
+SS_COMPAT = {GLOBAL_STATS, STREAMS_STATS}
ExportableStats = namedtuple('ExportableStats', ['raw_data', 'text_table'])
+# deep mrege of dicts dst = src + dst
+def deep_merge_dicts (dst, src):
+ for k, v in src.iteritems():
+ # if not exists - deep copy it
+ if not k in dst:
+ dst[k] = copy.deepcopy(v)
+ else:
+ if isinstance(v, dict):
+ deep_merge_dicts(dst[k], v)
+
# use to calculate diffs relative to the previous values
# for example, BW
def calculate_diff (samples):
STLClient and the ports.
"""
- def __init__(self, global_stats_ref, ports_dict_ref):
+ def __init__(self, global_stats_ref, ports_dict_ref, rx_stats_ref):
self._global_stats = global_stats_ref
self._ports_dict = ports_dict_ref
+ self._rx_stats_ref = rx_stats_ref
def generate_single_statistic(self, port_id_list, statistic_type):
if statistic_type == GLOBAL_STATS:
return self._generate_global_stats()
+
elif statistic_type == PORT_STATS:
return self._generate_port_stats(port_id_list)
- pass
+
elif statistic_type == PORT_STATUS:
return self._generate_port_status(port_id_list)
+
+ elif statistic_type == STREAMS_STATS:
+ return self._generate_streams_stats()
else:
# ignore by returning empty object
return {}
return {"global_statistics": ExportableStats(stats_data, stats_table)}
+ def _generate_streams_stats (self):
+
+ sstats_data = self._rx_stats_ref.generate_stats()
+ streams_keys = self._rx_stats_ref.get_streams_keys()
+ stream_count = len(streams_keys)
+
+
+ stats_table = text_tables.TRexTextTable()
+ stats_table.set_cols_align(["l"] + ["r"] * stream_count)
+ stats_table.set_cols_width([20] + [17] * stream_count)
+ stats_table.set_cols_dtype(['t'] + ['t'] * stream_count)
+
+ stats_table.add_rows([[k] + v
+ for k, v in sstats_data.iteritems()],
+ header=False)
+
+ header = ["PG ID"] + [key for key in streams_keys]
+ stats_table.header(header)
+
+ return {"streams_statistics": ExportableStats(sstats_data, stats_table)}
+
+
+
+ per_stream_stats = OrderedDict([("owner", []),
+ ("state", []),
+ ("--", []),
+ ("Tx bps L2", []),
+ ("Tx bps L1", []),
+ ("Tx pps", []),
+ ("Line Util.", []),
+
+ ("---", []),
+ ("Rx bps", []),
+ ("Rx pps", []),
+
+ ("----", []),
+ ("opackets", []),
+ ("ipackets", []),
+ ("obytes", []),
+ ("ibytes", []),
+ ("tx-bytes", []),
+ ("rx-bytes", []),
+ ("tx-pkts", []),
+ ("rx-pkts", []),
+
+ ("-----", []),
+ ("oerrors", []),
+ ("ierrors", []),
+
+ ]
+ )
+
+ total_stats = CPortStats(None)
+
+ for port_obj in relevant_ports:
+ # fetch port data
+ port_stats = port_obj.generate_port_stats()
+
+ total_stats += port_obj.port_stats
+
+ # populate to data structures
+ return_stats_data[port_obj.port_id] = port_stats
+ self.__update_per_field_dict(port_stats, per_field_stats)
+
+ total_cols = len(relevant_ports)
+ header = ["port"] + [port.port_id for port in relevant_ports]
+
+ if (total_cols > 1):
+ self.__update_per_field_dict(total_stats.generate_stats(), per_field_stats)
+ header += ['total']
+ total_cols += 1
+
+ stats_table = text_tables.TRexTextTable()
+ stats_table.set_cols_align(["l"] + ["r"] * total_cols)
+ stats_table.set_cols_width([10] + [17] * total_cols)
+ stats_table.set_cols_dtype(['t'] + ['t'] * total_cols)
+
+ stats_table.add_rows([[k] + v
+ for k, v in per_field_stats.iteritems()],
+ header=False)
+
+ stats_table.header(header)
+
+ return {"streams_statistics": ExportableStats(return_stats_data, stats_table)}
+
+
def _generate_port_stats(self, port_id_list):
relevant_ports = self.__get_relevant_ports(port_id_list)
("ipackets", []),
("obytes", []),
("ibytes", []),
- ("tx_bytes", []),
- ("rx_bytes", []),
- ("tx_pkts", []),
- ("rx_pkts", []),
+ ("tx-bytes", []),
+ ("rx-bytes", []),
+ ("tx-pkts", []),
+ ("rx-pkts", []),
("-----", []),
("oerrors", []),
class CTRexStats(object):
""" This is an abstract class to represent a stats object """
- def __init__(self):
+ def __init__(self, flowing = True):
self.reference_stats = {}
self.latest_stats = {}
self.last_update_ts = time.time()
self.history = deque(maxlen = 10)
self.lock = threading.Lock()
- def __getitem__(self, item):
- # override this to allow quick and clean access to fields
- if not item in self.latest_stats:
- return "N/A"
-
- # item must exist
- m = re.search('_(([a-z])ps)$', item)
- if m:
- # this is a non-relative item
- unit = m.group(2)
- if unit == "b":
- return self.get(item, format=True, suffix="b/sec")
- elif unit == "p":
- return self.get(item, format=True, suffix="pkt/sec")
- else:
- return self.get(item, format=True, suffix=m.group(1))
-
- m = re.search('^[i|o](a-z+)$', item)
- if m:
- # this is a non-relative item
- type = m.group(1)
- if type == "bytes":
- return self.get_rel(item, format=True, suffix="B")
- elif type == "packets":
- return self.get_rel(item, format=True, suffix="pkts")
- else:
- # do not format with suffix
- return self.get_rel(item, format=True)
+ # does the object gets a constant flow of data ?
+ self.flowing = flowing
- # can't match to any known pattern, return N/A
- return "N/A"
+ ######## abstract methods ##########
+ # get stats for user / API
+ def get_stats (self):
+ raise NotImplementedError()
+ # generate format stats (for TUI)
def generate_stats(self):
- # must be implemented by designated classes (such as port/ global stats)
raise NotImplementedError()
- def generate_extended_values (self, snapshot):
+ # called when a snapshot arrives - add more fields
+ def preprocess_snapshot (self, snapshot):
raise NotImplementedError()
+ ######## END abstract methods ##########
+
+ def __update_ref (self):
+ deep_merge_dicts(self.reference_stats, self.latest_stats)
+
def update(self, snapshot):
# some extended generated values (from base values)
- self.generate_extended_values(snapshot)
+ self.preprocess_snapshot(snapshot)
# update
self.latest_stats = snapshot
diff_time = time.time() - self.last_update_ts
- # 3 seconds is too much - this is the new reference
- if (not self.reference_stats) or (diff_time > 3):
- self.reference_stats = self.latest_stats
+ # handle the reference (base)
+ self.__update_ref()
+
+ # for flowing objects 3 seconds is too much
+ if self.flowing and (diff_time > 3):
+ self.clear_stats()
-
self.last_update_ts = time.time()
def clear_stats(self):
- self.reference_stats = self.latest_stats
+ self.reference_stats = copy.deepcopy(self.latest_stats)
def invalidate (self):
self.latest_stats = {}
+
+ def __get (self, src, field):
+ if isinstance(field, list):
+ # deep
+ value = src
+ for level in field:
+ if not level in value:
+ return None
+ value = value[level]
+ else:
+ # flat
+ if not field in src:
+ return None
+ value = src[field]
+
+ return value
+
def get(self, field, format=False, suffix=""):
- if not field in self.latest_stats:
+ value = self.__get(self.latest_stats, field)
+ if value == None:
return "N/A"
- if not format:
- return self.latest_stats[field]
- else:
- return format_num(self.latest_stats[field], suffix)
+
+ return value if not format else format_num(value, suffix)
+
def get_rel(self, field, format=False, suffix=""):
- if not field in self.latest_stats:
+ ref_value = self.__get(self.reference_stats, field)
+ if ref_value == None:
return "N/A"
- if not format:
- if not field in self.reference_stats:
- print "REF: " + str(self.reference_stats)
- print "BASE: " + str(self.latest_stats)
+ # if the latest does not have the value - its like the ref
+ latest_value = self.__get(self.latest_stats, field)
+ if latest_value == None:
+ latest_value = ref_value
+
+
+ value = latest_value - ref_value
+
+ return value if not format else format_num(value, suffix)
- return (self.latest_stats[field] - self.reference_stats[field])
- else:
- return format_num(self.latest_stats[field] - self.reference_stats[field], suffix)
# get trend for a field
def get_trend (self, field, use_raw = False, percision = 10.0):
return stats
- def generate_extended_values (self, snapshot):
+ def preprocess_snapshot (self, snapshot):
# L1 bps
bps = snapshot.get("m_tx_bps")
pps = snapshot.get("m_tx_pps")
return stats
- def generate_extended_values (self, snapshot):
+ def preprocess_snapshot (self, snapshot):
# L1 bps
bps = snapshot.get("m_total_tx_bps")
pps = snapshot.get("m_total_tx_pps")
"obytes" : self.get_rel("obytes"),
"ibytes" : self.get_rel("ibytes"),
- "tx_bytes": self.get_rel("obytes", format = True, suffix = "B"),
- "rx_bytes": self.get_rel("ibytes", format = True, suffix = "B"),
- "tx_pkts": self.get_rel("opackets", format = True, suffix = "pkts"),
- "rx_pkts": self.get_rel("ipackets", format = True, suffix = "pkts"),
+ "tx-bytes": self.get_rel("obytes", format = True, suffix = "B"),
+ "rx-bytes": self.get_rel("ibytes", format = True, suffix = "B"),
+ "tx-pkts": self.get_rel("opackets", format = True, suffix = "pkts"),
+ "rx-pkts": self.get_rel("ipackets", format = True, suffix = "pkts"),
"oerrors" : format_num(self.get_rel("oerrors"),
compact = False,
}
-class CRxStats(object):
+class CRxStats(CTRexStats):
def __init__(self):
- self.flow_stats = {}
+ super(CRxStats, self).__init__(flowing = False)
+
+
+
+ def preprocess_snapshot (self, snapshot):
+ # heavy pre-processing here...
+ new_snapshot = {}
+
+ if not 'timestamp' in snapshot:
+ raise ValueError("INTERNAL ERROR: RX stats snapshot MUST contain 'timestamp' field")
+
+ for key, value in snapshot.iteritems():
+ # skip non int values (simply copy)
+ if key == 'timestamp':
+ new_snapshot[key] = value
+ continue
+
+ # all the rest MUST be ints
+ try:
+ pg_id = int(key)
+ except ValueError:
+ assert(0)
+ # handle PG ID
+ new_snapshot[pg_id] = {}
+ for field in ['tx_pkts', 'tx_bytes', 'rx_pkts']:
+ new_snapshot[pg_id][field] = {'total': 0}
+ if field in value:
+ for port, pv in value[field].iteritems():
+ new_snapshot[pg_id][field][int(port)] = pv
+ new_snapshot[pg_id][field]['total'] += pv
- def update (self, snapshot):
- self.flow_stats = snapshot
+ snapshot.clear()
+ snapshot.update(new_snapshot)
+
def get_stats (self):
stats = {}
- for pg_id, pg_id_data in self.flow_stats.iteritems():
- # ignore non pg ID keys
+
+ for pg_id in self.get_streams_keys():
+ stats[pg_id] = {}
+ for field in ['tx_pkts', 'tx_bytes', 'rx_pkts']:
+ stats[pg_id][field] = {'total': self.get_rel([pg_id, field, 'total'])}
+
+ for port, pv in self.reference_stats[pg_id][field].iteritems():
+ stats[pg_id][field][port] = self.get_rel([pg_id, field, port])
+
+ return stats
+
+
+
+ def get_streams_keys (self):
+ keys = []
+ for user_id, user_id_data in self.reference_stats.iteritems():
+ # ignore non user ID keys
try:
- pg_id = int(pg_id)
+ keys.append(int(user_id))
except ValueError:
continue
- # handle pg id
- stats[pg_id] = {}
- for field, per_port_data in pg_id_data.iteritems():
- stats[pg_id][field] = {}
- for port, value in per_port_data.iteritems():
- stats[pg_id][field][int(port)] = value
+ return keys
- return stats
+ def generate_stats (self):
+
+ stats = self.get_stats()
+ pg_ids = stats.keys()[:4]
+ cnt = len(pg_ids)
+
+
+ formatted_stats = OrderedDict([ ('Tx bps L2', []),
+ ('Tx bps L1', []),
+ ('Tx pps', []),
+ ('---', [''] * cnt),
+ ('Rx pps', []),
+ ('----', [''] * cnt),
+ ('opackets', []),
+ ('ipackets', []),
+ ('obytes', []),
+ ('ibytes', []),
+ ('-----', [''] * cnt),
+ ('tx_pkts', []),
+ ('rx_pkts', []),
+ ('tx_bytes', []),
+ ('rx_bytes', [])
+ ])
+
+
+
+ # maximum 4
+ for pg_id in pg_ids:
+
+ #formatted_stats['TX packet count'].append(stats[key]['tx-pkts']['total'])
+ #formatted_stats['TX byte count'].append(stats[key]['tx-bytes']['total'])
+ #formatted_stats['RX packet count'].append(stats[key]['rx-pkts']['total'])
+ formatted_stats['Tx bps L2'].append(0)
+ formatted_stats['Tx bps L1'].append(0)
+ formatted_stats['Tx pps'].append(0)
+ formatted_stats['Rx pps'].append(0)
+ formatted_stats['opackets'].append(self.get_rel([pg_id, 'tx_pkts', 'total']))
+ formatted_stats['ipackets'].append(self.get_rel([pg_id, 'rx_pkts', 'total']))
+ formatted_stats['obytes'].append(self.get_rel([pg_id, 'tx_bytes', 'total']))
+ formatted_stats['ibytes'].append(self.get_rel([pg_id, 'rx_bytes', 'total']))
+ formatted_stats['tx_bytes'].append(self.get_rel([pg_id, 'tx_bytes', 'total'], format = True, suffix = "B"))
+ formatted_stats['rx_bytes'].append(self.get_rel([pg_id, 'rx_bytes', 'total'], format = True, suffix = "B"))
+ formatted_stats['tx_pkts'].append(self.get_rel([pg_id, 'tx_pkts', 'total'], format = True, suffix = "pkts"))
+ formatted_stats['rx_pkts'].append(self.get_rel([pg_id, 'rx_pkts', 'total'], format = True, suffix = "pkts"))
+
+
+
+ return formatted_stats
if __name__ == "__main__":
pass
Code Review / trex.git / commitdiff