resources/tools/telemetry/bundle_vpp.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """VPP execution bundle."""
  15
  16 from copy import deepcopy
  17 from logging import getLogger
  18 from re import fullmatch, sub
  19 import struct
  20 import sys
  21
  22 from vpp_papi.vpp_papi import VPPApiClient as vpp_class
  23
  24
  25 M_RUN_THREAD = (
  26     r"Thread\s"
  27     r"(?P<thread_id>\d+)\s"
  28     r"(?P<thread_name>\S+)\s.*"
  29     r"(?P<thread_lcore>\d+).*"
  30 )
  31 M_RUN_SEPARATOR = (
  32     r"(-)+"
  33 )
  34 M_RUN_NODES = (
  35     r"(?P<name>\S+)\s+"
  36     r"(?P<state>\S+\s\S+|\S+)\s+"
  37     r"(?P<calls>\d+)\s+"
  38     r"(?P<vectors>\d+)\s+"
  39     r"(?P<suspends>\d+)\s+"
  40     r"(?P<clocks>\S+)\s+"
  41     r"(?P<vectors_calls>\S+)"
  42 )
  43 M_RUN_TIME = (
  44     r"Time\s\S+,\s\d+\ssec\sinternal\snode\svector\srate\s"
  45     r"(?P<rate>\S+)\sloops/sec\s"
  46     r"(?P<loops>\S+)"
  47 )
  48 M_INT_BEGIN = (
  49     r"(?P<name>\S+)\s+"
  50     r"(?P<index>\S+)\s+"
  51     r"(?P<state>\S+)\s+"
  52     r"(?P<mtu>\S+)\s+"
  53     r"(?P<counter>\S+\s\S+|\S+)\s+"
  54     r"(?P<count>\d+)"
  55 )
  56 M_INT_CONT = (
  57     r"\s+"
  58     r"(?P<counter>\S+\s\S+|\S+)\s+"
  59     r"(?P<count>\d+)"
  60 )
  61 M_NODE_COUNTERS_THREAD = (
  62     r"Thread\s"
  63     r"(?P<thread_id>\d+)\s\("
  64     r"(?P<thread_name>\S+)\):\s*"
  65 )
  66 M_NODE_COUNTERS = (
  67     r"\s*"
  68     r"(?P<count>\d+)\s+"
  69     r"(?P<name>\S+)\s+"
  70     r"(?P<reason>(\S+\s)+)\s+"
  71     r"(?P<severity>\S+)\s+"
  72     r"(?P<index>\d+)\s*"
  73 )
  74 M_PMB_CS_HEADER = (
  75     r"\s*per-thread\s+context\s+switches.*"
  76 )
  77 M_PMB_CS = (
  78     r"(?P<thread_name>\S+)\s+\("
  79     r"(?P<thread_id>\S+)\)\s+\S+\s+"
  80     r"(?P<context_switches>[\d\.]+)"
  81 )
  82 M_PMB_PF_HEADER = (
  83     r"\s*per-thread\s+page\s+faults.*"
  84 )
  85 M_PMB_PF = (
  86     r"(?P<thread_name>\S+)\s+\("
  87     r"(?P<thread_id>\S+)\)\s+\S+\s+"
  88     r"(?P<minor_page_faults>[\d\.]+)\s+"
  89     r"(?P<major_page_faults>[\d\.]+)"
  90 )
  91 M_PMB_THREAD = (
  92     r"\s*"
  93     r"(?P<thread_name>\S+)\s+\("
  94     r"(?P<thread_id>\d+)\)\s*"
  95 )
  96 M_PMB_IC_HEADER = (
  97     r"\s*instructions/packet,\s+cycles/packet\s+and\s+IPC.*"
  98 )
  99 M_PMB_IC_NODE = (
 100     r"\s*"
 101     r"(?P<node_name>\S+)\s+"
 102     r"(?P<calls>[\d\.]+)\s+"
 103     r"(?P<packets>[\d\.]+)\s+"
 104     r"(?P<packets_per_call>[\d\.]+)\s+"
 105     r"(?P<clocks_per_packets>[\d\.]+)\s+"
 106     r"(?P<instructions_per_packets>[\d\.]+)\s+"
 107     r"(?P<ipc>[\d\.]+)"
 108 )
 109 M_PMB_CM_HEADER = (
 110     r"\s*cache\s+hits\s+and\s+misses.*"
 111 )
 112 M_PMB_CM_NODE = (
 113     r"\s*"
 114     r"(?P<node_name>\S+)\s+"
 115     r"(?P<l1_hit>[\d\.]+)\s+"
 116     r"(?P<l1_miss>[\d\.]+)\s+"
 117     r"(?P<l2_hit>[\d\.]+)\s+"
 118     r"(?P<l2_miss>[\d\.]+)\s+"
 119     r"(?P<l3_hit>[\d\.]+)\s+"
 120     r"(?P<l3_miss>[\d\.]+)"
 121 )
 122 M_PMB_LO_HEADER = (
 123     r"\s*load\s+operations.*"
 124 )
 125 M_PMB_LO_NODE = (
 126     r"\s*"
 127     r"(?P<node_name>\S+)\s+"
 128     r"(?P<calls>[\d\.]+)\s+"
 129     r"(?P<packets>[\d\.]+)\s+"
 130     r"(?P<one>[\d\.]+)\s+"
 131     r"(?P<two>[\d\.]+)\s+"
 132     r"(?P<three>[\d\.]+)"
 133 )
 134 M_PMB_BM_HEADER = (
 135     r"\s*Branches,\s+branches\s+taken\s+and\s+mis-predictions.*"
 136 )
 137 M_PMB_BM_NODE = (
 138     r"\s*"
 139     r"(?P<node_name>\S+)\s+"
 140     r"(?P<branches_per_call>[\d\.]+)\s+"
 141     r"(?P<branches_per_packet>[\d\.]+)\s+"
 142     r"(?P<taken_per_call>[\d\.]+)\s+"
 143     r"(?P<taken_per_packet>[\d\.]+)\s+"
 144     r"(?P<mis_predictions>[\d\.]+)"
 145 )
 146 M_PMB_PL_HEADER = (
 147     r"\s*Thread\s+power\s+licensing.*"
 148 )
 149 M_PMB_PL_NODE = (
 150     r"\s*"
 151     r"(?P<node_name>\S+)\s+"
 152     r"(?P<lvl0>[\d\.]+)\s+"
 153     r"(?P<lvl1>[\d\.]+)\s+"
 154     r"(?P<lvl2>[\d\.]+)\s+"
 155     r"(?P<throttle>[\d\.]+)"
 156 )
 157 M_PMB_MB_HEADER = (
 158     r"\s*memory\s+reads\s+and\s+writes\s+per\s+memory\s+controller.*"
 159 )
 160 M_PMB_MB = (
 161     r"\s*"
 162     r"(?P<name>\S+)\s+"
 163     r"(?P<runtime>[\d\.]+)\s+"
 164     r"(?P<reads_mbs>[\d\.]+)\s+"
 165     r"(?P<writes_mbs>[\d\.]+)\s+"
 166     r"(?P<total_mbs>[\d\.]+)"
 167 )
 168
 169
 170 class BundleVpp:
 171     """
 172     Creates a VPP object. This is the main object for defining a VPP program,
 173     and interacting with its output.
 174     """
 175     def __init__(self, program, serializer, hook):
 176         """
 177         Initialize Bundle VPP class.
 178
 179         :param program: VPP instructions.
 180         :param serializer: Metric serializer.
 181         :param hook: VPP API socket.
 182         :type program: dict
 183         :type serializer: Serializer
 184         :type hook: int
 185         """
 186         self.obj = None
 187         self.code = program[u"code"]
 188         self.metrics = program[u"metrics"]
 189         self.api_command_list = list()
 190         self.api_replies_list = list()
 191         self.serializer = serializer
 192
 193         vpp_class.apidir = u"/usr/share/vpp/api"
 194         self.obj = vpp_class(
 195             use_socket=True,
 196             server_address=hook,
 197             async_thread=False,
 198             read_timeout=14,
 199             logger=getLogger(__name__)
 200         )
 201
 202     def attach(self, duration):
 203         """
 204         Attach events to VPP.
 205
 206         :param duration: Trial duration.
 207         :type duration: int
 208         """
 209         try:
 210             self.obj.connect(name=u"telemetry")
 211         except (ConnectionRefusedError, OSError):
 212             getLogger(__name__).error(u"Cannot connect to VPP!")
 213             sys.exit(1)
 214
 215         for command in self.code.splitlines():
 216             api_name = u"cli_inband"
 217             api_args = dict(cmd=command.format(duration=duration))
 218             self.api_command_list.append(
 219                 dict(api_name=api_name, api_args=deepcopy(api_args))
 220             )
 221
 222     def detach(self):
 223         """
 224         Detach from VPP.
 225         """
 226         self.obj.disconnect()
 227
 228     def fetch_data(self):
 229         """
 230         Fetch data by invoking API calls to VPP socket.
 231         """
 232         for command in self.api_command_list:
 233             try:
 234                 papi_fn = getattr(self.obj.api, command[u"api_name"])
 235                 getLogger(__name__).info(command[u"api_args"][u"cmd"])
 236                 replies = papi_fn(**command[u"api_args"])
 237             except (AttributeError, IOError, struct.error) as err:
 238                 raise AssertionError(err)
 239
 240             if not isinstance(replies, list):
 241                 replies = [replies]
 242             for reply in replies:
 243                 self.api_replies_list.append(reply)
 244                 reply = sub(r"\x1b[^m]*m", u"", reply.reply)
 245                 if reply:
 246                     getLogger(__name__).info(reply)
 247                 else:
 248                     getLogger(__name__).info(u"<no reply>")
 249         self.serializer.create(metrics=self.metrics)
 250
 251     def process_data(self):
 252         """
 253         Post process command reply.
 254         """
 255         for command in zip(self.api_command_list, self.api_replies_list):
 256             self_fn = command[0][u"api_args"][u"cmd"]
 257             try:
 258                 self_fn = getattr(self, self_fn.replace(u" ", u"_"))
 259                 self_fn(command[1].reply)
 260             except AttributeError:
 261                 pass
 262
 263     def show_interface(self, reply):
 264         """
 265         Parse the show interface output.
 266
 267         :param reply: API reply.
 268         :type reply: str
 269         """
 270         for line in reply.splitlines():
 271             item = dict()
 272             labels = dict()
 273             if fullmatch(M_INT_BEGIN, line):
 274                 ifc = fullmatch(M_INT_BEGIN, line).groupdict()
 275                 metric = ifc[u"counter"].replace(" ", "_").replace("-", "_")
 276                 item[u"name"] = metric
 277                 item[u"value"] = ifc[u"count"]
 278             if fullmatch(M_INT_CONT, line):
 279                 ifc_cnt = fullmatch(M_INT_CONT, line).groupdict()
 280                 metric = ifc_cnt[u"counter"].replace(" ", "_").replace("-", "_")
 281                 item[u"name"] = metric
 282                 item[u"value"] = ifc_cnt[u"count"]
 283             if fullmatch(M_INT_BEGIN, line) or fullmatch(M_INT_CONT, line):
 284                 labels[u"name"] = ifc[u"name"]
 285                 labels[u"index"] = ifc[u"index"]
 286                 item[u"labels"] = labels
 287                 self.serializer.serialize(
 288                     metric=metric, labels=labels, item=item
 289                 )
 290
 291     def show_runtime(self, reply):
 292         """
 293         Parse the show runtime output.
 294
 295         :param reply: API reply.
 296         :type reply: str
 297         """
 298         for line in reply.splitlines():
 299             if fullmatch(M_RUN_THREAD, line):
 300                 thread = fullmatch(M_RUN_THREAD, line).groupdict()
 301             if fullmatch(M_RUN_NODES, line):
 302                 nodes = fullmatch(M_RUN_NODES, line).groupdict()
 303                 for metric in self.serializer.metric_registry:
 304                     item = dict()
 305                     labels = dict()
 306                     item[u"name"] = metric
 307                     labels[u"name"] = nodes[u"name"]
 308                     labels[u"state"] = nodes[u"state"]
 309                     try:
 310                         labels[u"thread_name"] = thread[u"thread_name"]
 311                         labels[u"thread_id"] = thread[u"thread_id"]
 312                         labels[u"thread_lcore"] = thread[u"thread_lcore"]
 313                     except UnboundLocalError:
 314                         labels[u"thread_name"] = u"vpp_main"
 315                         labels[u"thread_id"] = u"0"
 316                         labels[u"thread_lcore"] = u"0"
 317                     item[u"labels"] = labels
 318                     item[u"value"] = nodes[metric]
 319                     self.serializer.serialize(
 320                         metric=metric, labels=labels, item=item
 321                     )
 322
 323     def show_node_counters_verbose(self, reply):
 324         """
 325         Parse the show node conuter output.
 326
 327         :param reply: API reply.
 328         :type reply: str
 329         """
 330         for line in reply.splitlines():
 331             if fullmatch(M_NODE_COUNTERS_THREAD, line):
 332                 thread = fullmatch(M_NODE_COUNTERS_THREAD, line).groupdict()
 333             if fullmatch(M_NODE_COUNTERS, line):
 334                 nodes = fullmatch(M_NODE_COUNTERS, line).groupdict()
 335                 for metric in self.serializer.metric_registry_registry:
 336                     item = dict()
 337                     labels = dict()
 338                     item[u"name"] = metric
 339                     labels[u"name"] = nodes[u"name"]
 340                     labels[u"reason"] = nodes[u"reason"]
 341                     labels[u"severity"] = nodes[u"severity"]
 342                     try:
 343                         labels[u"thread_name"] = thread[u"thread_name"]
 344                         labels[u"thread_id"] = thread[u"thread_id"]
 345                     except UnboundLocalError:
 346                         labels[u"thread_name"] = u"vpp_main"
 347                         labels[u"thread_id"] = u"0"
 348                     item[u"labels"] = labels
 349                     item[u"value"] = nodes[u"count"]
 350                     self.serializer.serialize(
 351                         metric=metric, labels=labels, item=item
 352                     )
 353
 354     def show_perfmon_statistics(self, reply):
 355         """
 356         Parse the permon output.
 357
 358         :param reply: API reply.
 359         :type reply: str
 360         """
 361         def perfmon_threads(reply, regex_threads):
 362             for line in reply.splitlines():
 363                 if fullmatch(regex_threads, line):
 364                     threads = fullmatch(regex_threads, line).groupdict()
 365                     for metric in self.serializer.metric_registry:
 366                         item = dict()
 367                         labels = dict()
 368                         item[u"name"] = metric
 369                         labels[u"name"] = threads[u"thread_name"]
 370                         labels[u"id"] = threads[u"thread_id"]
 371                         item[u"labels"] = labels
 372                         item[u"value"] = threads[metric]
 373                         self.serializer.serialize(
 374                             metric=metric, labels=labels, item=item
 375                         )
 376
 377         def perfmon_nodes(reply, regex_threads, regex_nodes):
 378             for line in reply.splitlines():
 379                 if fullmatch(regex_threads, line):
 380                     thread = fullmatch(regex_threads, line).groupdict()
 381                 if fullmatch(regex_nodes, line):
 382                     node = fullmatch(regex_nodes, line).groupdict()
 383                     for metric in self.serializer.metric_registry:
 384                         item = dict()
 385                         labels = dict()
 386                         item[u"name"] = metric
 387                         labels[u"name"] = node[u"node_name"]
 388                         labels[u"thread_name"] = thread[u"thread_name"]
 389                         labels[u"thread_id"] = thread[u"thread_id"]
 390                         item[u"labels"] = labels
 391                         item[u"value"] = node[metric]
 392                         self.serializer.serialize(
 393                             metric=metric, labels=labels, item=item
 394                         )
 395
 396         def perfmon_system(reply, regex_line):
 397             for line in reply.splitlines():
 398                 if fullmatch(regex_line, line):
 399                     name = fullmatch(regex_line, line).groupdict()
 400                     for metric in self.serializer.metric_registry:
 401                         item = dict()
 402                         labels = dict()
 403                         item[u"name"] = metric
 404                         labels[u"name"] = name[u"name"]
 405                         item[u"labels"] = labels
 406                         item[u"value"] = name[metric]
 407                         self.serializer.serialize(
 408                             metric=metric, labels=labels, item=item
 409                         )
 410
 411         reply = sub(r"\x1b[^m]*m", u"", reply)
 412
 413         if fullmatch(M_PMB_CS_HEADER, reply.splitlines()[0]):
 414             perfmon_threads(reply, M_PMB_CS)
 415         if fullmatch(M_PMB_PF_HEADER, reply.splitlines()[0]):
 416             perfmon_threads(reply, M_PMB_PF)
 417         if fullmatch(M_PMB_IC_HEADER, reply.splitlines()[0]):
 418             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_IC_NODE)
 419         if fullmatch(M_PMB_CM_HEADER, reply.splitlines()[0]):
 420             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_CM_NODE)
 421         if fullmatch(M_PMB_LO_HEADER, reply.splitlines()[0]):
 422             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_LO_NODE)
 423         if fullmatch(M_PMB_BM_HEADER, reply.splitlines()[0]):
 424             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_BM_NODE)
 425         if fullmatch(M_PMB_PL_HEADER, reply.splitlines()[0]):
 426             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_PL_NODE)
 427         if fullmatch(M_PMB_MB_HEADER, reply.splitlines()[0]):
 428             perfmon_system(reply, M_PMB_MB)
 429
 430     def show_version(self, reply):
 431         """
 432         Parse the version output.
 433
 434         :param reply: API reply.
 435         :type reply: str
 436         """
 437         for metric in self.serializer.metric_registry:
 438             version = reply.split()[1]
 439             item = dict()
 440             labels = dict()
 441             item[u"name"] = metric
 442             labels[u"version"] = version
 443             item[u"labels"] = labels
 444             item[u"value"] = 1.0
 445             self.serializer.serialize(
 446                 metric=metric, labels=labels, item=item
 447             )