resources/tools/telemetry/bundle_vppctl.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """VPP execution bundle."""
  15
  16 from logging import getLogger
  17 from re import fullmatch, sub
  18 import subprocess
  19 import sys
  20
  21 from .constants import Constants
  22
  23 M_RUN_THREAD = (
  24     r"Thread\s"
  25     r"(?P<thread_id>\d+)\s"
  26     r"(?P<thread_name>\S+)\s.*"
  27     r"(?P<thread_lcore>\d+).*"
  28 )
  29 M_RUN_SEPARATOR = (
  30     r"(-)+"
  31 )
  32 M_RUN_NODES = (
  33     r"(?P<name>\S+)\s+"
  34     r"(?P<state>\S+\s\S+|\S+)\s+"
  35     r"(?P<calls>\d+)\s+"
  36     r"(?P<vectors>\d+)\s+"
  37     r"(?P<suspends>\d+)\s+"
  38     r"(?P<clocks>\S+)\s+"
  39     r"(?P<vectors_calls>\S+)"
  40 )
  41 M_RUN_TIME = (
  42     r"Time\s\S+,\s\d+\ssec\sinternal\snode\svector\srate\s"
  43     r"(?P<rate>\S+)\sloops/sec\s"
  44     r"(?P<loops>\S+)"
  45 )
  46 M_INT_BEGIN = (
  47     r"(?P<name>\S+)\s+"
  48     r"(?P<index>\S+)\s+"
  49     r"(?P<state>\S+)\s+"
  50     r"(?P<mtu>\S+)\s+"
  51     r"(?P<counter>\S+\s\S+|\S+)\s+"
  52     r"(?P<count>\d+)"
  53 )
  54 M_INT_CONT = (
  55     r"\s+"
  56     r"(?P<counter>\S+\s\S+|\S+)\s+"
  57     r"(?P<count>\d+)"
  58 )
  59 M_NODE_COUNTERS_THREAD = (
  60     r"Thread\s"
  61     r"(?P<thread_id>\d+)\s\("
  62     r"(?P<thread_name>\S+)\):\s*"
  63 )
  64 M_NODE_COUNTERS = (
  65     r"\s*"
  66     r"(?P<count>\d+)\s+"
  67     r"(?P<name>\S+)\s+"
  68     r"(?P<reason>(\S+\s)+)\s+"
  69     r"(?P<severity>\S+)\s+"
  70     r"(?P<index>\d+)\s*"
  71 )
  72 M_PMB_CS_HEADER = (
  73     r"\s*per-thread\s+context\s+switches.*"
  74 )
  75 M_PMB_CS = (
  76     r"(?P<thread_name>\S+)\s+\("
  77     r"(?P<thread_id>\S+)\)\s+\S+\s+"
  78     r"(?P<context_switches>[\d\.]+)"
  79 )
  80 M_PMB_PF_HEADER = (
  81     r"\s*per-thread\s+page\s+faults.*"
  82 )
  83 M_PMB_PF = (
  84     r"(?P<thread_name>\S+)\s+\("
  85     r"(?P<thread_id>\S+)\)\s+\S+\s+"
  86     r"(?P<minor_page_faults>[\d\.]+)\s+"
  87     r"(?P<major_page_faults>[\d\.]+)"
  88 )
  89 M_PMB_THREAD = (
  90     r"\s*"
  91     r"(?P<thread_name>\S+)\s+\("
  92     r"(?P<thread_id>\d+)\)\s*"
  93 )
  94 M_PMB_IC_HEADER = (
  95     r"\s*instructions/packet,\s+cycles/packet\s+and\s+IPC.*"
  96 )
  97 M_PMB_IC_NODE = (
  98     r"\s*"
  99     r"(?P<node_name>\S+)\s+"
 100     r"(?P<calls>[\d\.]+)\s+"
 101     r"(?P<packets>[\d\.]+)\s+"
 102     r"(?P<packets_per_call>[\d\.]+)\s+"
 103     r"(?P<clocks_per_packets>[\d\.]+)\s+"
 104     r"(?P<instructions_per_packets>[\d\.]+)\s+"
 105     r"(?P<ipc>[\d\.]+)"
 106 )
 107 M_PMB_CM_HEADER = (
 108     r"\s*cache\s+hits\s+and\s+misses.*"
 109 )
 110 M_PMB_CM_NODE = (
 111     r"\s*"
 112     r"(?P<node_name>\S+)\s+"
 113     r"(?P<l1_hit>[\d\.]+)\s+"
 114     r"(?P<l1_miss>[\d\.]+)\s+"
 115     r"(?P<l2_hit>[\d\.]+)\s+"
 116     r"(?P<l2_miss>[\d\.]+)\s+"
 117     r"(?P<l3_hit>[\d\.]+)\s+"
 118     r"(?P<l3_miss>[\d\.]+)"
 119 )
 120 M_PMB_LO_HEADER = (
 121     r"\s*load\s+operations.*"
 122 )
 123 M_PMB_LO_NODE = (
 124     r"\s*"
 125     r"(?P<node_name>\S+)\s+"
 126     r"(?P<calls>[\d\.]+)\s+"
 127     r"(?P<packets>[\d\.]+)\s+"
 128     r"(?P<one>[\d\.]+)\s+"
 129     r"(?P<two>[\d\.]+)\s+"
 130     r"(?P<three>[\d\.]+)"
 131 )
 132 M_PMB_BM_HEADER = (
 133     r"\s*Branches,\s+branches\s+taken\s+and\s+mis-predictions.*"
 134 )
 135 M_PMB_BM_NODE = (
 136     r"\s*"
 137     r"(?P<node_name>\S+)\s+"
 138     r"(?P<branches_per_call>[\d\.]+)\s+"
 139     r"(?P<branches_per_packet>[\d\.]+)\s+"
 140     r"(?P<taken_per_call>[\d\.]+)\s+"
 141     r"(?P<taken_per_packet>[\d\.]+)\s+"
 142     r"(?P<mis_predictions>[\d\.]+)"
 143 )
 144 M_PMB_MB_HEADER = (
 145     r"\s*memory\s+reads\s+and\s+writes\s+per\s+memory\s+controller.*"
 146 )
 147 M_PMB_MB = (
 148     r"\s*"
 149     r"(?P<name>\S+)\s+"
 150     r"(?P<runtime>[\d\.]+)\s+"
 151     r"(?P<reads_mbs>[\d\.]+)\s+"
 152     r"(?P<writes_mbs>[\d\.]+)\s+"
 153     r"(?P<total_mbs>[\d\.]+)"
 154 )
 155
 156
 157 class BundleVppctl:
 158     """
 159     Creates a VPP object. This is the main object for defining a VPP program,
 160     and interacting with its output.
 161     """
 162     def __init__(self, program, serializer, hook):
 163         """
 164         Initialize Bundle VPP class.
 165
 166         :param program: VPP instructions.
 167         :param serializer: Metric serializer.
 168         :param hook: VPP CLI socket.
 169         :type program: dict
 170         :type serializer: Serializer
 171         :type hook: int
 172         """
 173         self.obj = None
 174         self.code = program["code"]
 175         self.metrics = program["metrics"]
 176         self.cli_command_list = list()
 177         self.cli_replies_list = list()
 178         self.serializer = serializer
 179         self.hook = hook
 180
 181     def attach(self, duration):
 182         """
 183         Attach events to VPP.
 184
 185         :param duration: Trial duration.
 186         :type duration: int
 187         """
 188         for command in self.code.splitlines():
 189             self.cli_command_list.append(
 190                 command.format(duration=duration, socket=self.hook)
 191             )
 192
 193     def detach(self):
 194         """
 195         Detach from VPP.
 196         """
 197
 198     def fetch_data(self):
 199         """
 200         Fetch data by invoking subprocess calls.
 201         """
 202         for command in self.cli_command_list:
 203             try:
 204                 replies = subprocess.getoutput(command)
 205             except (AssertionError, AttributeError):
 206                 getLogger("console_stderr").error(
 207                     f"Failed when executing command: {command}"
 208                 )
 209                 sys.exit(Constants.err_vpp_execute)
 210
 211             self.cli_replies_list.append(replies)
 212             replies = sub(r"\x1b[^m]*m", "", replies)
 213             if replies:
 214                 getLogger(__name__).info(replies)
 215             else:
 216                 getLogger(__name__).info("<no reply>")
 217         self.serializer.create(metrics=self.metrics)
 218
 219     def process_data(self):
 220         """
 221         Post process command reply.
 222         """
 223         for command in zip(self.cli_command_list, self.cli_replies_list):
 224             self_fn = command[0].replace(
 225                 f"-s {self.hook} ", "").replace(" ", "_")
 226             self_method_list = [meth for meth in dir(self)
 227                                 if callable(getattr(self, meth)) and
 228                                 meth.startswith('__') is False]
 229             if self_fn not in self_method_list:
 230                 continue
 231             try:
 232                 self_fn = getattr(self, self_fn)
 233                 self_fn(command[1])
 234             except AttributeError:
 235                 pass
 236             except (KeyError, ValueError, TypeError) as exc:
 237                 getLogger("console_stderr").error(
 238                     f"Failed when processing data. Error message {exc}"
 239                 )
 240                 sys.exit(Constants.err_telemetry_process)
 241
 242     def vppctl_show_interface(self, reply):
 243         """
 244         Parse the show interface output.
 245
 246         :param reply: VPP reply.
 247         :type reply: str
 248         """
 249         for line in reply.splitlines():
 250             item = dict()
 251             labels = dict()
 252             if fullmatch(M_INT_BEGIN, line):
 253                 ifc = fullmatch(M_INT_BEGIN, line).groupdict()
 254                 metric = ifc["counter"].replace(" ", "_").replace("-", "_")
 255                 item["name"] = metric
 256                 item["value"] = ifc["count"]
 257             if fullmatch(M_INT_CONT, line):
 258                 ifc_cnt = fullmatch(M_INT_CONT, line).groupdict()
 259                 metric = ifc_cnt["counter"].replace(" ", "_").replace("-", "_")
 260                 item["name"] = metric
 261                 item["value"] = ifc_cnt["count"]
 262             if fullmatch(M_INT_BEGIN, line) or fullmatch(M_INT_CONT, line):
 263                 labels["name"] = ifc["name"]
 264                 labels["index"] = ifc["index"]
 265                 item["labels"] = labels
 266                 self.serializer.serialize(
 267                     metric=metric, labels=labels, item=item
 268                 )
 269
 270     def vppctl_show_runtime(self, reply):
 271         """
 272         Parse the show runtime output.
 273
 274         :param reply: VPP reply.
 275         :type reply: str
 276         """
 277         for line in reply.splitlines():
 278             if fullmatch(M_RUN_THREAD, line):
 279                 thread = fullmatch(M_RUN_THREAD, line).groupdict()
 280             if fullmatch(M_RUN_NODES, line):
 281                 nodes = fullmatch(M_RUN_NODES, line).groupdict()
 282                 for metric in self.serializer.metric_registry:
 283                     item = dict()
 284                     labels = dict()
 285                     item["name"] = metric
 286                     labels["name"] = nodes["name"]
 287                     labels["state"] = nodes["state"]
 288                     try:
 289                         labels["thread_name"] = thread["thread_name"]
 290                         labels["thread_id"] = thread["thread_id"]
 291                         labels["thread_lcore"] = thread["thread_lcore"]
 292                     except UnboundLocalError:
 293                         labels["thread_name"] = "vpp_main"
 294                         labels["thread_id"] = "0"
 295                         labels["thread_lcore"] = "0"
 296                     item["labels"] = labels
 297                     item["value"] = nodes[metric]
 298                     self.serializer.serialize(
 299                         metric=metric, labels=labels, item=item
 300                     )
 301
 302     def vppctl_show_node_counters_verbose(self, reply):
 303         """
 304         Parse the show node conuter output.
 305
 306         :param reply: VPP reply.
 307         :type reply: str
 308         """
 309         for line in reply.splitlines():
 310             if fullmatch(M_NODE_COUNTERS_THREAD, line):
 311                 thread = fullmatch(M_NODE_COUNTERS_THREAD, line).groupdict()
 312             if fullmatch(M_NODE_COUNTERS, line):
 313                 nodes = fullmatch(M_NODE_COUNTERS, line).groupdict()
 314                 for metric in self.serializer.metric_registry_registry:
 315                     item = dict()
 316                     labels = dict()
 317                     item["name"] = metric
 318                     labels["name"] = nodes["name"]
 319                     labels["reason"] = nodes["reason"]
 320                     labels["severity"] = nodes["severity"]
 321                     try:
 322                         labels["thread_name"] = thread["thread_name"]
 323                         labels["thread_id"] = thread["thread_id"]
 324                     except UnboundLocalError:
 325                         labels["thread_name"] = "vpp_main"
 326                         labels["thread_id"] = "0"
 327                     item["labels"] = labels
 328                     item["value"] = nodes["count"]
 329                     self.serializer.serialize(
 330                         metric=metric, labels=labels, item=item
 331                     )
 332
 333     def vppctl_show_perfmon_statistics(self, reply):
 334         """
 335         Parse the perfmon output.
 336
 337         :param reply: VPP reply.
 338         :type reply: str
 339         """
 340         def perfmon_threads(reply, regex_threads):
 341             for line in reply.splitlines():
 342                 if fullmatch(regex_threads, line):
 343                     threads = fullmatch(regex_threads, line).groupdict()
 344                     for metric in self.serializer.metric_registry:
 345                         item = dict()
 346                         labels = dict()
 347                         item["name"] = metric
 348                         labels["name"] = threads["thread_name"]
 349                         labels["id"] = threads["thread_id"]
 350                         item["labels"] = labels
 351                         item["value"] = threads[metric]
 352                         self.serializer.serialize(
 353                             metric=metric, labels=labels, item=item
 354                         )
 355
 356         def perfmon_nodes(reply, regex_threads, regex_nodes):
 357             for line in reply.splitlines():
 358                 if fullmatch(regex_threads, line):
 359                     thread = fullmatch(regex_threads, line).groupdict()
 360                 if fullmatch(regex_nodes, line):
 361                     node = fullmatch(regex_nodes, line).groupdict()
 362                     for metric in self.serializer.metric_registry:
 363                         item = dict()
 364                         labels = dict()
 365                         item["name"] = metric
 366                         labels["name"] = node["node_name"]
 367                         labels["thread_name"] = thread["thread_name"]
 368                         labels["thread_id"] = thread["thread_id"]
 369                         item["labels"] = labels
 370                         item["value"] = node[metric]
 371                         self.serializer.serialize(
 372                             metric=metric, labels=labels, item=item
 373                         )
 374
 375         def perfmon_system(reply, regex_line):
 376             for line in reply.splitlines():
 377                 if fullmatch(regex_line, line):
 378                     name = fullmatch(regex_line, line).groupdict()
 379                     for metric in self.serializer.metric_registry:
 380                         item = dict()
 381                         labels = dict()
 382                         item["name"] = metric
 383                         labels["name"] = name["name"]
 384                         item["labels"] = labels
 385                         item["value"] = name[metric]
 386                         self.serializer.serialize(
 387                             metric=metric, labels=labels, item=item
 388                         )
 389
 390         reply = sub(r"\x1b[^m]*m", "", reply)
 391
 392         if fullmatch(M_PMB_CS_HEADER, reply.splitlines()[0]):
 393             perfmon_threads(reply, M_PMB_CS)
 394         if fullmatch(M_PMB_PF_HEADER, reply.splitlines()[0]):
 395             perfmon_threads(reply, M_PMB_PF)
 396         if fullmatch(M_PMB_IC_HEADER, reply.splitlines()[0]):
 397             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_IC_NODE)
 398         if fullmatch(M_PMB_CM_HEADER, reply.splitlines()[0]):
 399             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_CM_NODE)
 400         if fullmatch(M_PMB_LO_HEADER, reply.splitlines()[0]):
 401             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_LO_NODE)
 402         if fullmatch(M_PMB_BM_HEADER, reply.splitlines()[0]):
 403             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_BM_NODE)
 404         if fullmatch(M_PMB_MB_HEADER, reply.splitlines()[0]):
 405             perfmon_system(reply, M_PMB_MB)
 406
 407     def vppctl_show_version(self, reply):
 408         """
 409         Parse the version output.
 410
 411         :param reply: VPP reply.
 412         :type reply: str
 413         """
 414         for metric in self.serializer.metric_registry:
 415             version = reply.split()[1]
 416             item = dict()
 417             labels = dict()
 418             item["name"] = metric
 419             labels["version"] = version
 420             item["labels"] = labels
 421             item["value"] = {}
 422             self.serializer.serialize(
 423                 metric=metric, labels=labels, item=item
 424             )