resources/tools/telemetry/bundle_vppctl.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """VPP execution bundle."""
  15
  16 from logging import getLogger
  17 from re import fullmatch, sub
  18 import subprocess
  19 import sys
  20
  21 from .constants import Constants
  22
  23 M_RUN_THREAD = (
  24     r"Thread\s"
  25     r"(?P<thread_id>\d+)\s"
  26     r"(?P<thread_name>\S+)\s.*"
  27     r"(?P<thread_lcore>\d+).*"
  28 )
  29 M_RUN_SEPARATOR = (
  30     r"(-)+"
  31 )
  32 M_RUN_NODES = (
  33     r"(?P<node_name>\S+)\s+"
  34     r"(?P<state>\S+\s\S+|\S+)\s+"
  35     r"(?P<calls>\d+)\s+"
  36     r"(?P<vectors>\d+)\s+"
  37     r"(?P<suspends>\d+)\s+"
  38     r"(?P<clocks>\S+)\s+"
  39     r"(?P<vectors_calls>\S+)"
  40 )
  41 M_RUN_TIME = (
  42     r"Time\s\S+,\s\d+\ssec\sinternal\snode\svector\srate\s"
  43     r"(?P<rate>\S+)\sloops/sec\s"
  44     r"(?P<loops>\S+)"
  45 )
  46 M_INT_BEGIN = (
  47     r"(?P<name>\S+)\s+"
  48     r"(?P<index>\S+)\s+"
  49     r"(?P<state>\S+)\s+"
  50     r"(?P<mtu>\S+)\s+"
  51     r"(?P<counter>\S+\s\S+|\S+)\s+"
  52     r"(?P<count>\d+)"
  53 )
  54 M_INT_CONT = (
  55     r"\s+"
  56     r"(?P<counter>\S+\s\S+|\S+)\s+"
  57     r"(?P<count>\d+)"
  58 )
  59 M_NODE_COUNTERS_THREAD = (
  60     r"Thread\s"
  61     r"(?P<thread_id>\d+)\s\("
  62     r"(?P<thread_name>\S+)\):\s*"
  63 )
  64 M_NODE_COUNTERS = (
  65     r"\s*"
  66     r"(?P<count>\d+)\s+"
  67     r"(?P<name>\S+)\s+"
  68     r"(?P<reason>(\S+\s)+)\s+"
  69     r"(?P<severity>\S+)\s+"
  70     r"(?P<index>\d+)\s*"
  71 )
  72 M_PMB_CS_HEADER = (
  73     r"\s*per-thread\s+context\s+switches.*"
  74 )
  75 M_PMB_CS = (
  76     r"(?P<thread_name>\S+)\s+\("
  77     r"(?P<thread_id>\S+)\)\s+\S+\s+"
  78     r"(?P<context_switches>[\d\.]+)"
  79 )
  80 M_PMB_PF_HEADER = (
  81     r"\s*per-thread\s+page\s+faults.*"
  82 )
  83 M_PMB_PF = (
  84     r"(?P<thread_name>\S+)\s+\("
  85     r"(?P<thread_id>\S+)\)\s+\S+\s+"
  86     r"(?P<minor_page_faults>[\d\.]+)\s+"
  87     r"(?P<major_page_faults>[\d\.]+)"
  88 )
  89 M_PMB_THREAD = (
  90     r"\s*"
  91     r"(?P<thread_name>\S+)\s+\("
  92     r"(?P<thread_id>\d+)\)\s*"
  93 )
  94 M_PMB_IC_HEADER = (
  95     r"\s*instructions/packet,\s+cycles/packet\s+and\s+IPC.*"
  96 )
  97 M_PMB_IC_NODE = (
  98     r"\s*"
  99     r"(?P<node_name>\S+)\s+"
 100     r"(?P<calls>[\d\.]+)\s+"
 101     r"(?P<packets>[\d\.]+)\s+"
 102     r"(?P<packets_per_call>[\d\.]+)\s+"
 103     r"(?P<clocks_per_packets>[\d\.]+)\s+"
 104     r"(?P<instructions_per_packets>[\d\.]+)\s+"
 105     r"(?P<ipc>[\d\.]+)"
 106 )
 107 M_PMB_CM_HEADER = (
 108     r"\s*cache\s+hits\s+and\s+misses.*"
 109 )
 110 M_PMB_CM_NODE = (
 111     r"\s*"
 112     r"(?P<node_name>\S+)\s+"
 113     r"(?P<l1_hit>[\d\.]+)\s+"
 114     r"(?P<l1_miss>[\d\.]+)\s+"
 115     r"(?P<l2_hit>[\d\.]+)\s+"
 116     r"(?P<l2_miss>[\d\.]+)\s+"
 117     r"(?P<l3_hit>[\d\.]+)\s+"
 118     r"(?P<l3_miss>[\d\.]+)"
 119 )
 120 M_PMB_LO_HEADER = (
 121     r"\s*load\s+operations.*"
 122 )
 123 M_PMB_LO_NODE = (
 124     r"\s*"
 125     r"(?P<node_name>\S+)\s+"
 126     r"(?P<calls>[\d\.]+)\s+"
 127     r"(?P<packets>[\d\.]+)\s+"
 128     r"(?P<one>[\d\.]+)\s+"
 129     r"(?P<two>[\d\.]+)\s+"
 130     r"(?P<three>[\d\.]+)"
 131 )
 132 M_PMB_BM_HEADER = (
 133     r"\s*Branches,\s+branches\s+taken\s+and\s+mis-predictions.*"
 134 )
 135 M_PMB_BM_NODE = (
 136     r"\s*"
 137     r"(?P<node_name>\S+)\s+"
 138     r"(?P<branches_per_call>[\d\.]+)\s+"
 139     r"(?P<branches_per_packet>[\d\.]+)\s+"
 140     r"(?P<taken_per_call>[\d\.]+)\s+"
 141     r"(?P<taken_per_packet>[\d\.]+)\s+"
 142     r"(?P<mis_predictions>[\d\.]+)"
 143 )
 144 M_PMB_MB_HEADER = (
 145     r"\s*memory\s+reads\s+and\s+writes\s+per\s+memory\s+controller.*"
 146 )
 147 M_PMB_MB = (
 148     r"\s*"
 149     r"(?P<name>\S+)\s+"
 150     r"(?P<runtime>[\d\.]+)\s+"
 151     r"(?P<reads_mbs>[\d\.]+)\s+"
 152     r"(?P<writes_mbs>[\d\.]+)\s+"
 153     r"(?P<total_mbs>[\d\.]+)"
 154 )
 155
 156
 157 class BundleVppctl:
 158     """
 159     Creates a VPP object. This is the main object for defining a VPP program,
 160     and interacting with its output.
 161     """
 162     def __init__(self, program, serializer, hook):
 163         """
 164         Initialize Bundle VPP class.
 165
 166         :param program: VPP instructions.
 167         :param serializer: Metric serializer.
 168         :param hook: VPP CLI socket.
 169         :type program: dict
 170         :type serializer: Serializer
 171         :type hook: int
 172         """
 173         self.obj = None
 174         self.code = program["code"]
 175         self.metrics = program["metrics"]
 176         self.cli_command_list = list()
 177         self.cli_replies_list = list()
 178         self.serializer = serializer
 179         self.hook = hook
 180
 181     def attach(self, duration):
 182         """
 183         Attach events to VPP.
 184
 185         :param duration: Trial duration.
 186         :type duration: int
 187         """
 188         for command in self.code.splitlines():
 189             self.cli_command_list.append(
 190                 command.format(duration=duration, socket=self.hook)
 191             )
 192
 193     def detach(self):
 194         """
 195         Detach from VPP.
 196         """
 197
 198     def fetch_data(self):
 199         """
 200         Fetch data by invoking subprocess calls.
 201         """
 202         for command in self.cli_command_list:
 203             try:
 204                 getLogger(__name__).info(command)
 205                 replies = subprocess.getoutput(command)
 206             except (AssertionError, AttributeError):
 207                 getLogger("console_stderr").error(
 208                     f"Failed when executing command: {command}"
 209                 )
 210                 sys.exit(Constants.err_vpp_execute)
 211
 212             self.cli_replies_list.append(replies)
 213             replies = sub(r"\x1b[^m]*m", "", replies)
 214             if replies:
 215                 getLogger(__name__).info(replies)
 216             else:
 217                 getLogger(__name__).info("<no reply>")
 218         self.serializer.create(metrics=self.metrics)
 219
 220     def process_data(self):
 221         """
 222         Post process command reply.
 223         """
 224         for command in zip(self.cli_command_list, self.cli_replies_list):
 225             self_fn = command[0].replace(
 226                 f"-s {self.hook} ", "").replace(" ", "_")
 227             self_method_list = [meth for meth in dir(self)
 228                                 if callable(getattr(self, meth)) and
 229                                 meth.startswith('__') is False]
 230             if self_fn not in self_method_list:
 231                 continue
 232             try:
 233                 self_fn = getattr(self, self_fn)
 234                 self_fn(command[1])
 235             except AttributeError:
 236                 pass
 237             except (KeyError, ValueError, TypeError) as exc:
 238                 getLogger("console_stderr").error(
 239                     f"Failed when processing data. Error message {exc}"
 240                 )
 241                 sys.exit(Constants.err_telemetry_process)
 242
 243     def vppctl_show_interface(self, reply):
 244         """
 245         Parse the show interface output.
 246
 247         :param reply: VPP reply.
 248         :type reply: str
 249         """
 250         for line in reply.splitlines():
 251             item = dict()
 252             labels = dict()
 253             if fullmatch(M_INT_BEGIN, line):
 254                 ifc = fullmatch(M_INT_BEGIN, line).groupdict()
 255                 metric = ifc["counter"].replace(" ", "_").replace("-", "_")
 256                 item["name"] = metric
 257                 item["value"] = ifc["count"]
 258             if fullmatch(M_INT_CONT, line):
 259                 ifc_cnt = fullmatch(M_INT_CONT, line).groupdict()
 260                 metric = ifc_cnt["counter"].replace(" ", "_").replace("-", "_")
 261                 item["name"] = metric
 262                 item["value"] = ifc_cnt["count"]
 263             if fullmatch(M_INT_BEGIN, line) or fullmatch(M_INT_CONT, line):
 264                 labels["name"] = ifc["name"]
 265                 labels["index"] = ifc["index"]
 266                 item["labels"] = labels
 267                 self.serializer.serialize(
 268                     metric=metric, labels=labels, item=item
 269                 )
 270
 271     def vppctl_show_runtime(self, reply):
 272         """
 273         Parse the show runtime output.
 274
 275         :param reply: VPP reply.
 276         :type reply: str
 277         """
 278         for line in reply.splitlines():
 279             if fullmatch(M_RUN_THREAD, line):
 280                 thread = fullmatch(M_RUN_THREAD, line).groupdict()
 281             if fullmatch(M_RUN_NODES, line):
 282                 nodes = fullmatch(M_RUN_NODES, line).groupdict()
 283                 for metric in self.serializer.metric_registry:
 284                     item = dict()
 285                     labels = dict()
 286                     item["name"] = metric
 287                     labels["node_name"] = nodes["node_name"]
 288                     labels["state"] = nodes["state"]
 289                     try:
 290                         labels["thread_name"] = thread["thread_name"]
 291                         labels["thread_id"] = thread["thread_id"]
 292                         labels["thread_lcore"] = thread["thread_lcore"]
 293                     except UnboundLocalError:
 294                         labels["thread_name"] = "vpp_main"
 295                         labels["thread_id"] = "0"
 296                         labels["thread_lcore"] = "0"
 297                     item["labels"] = labels
 298                     item["value"] = nodes[metric]
 299                     self.serializer.serialize(
 300                         metric=metric, labels=labels, item=item
 301                     )
 302
 303     def vppctl_show_node_counters_verbose(self, reply):
 304         """
 305         Parse the show node conuter output.
 306
 307         :param reply: VPP reply.
 308         :type reply: str
 309         """
 310         for line in reply.splitlines():
 311             if fullmatch(M_NODE_COUNTERS_THREAD, line):
 312                 thread = fullmatch(M_NODE_COUNTERS_THREAD, line).groupdict()
 313             if fullmatch(M_NODE_COUNTERS, line):
 314                 nodes = fullmatch(M_NODE_COUNTERS, line).groupdict()
 315                 for metric in self.serializer.metric_registry_registry:
 316                     item = dict()
 317                     labels = dict()
 318                     item["name"] = metric
 319                     labels["name"] = nodes["name"]
 320                     labels["reason"] = nodes["reason"]
 321                     labels["severity"] = nodes["severity"]
 322                     try:
 323                         labels["thread_name"] = thread["thread_name"]
 324                         labels["thread_id"] = thread["thread_id"]
 325                     except UnboundLocalError:
 326                         labels["thread_name"] = "vpp_main"
 327                         labels["thread_id"] = "0"
 328                     item["labels"] = labels
 329                     item["value"] = nodes["count"]
 330                     self.serializer.serialize(
 331                         metric=metric, labels=labels, item=item
 332                     )
 333
 334     def vppctl_show_perfmon_statistics(self, reply):
 335         """
 336         Parse the perfmon output.
 337
 338         :param reply: VPP reply.
 339         :type reply: str
 340         """
 341         def perfmon_threads(reply, regex_threads):
 342             for line in reply.splitlines():
 343                 if fullmatch(regex_threads, line):
 344                     threads = fullmatch(regex_threads, line).groupdict()
 345                     for metric in self.serializer.metric_registry:
 346                         item = dict()
 347                         labels = dict()
 348                         item["name"] = metric
 349                         labels["name"] = threads["thread_name"]
 350                         labels["id"] = threads["thread_id"]
 351                         item["labels"] = labels
 352                         item["value"] = threads[metric]
 353                         self.serializer.serialize(
 354                             metric=metric, labels=labels, item=item
 355                         )
 356
 357         def perfmon_nodes(reply, regex_threads, regex_nodes):
 358             for line in reply.splitlines():
 359                 if fullmatch(regex_threads, line):
 360                     thread = fullmatch(regex_threads, line).groupdict()
 361                 if fullmatch(regex_nodes, line):
 362                     node = fullmatch(regex_nodes, line).groupdict()
 363                     for metric in self.serializer.metric_registry:
 364                         item = dict()
 365                         labels = dict()
 366                         item["name"] = metric
 367                         labels["node_name"] = node["node_name"]
 368                         labels["thread_name"] = thread["thread_name"]
 369                         labels["thread_id"] = thread["thread_id"]
 370                         item["labels"] = labels
 371                         item["value"] = node[metric]
 372                         self.serializer.serialize(
 373                             metric=metric, labels=labels, item=item
 374                         )
 375
 376         def perfmon_system(reply, regex_line):
 377             for line in reply.splitlines():
 378                 if fullmatch(regex_line, line):
 379                     name = fullmatch(regex_line, line).groupdict()
 380                     for metric in self.serializer.metric_registry:
 381                         item = dict()
 382                         labels = dict()
 383                         item["name"] = metric
 384                         labels["name"] = name["name"]
 385                         item["labels"] = labels
 386                         item["value"] = name[metric]
 387                         self.serializer.serialize(
 388                             metric=metric, labels=labels, item=item
 389                         )
 390
 391         reply = sub(r"\x1b[^m]*m", "", reply)
 392
 393         if fullmatch(M_PMB_CS_HEADER, reply.splitlines()[0]):
 394             perfmon_threads(reply, M_PMB_CS)
 395         if fullmatch(M_PMB_PF_HEADER, reply.splitlines()[0]):
 396             perfmon_threads(reply, M_PMB_PF)
 397         if fullmatch(M_PMB_IC_HEADER, reply.splitlines()[0]):
 398             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_IC_NODE)
 399         if fullmatch(M_PMB_CM_HEADER, reply.splitlines()[0]):
 400             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_CM_NODE)
 401         if fullmatch(M_PMB_LO_HEADER, reply.splitlines()[0]):
 402             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_LO_NODE)
 403         if fullmatch(M_PMB_BM_HEADER, reply.splitlines()[0]):
 404             perfmon_nodes(reply, M_PMB_THREAD, M_PMB_BM_NODE)
 405         if fullmatch(M_PMB_MB_HEADER, reply.splitlines()[0]):
 406             perfmon_system(reply, M_PMB_MB)
 407
 408     def vppctl_show_version(self, reply):
 409         """
 410         Parse the version output.
 411
 412         :param reply: VPP reply.
 413         :type reply: str
 414         """
 415         for metric in self.serializer.metric_registry:
 416             version = reply.split()[1]
 417             item = dict()
 418             labels = dict()
 419             item["name"] = metric
 420             labels["version"] = version
 421             item["labels"] = labels
 422             item["value"] = {}
 423             self.serializer.serialize(
 424                 metric=metric, labels=labels, item=item
 425             )