test/run.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (c) 2022 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 #
  16 # Build the Virtual Environment & run VPP unit tests
  17
  18 import argparse
  19 import glob
  20 import logging
  21 import os
  22 from pathlib import Path
  23 import signal
  24 from subprocess import Popen, PIPE, STDOUT, call
  25 import sys
  26 import time
  27 import venv
  28 import datetime
  29 import re
  30
  31
  32 # Required Std. Path Variables
  33 test_dir = os.path.dirname(os.path.realpath(__file__))
  34 ws_root = os.path.dirname(test_dir)
  35 build_root = os.path.join(ws_root, "build-root")
  36 venv_dir = os.path.join(build_root, "test", "venv")
  37 venv_bin_dir = os.path.join(venv_dir, "bin")
  38 venv_lib_dir = os.path.join(venv_dir, "lib")
  39 venv_run_dir = os.path.join(venv_dir, "run")
  40 venv_install_done = os.path.join(venv_run_dir, "venv_install.done")
  41 papi_python_src_dir = os.path.join(ws_root, "src", "vpp-api", "python")
  42
  43 # Path Variables Set after VPP Build/Install
  44 vpp_build_dir = vpp_install_path = vpp_bin = vpp_lib = vpp_lib64 = None
  45 vpp_plugin_path = vpp_test_plugin_path = ld_library_path = None
  46
  47 # Pip version pinning
  48 pip_version = "22.0.4"
  49 pip_tools_version = "6.6.0"
  50
  51 # Compiled pip requirements file
  52 pip_compiled_requirements_file = os.path.join(test_dir, "requirements-3.txt")
  53
  54 # Gracefully exit after executing cleanup scripts
  55 # upon receiving a SIGINT or SIGTERM
  56 def handler(signum, frame):
  57     print("Received Signal {0}".format(signum))
  58     post_vm_test_run()
  59
  60
  61 signal.signal(signal.SIGINT, handler)
  62 signal.signal(signal.SIGTERM, handler)
  63
  64
  65 def show_progress(stream, exclude_pattern=None):
  66     """
  67     Read lines from a subprocess stdout/stderr streams and write
  68     to sys.stdout & the logfile
  69
  70     arguments:
  71     stream - subprocess stdout or stderr data stream
  72     exclude_pattern - lines matching this reg-ex will be excluded
  73                       from stdout.
  74     """
  75     while True:
  76         s = stream.readline()
  77         if not s:
  78             break
  79         data = s.decode("utf-8")
  80         # Filter the annoying SIGTERM signal from the output when VPP is
  81         # terminated after a test run
  82         if "SIGTERM" not in data:
  83             if exclude_pattern is not None:
  84                 if bool(re.search(exclude_pattern, data)) is False:
  85                     sys.stdout.write(data)
  86             else:
  87                 sys.stdout.write(data)
  88             logging.debug(data)
  89         sys.stdout.flush()
  90     stream.close()
  91
  92
  93 class ExtendedEnvBuilder(venv.EnvBuilder):
  94     """
  95     1. Builds a Virtual Environment for running VPP unit tests
  96     2. Installs all necessary scripts, pkgs & patches into the vEnv
  97          - python3, pip, pip-tools, papi, scapy patches &
  98            test-requirement pkgs
  99     """
 100
 101     def __init__(self, *args, **kwargs):
 102         super().__init__(*args, **kwargs)
 103
 104     def post_setup(self, context):
 105         """
 106         Setup all packages that need to be pre-installed into the venv
 107         prior to running VPP unit tests.
 108
 109         :param context: The context of the virtual environment creation
 110                         request being processed.
 111         """
 112         os.environ["VIRTUAL_ENV"] = context.env_dir
 113         os.environ[
 114             "CUSTOM_COMPILE_COMMAND"
 115         ] = "make test-refresh-deps (or update requirements.txt)"
 116         # Set the venv python executable & binary install path
 117         env_exe = context.env_exe
 118         bin_path = context.bin_path
 119         # Packages/requirements to be installed in the venv
 120         # [python-module, cmdline-args, package-name_or_requirements-file-name]
 121         test_req = [
 122             ["pip", "install", "pip===%s" % pip_version],
 123             ["pip", "install", "pip-tools===%s" % pip_tools_version],
 124             ["piptools", "sync", pip_compiled_requirements_file],
 125             ["pip", "install", "-e", papi_python_src_dir],
 126         ]
 127         for req in test_req:
 128             args = [env_exe, "-m"]
 129             args.extend(req)
 130             print(args)
 131             p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=bin_path)
 132             show_progress(p.stdout)
 133         self.pip_patch()
 134
 135     def pip_patch(self):
 136         """
 137         Apply scapy patch files
 138         """
 139         scapy_patch_dir = Path(os.path.join(test_dir, "patches", "scapy-2.4.3"))
 140         scapy_source_dir = glob.glob(
 141             os.path.join(venv_lib_dir, "python3.*", "site-packages")
 142         )[0]
 143         for f in scapy_patch_dir.iterdir():
 144             print("Applying patch: {}".format(os.path.basename(str(f))))
 145             args = ["patch", "--forward", "-p1", "-d", scapy_source_dir, "-i", str(f)]
 146             print(args)
 147             p = Popen(args, stdout=PIPE, stderr=STDOUT)
 148             show_progress(p.stdout)
 149
 150
 151 # Build VPP Release/Debug binaries
 152 def build_vpp(debug=True, release=False):
 153     """
 154     Install VPP Release(if release=True) or Debug(if debug=True) Binaries.
 155
 156     Default is to build the debug binaries.
 157     """
 158     global vpp_build_dir, vpp_install_path, vpp_bin, vpp_lib, vpp_lib64
 159     global vpp_plugin_path, vpp_test_plugin_path, ld_library_path
 160     if debug:
 161         print("Building VPP debug binaries")
 162         args = ["make", "build"]
 163         build = "build-vpp_debug-native"
 164         install = "install-vpp_debug-native"
 165     elif release:
 166         print("Building VPP release binaries")
 167         args = ["make", "build-release"]
 168         build = "build-vpp-native"
 169         install = "install-vpp-native"
 170     p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=ws_root)
 171     show_progress(p.stdout)
 172     vpp_build_dir = os.path.join(build_root, build)
 173     vpp_install_path = os.path.join(build_root, install)
 174     vpp_bin = os.path.join(vpp_install_path, "vpp", "bin", "vpp")
 175     vpp_lib = os.path.join(vpp_install_path, "vpp", "lib")
 176     vpp_lib64 = os.path.join(vpp_install_path, "vpp", "lib64")
 177     vpp_plugin_path = (
 178         os.path.join(vpp_lib, "vpp_plugins")
 179         + ":"
 180         + os.path.join(vpp_lib64, "vpp_plugins")
 181     )
 182     vpp_test_plugin_path = (
 183         os.path.join(vpp_lib, "vpp_api_test_plugins")
 184         + ":"
 185         + os.path.join(vpp_lib64, "vpp_api_test_plugins")
 186     )
 187     ld_library_path = os.path.join(vpp_lib) + ":" + os.path.join(vpp_lib64)
 188
 189
 190 # Environment Vars required by the test framework,
 191 # papi_provider & unittests
 192 def set_environ():
 193     os.environ["WS_ROOT"] = ws_root
 194     os.environ["BR"] = build_root
 195     os.environ["VENV_PATH"] = venv_dir
 196     os.environ["VENV_BIN"] = venv_bin_dir
 197     os.environ["RND_SEED"] = str(time.time())
 198     os.environ["VPP_BUILD_DIR"] = vpp_build_dir
 199     os.environ["VPP_BIN"] = vpp_bin
 200     os.environ["VPP_PLUGIN_PATH"] = vpp_plugin_path
 201     os.environ["VPP_TEST_PLUGIN_PATH"] = vpp_test_plugin_path
 202     os.environ["VPP_INSTALL_PATH"] = vpp_install_path
 203     os.environ["LD_LIBRARY_PATH"] = ld_library_path
 204     os.environ["FAILED_DIR"] = "/tmp/vpp-failed-unittests/"
 205     if not os.environ.get("TEST_JOBS"):
 206         os.environ["TEST_JOBS"] = "1"
 207
 208
 209 # Runs a test inside a spawned QEMU VM
 210 # If a kernel image is not provided, a linux-image-kvm image is
 211 # downloaded to the test_data_dir
 212 def vm_test_runner(test_name, kernel_image, test_data_dir, cpu_mask, mem, jobs="auto"):
 213     script = os.path.join(test_dir, "scripts", "run_vpp_in_vm.sh")
 214     os.environ["TEST_JOBS"] = str(jobs)
 215     p = Popen(
 216         [script, test_name, kernel_image, test_data_dir, cpu_mask, mem],
 217         stdout=PIPE,
 218         cwd=ws_root,
 219     )
 220     # Show only the test result without clobbering the stdout.
 221     # The VM console displays VPP stderr & Linux IPv6 netdev change
 222     # messages, which is logged by default and can be excluded.
 223     exclude_pattern = r"vpp\[\d+\]:|ADDRCONF\(NETDEV_CHANGE\):"
 224     show_progress(p.stdout, exclude_pattern)
 225     post_vm_test_run()
 226
 227
 228 def post_vm_test_run():
 229     # Revert the ownership of certain directories from root to the
 230     # original user after running in QEMU
 231     print("Running post test cleanup tasks")
 232     dirs = ["/tmp/vpp-failed-unittests", os.path.join(ws_root, "test", "__pycache__")]
 233     dirs.extend(glob.glob("/tmp/vpp-unittest-*"))
 234     dirs.extend(glob.glob("/tmp/api_post_mortem.*"))
 235     user = os.getlogin()
 236     for dir in dirs:
 237         if os.path.exists(dir) and Path(dir).owner() != user:
 238             cmd = ["sudo", "chown", "-R", "{0}:{0}".format(user), dir]
 239             p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
 240             show_progress(p.stdout)
 241
 242
 243 def build_venv():
 244     # Builds a virtual env containing all the required packages and patches
 245     # for running VPP unit tests
 246     if not os.path.exists(venv_install_done):
 247         env_builder = ExtendedEnvBuilder(clear=True, with_pip=True)
 248         print("Creating a vEnv for running VPP unit tests in {}".format(venv_dir))
 249         env_builder.create(venv_dir)
 250         # Write state to the venv run dir
 251         Path(venv_run_dir).mkdir(exist_ok=True)
 252         Path(venv_install_done).touch()
 253
 254
 255 def expand_mix_string(s):
 256     # Returns an expanded string computed from a mixrange string (s)
 257     # E.g: If param s = '5-8,10,11' returns '5,6,7,8,10,11'
 258     result = []
 259     for val in s.split(","):
 260         if "-" in val:
 261             start, end = val.split("-")
 262             result.extend(list(range(int(start), int(end) + 1)))
 263         else:
 264             result.append(int(val))
 265     return ",".join(str(i) for i in set(result))
 266
 267
 268 def set_logging(test_data_dir, test_name):
 269     Path(test_data_dir).mkdir(exist_ok=True)
 270     log_file = "vm_{0}_{1}.log".format(test_name, str(time.time())[-5:])
 271     filename = "{0}/{1}".format(test_data_dir, log_file)
 272     Path(filename).touch()
 273     logging.basicConfig(filename=filename, level=logging.DEBUG)
 274
 275
 276 def run_tests_in_venv(
 277     test,
 278     jobs,
 279     log_dir,
 280     socket_dir="",
 281     running_vpp=False,
 282     extended=False,
 283 ):
 284     """Runs tests in the virtual environment set by venv_dir.
 285
 286     Arguments:
 287     test: Name of the test to run
 288     jobs: Maximum concurrent test jobs
 289     log_dir: Directory location for storing log files
 290     socket_dir: Use running VPP's socket files
 291     running_vpp: True if tests are run against a running VPP
 292     extended: Run extended tests
 293     """
 294     script = os.path.join(test_dir, "scripts", "run.sh")
 295     args = [
 296         f"--venv-dir={venv_dir}",
 297         f"--vpp-ws-dir={ws_root}",
 298         f"--socket-dir={socket_dir}",
 299         f"--filter={test}",
 300         f"--jobs={jobs}",
 301         f"--log-dir={log_dir}",
 302         f"--tmp-dir={log_dir}",
 303         f"--cache-vpp-output",
 304     ]
 305     if running_vpp:
 306         args = args + [f"--use-running-vpp"]
 307     if extended:
 308         args = args + [f"--extended"]
 309     print(f"Running script: {script} " f"{' '.join(args)}")
 310     process_args = [script] + args
 311     call(process_args)
 312
 313
 314 if __name__ == "__main__":
 315     # Build a Virtual Environment for running tests on host & QEMU
 316     # (TODO): Create a single config object by merging the below args with
 317     # config.py after gathering dev use-cases.
 318     parser = argparse.ArgumentParser(
 319         description="Run VPP Unit Tests", formatter_class=argparse.RawTextHelpFormatter
 320     )
 321     parser.add_argument(
 322         "--vm",
 323         dest="vm",
 324         required=False,
 325         action="store_true",
 326         help="Run Test Inside a QEMU VM",
 327     )
 328     parser.add_argument(
 329         "--debug",
 330         dest="debug",
 331         required=False,
 332         default=True,
 333         action="store_true",
 334         help="Run Tests on Debug Build",
 335     )
 336     parser.add_argument(
 337         "--release",
 338         dest="release",
 339         required=False,
 340         default=False,
 341         action="store_true",
 342         help="Run Tests on release Build",
 343     )
 344     parser.add_argument(
 345         "-t",
 346         "--test",
 347         dest="test_name",
 348         required=False,
 349         action="store",
 350         default="",
 351         help="Test Name or Test filter",
 352     )
 353     parser.add_argument(
 354         "--vm-kernel-image",
 355         dest="kernel_image",
 356         required=False,
 357         action="store",
 358         default="",
 359         help="Kernel Image Selection to Boot",
 360     )
 361     parser.add_argument(
 362         "--vm-cpu-list",
 363         dest="vm_cpu_list",
 364         required=False,
 365         action="store",
 366         default="5-8",
 367         help="Set CPU Affinity\n"
 368         "E.g. 5-7,10 will schedule on processors "
 369         "#5, #6, #7 and #10. (Default: 5-8)",
 370     )
 371     parser.add_argument(
 372         "--vm-mem",
 373         dest="vm_mem",
 374         required=False,
 375         action="store",
 376         default="2",
 377         help="Guest Memory in Gibibytes\n" "E.g. 4 (Default: 2)",
 378     )
 379     parser.add_argument(
 380         "--log-dir",
 381         action="store",
 382         default=os.path.abspath(f"./test-run-{datetime.date.today()}"),
 383         help="directory where to store directories "
 384         "containing log files (default: ./test-run-YYYY-MM-DD)",
 385     )
 386     parser.add_argument(
 387         "--jobs",
 388         action="store",
 389         default="auto",
 390         help="maximum concurrent test jobs",
 391     )
 392     parser.add_argument(
 393         "-r",
 394         "--use-running-vpp",
 395         dest="running_vpp",
 396         required=False,
 397         action="store_true",
 398         default=False,
 399         help="Runs tests against a running VPP.",
 400     )
 401     parser.add_argument(
 402         "-d",
 403         "--socket-dir",
 404         dest="socket_dir",
 405         required=False,
 406         action="store",
 407         default="",
 408         help="Relative or absolute path of running VPP's socket directory "
 409         "containing api.sock & stats.sock files.\n"
 410         "Default: /var/run/vpp if VPP is started as the root user, else "
 411         "/var/run/user/${uid}/vpp.",
 412     )
 413     parser.add_argument(
 414         "-e",
 415         "--extended",
 416         dest="extended",
 417         required=False,
 418         action="store_true",
 419         default=False,
 420         help="Run extended tests.",
 421     )
 422     args = parser.parse_args()
 423     vm_tests = False
 424     # Enable VM tests
 425     if args.vm and args.test_name:
 426         test_data_dir = "/tmp/vpp-vm-tests"
 427         set_logging(test_data_dir, args.test_name)
 428         vm_tests = True
 429     elif args.vm and not args.test_name:
 430         print("Error: The --test argument must be set for running VM tests")
 431         sys.exit(1)
 432     build_venv()
 433     # Build VPP release or debug binaries
 434     debug = False if args.release else True
 435     build_vpp(debug, args.release)
 436     set_environ()
 437     if args.running_vpp:
 438         print("Tests will be run against a running VPP..")
 439     elif not vm_tests:
 440         print("Tests will be run by spawning a new VPP instance..")
 441     # Run tests against a running VPP or a new instance of VPP
 442     if not vm_tests:
 443         run_tests_in_venv(
 444             test=args.test_name,
 445             jobs=args.jobs,
 446             log_dir=args.log_dir,
 447             socket_dir=args.socket_dir,
 448             running_vpp=args.running_vpp,
 449             extended=args.extended,
 450         )
 451     # Run tests against a VPP inside a VM
 452     else:
 453         print("Running VPP unit test(s):{0} inside a QEMU VM".format(args.test_name))
 454         # Check Available CPUs & Usable Memory
 455         cpus = expand_mix_string(args.vm_cpu_list)
 456         num_cpus, usable_cpus = (len(cpus.split(",")), len(os.sched_getaffinity(0)))
 457         if num_cpus > usable_cpus:
 458             print(f"Error:# of CPUs:{num_cpus} > Avail CPUs:{usable_cpus}")
 459             sys.exit(1)
 460         avail_mem = int(os.popen("free -t -g").readlines()[-1].split()[-1])
 461         if int(args.vm_mem) > avail_mem:
 462             print(f"Error: Mem Size:{args.vm_mem}G > Avail Mem:{avail_mem}G")
 463             sys.exit(1)
 464         vm_test_runner(
 465             args.test_name,
 466             args.kernel_image,
 467             test_data_dir,
 468             cpus,
 469             f"{args.vm_mem}G",
 470             args.jobs,
 471         )