test/run.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (c) 2022 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 #
  16 # Build the Virtual Environment & run VPP unit tests
  17
  18 import argparse
  19 import glob
  20 import logging
  21 import os
  22 from pathlib import Path
  23 import signal
  24 from subprocess import Popen, PIPE, STDOUT, call
  25 import sys
  26 import time
  27 import venv
  28 import datetime
  29 import re
  30
  31
  32 # Required Std. Path Variables
  33 test_dir = os.path.dirname(os.path.realpath(__file__))
  34 ws_root = os.path.dirname(test_dir)
  35 build_root = os.path.join(ws_root, "build-root")
  36 venv_dir = os.path.join(build_root, "test", "venv")
  37 venv_bin_dir = os.path.join(venv_dir, "bin")
  38 venv_lib_dir = os.path.join(venv_dir, "lib")
  39 venv_run_dir = os.path.join(venv_dir, "run")
  40 venv_install_done = os.path.join(venv_run_dir, "venv_install.done")
  41 papi_python_src_dir = os.path.join(ws_root, "src", "vpp-api", "python")
  42
  43 # Path Variables Set after VPP Build/Install
  44 vpp_build_dir = vpp_install_path = vpp_bin = vpp_lib = vpp_lib64 = None
  45 vpp_plugin_path = vpp_test_plugin_path = ld_library_path = None
  46
  47 # Pip version pinning
  48 pip_version = "22.0.4"
  49 pip_tools_version = "6.6.0"
  50
  51 # Compiled pip requirements file
  52 pip_compiled_requirements_file = os.path.join(test_dir, "requirements-3.txt")
  53
  54
  55 # Gracefully exit after executing cleanup scripts
  56 # upon receiving a SIGINT or SIGTERM
  57 def handler(signum, frame):
  58     print("Received Signal {0}".format(signum))
  59     post_vm_test_run()
  60
  61
  62 signal.signal(signal.SIGINT, handler)
  63 signal.signal(signal.SIGTERM, handler)
  64
  65
  66 def show_progress(stream, exclude_pattern=None):
  67     """
  68     Read lines from a subprocess stdout/stderr streams and write
  69     to sys.stdout & the logfile
  70
  71     arguments:
  72     stream - subprocess stdout or stderr data stream
  73     exclude_pattern - lines matching this reg-ex will be excluded
  74                       from stdout.
  75     """
  76     while True:
  77         s = stream.readline()
  78         if not s:
  79             break
  80         data = s.decode("utf-8")
  81         # Filter the annoying SIGTERM signal from the output when VPP is
  82         # terminated after a test run
  83         if "SIGTERM" not in data:
  84             if exclude_pattern is not None:
  85                 if bool(re.search(exclude_pattern, data)) is False:
  86                     sys.stdout.write(data)
  87             else:
  88                 sys.stdout.write(data)
  89             logging.debug(data)
  90         sys.stdout.flush()
  91     stream.close()
  92
  93
  94 class ExtendedEnvBuilder(venv.EnvBuilder):
  95     """
  96     1. Builds a Virtual Environment for running VPP unit tests
  97     2. Installs all necessary scripts, pkgs & patches into the vEnv
  98          - python3, pip, pip-tools, papi, scapy patches &
  99            test-requirement pkgs
 100     """
 101
 102     def __init__(self, *args, **kwargs):
 103         super().__init__(*args, **kwargs)
 104
 105     def post_setup(self, context):
 106         """
 107         Setup all packages that need to be pre-installed into the venv
 108         prior to running VPP unit tests.
 109
 110         :param context: The context of the virtual environment creation
 111                         request being processed.
 112         """
 113         os.environ["VIRTUAL_ENV"] = context.env_dir
 114         os.environ[
 115             "CUSTOM_COMPILE_COMMAND"
 116         ] = "make test-refresh-deps (or update requirements.txt)"
 117         # Set the venv python executable & binary install path
 118         env_exe = context.env_exe
 119         bin_path = context.bin_path
 120         # Packages/requirements to be installed in the venv
 121         # [python-module, cmdline-args, package-name_or_requirements-file-name]
 122         test_req = [
 123             ["pip", "install", "pip===%s" % pip_version],
 124             ["pip", "install", "pip-tools===%s" % pip_tools_version],
 125             ["piptools", "sync", pip_compiled_requirements_file],
 126             ["pip", "install", "-e", papi_python_src_dir],
 127         ]
 128         for req in test_req:
 129             args = [env_exe, "-m"]
 130             args.extend(req)
 131             print(args)
 132             p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=bin_path)
 133             show_progress(p.stdout)
 134         self.pip_patch()
 135
 136     def pip_patch(self):
 137         """
 138         Apply scapy patch files
 139         """
 140         scapy_patch_dir = Path(os.path.join(test_dir, "patches", "scapy-2.4.3"))
 141         scapy_source_dir = glob.glob(
 142             os.path.join(venv_lib_dir, "python3.*", "site-packages")
 143         )[0]
 144         for f in scapy_patch_dir.iterdir():
 145             print("Applying patch: {}".format(os.path.basename(str(f))))
 146             args = ["patch", "--forward", "-p1", "-d", scapy_source_dir, "-i", str(f)]
 147             print(args)
 148             p = Popen(args, stdout=PIPE, stderr=STDOUT)
 149             show_progress(p.stdout)
 150
 151
 152 # Build VPP Release/Debug binaries
 153 def build_vpp(debug=True, release=False):
 154     """
 155     Install VPP Release(if release=True) or Debug(if debug=True) Binaries.
 156
 157     Default is to build the debug binaries.
 158     """
 159     global vpp_build_dir, vpp_install_path, vpp_bin, vpp_lib, vpp_lib64
 160     global vpp_plugin_path, vpp_test_plugin_path, ld_library_path
 161     if debug:
 162         print("Building VPP debug binaries")
 163         args = ["make", "build"]
 164         build = "build-vpp_debug-native"
 165         install = "install-vpp_debug-native"
 166     elif release:
 167         print("Building VPP release binaries")
 168         args = ["make", "build-release"]
 169         build = "build-vpp-native"
 170         install = "install-vpp-native"
 171     p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=ws_root)
 172     show_progress(p.stdout)
 173     vpp_build_dir = os.path.join(build_root, build)
 174     vpp_install_path = os.path.join(build_root, install)
 175     vpp_bin = os.path.join(vpp_install_path, "vpp", "bin", "vpp")
 176     vpp_lib = os.path.join(vpp_install_path, "vpp", "lib")
 177     vpp_lib64 = os.path.join(vpp_install_path, "vpp", "lib64")
 178     vpp_plugin_path = (
 179         os.path.join(vpp_lib, "vpp_plugins")
 180         + ":"
 181         + os.path.join(vpp_lib64, "vpp_plugins")
 182     )
 183     vpp_test_plugin_path = (
 184         os.path.join(vpp_lib, "vpp_api_test_plugins")
 185         + ":"
 186         + os.path.join(vpp_lib64, "vpp_api_test_plugins")
 187     )
 188     ld_library_path = os.path.join(vpp_lib) + ":" + os.path.join(vpp_lib64)
 189
 190
 191 # Environment Vars required by the test framework,
 192 # papi_provider & unittests
 193 def set_environ():
 194     os.environ["WS_ROOT"] = ws_root
 195     os.environ["BR"] = build_root
 196     os.environ["VENV_PATH"] = venv_dir
 197     os.environ["VENV_BIN"] = venv_bin_dir
 198     os.environ["RND_SEED"] = str(time.time())
 199     os.environ["VPP_BUILD_DIR"] = vpp_build_dir
 200     os.environ["VPP_BIN"] = vpp_bin
 201     os.environ["VPP_PLUGIN_PATH"] = vpp_plugin_path
 202     os.environ["VPP_TEST_PLUGIN_PATH"] = vpp_test_plugin_path
 203     os.environ["VPP_INSTALL_PATH"] = vpp_install_path
 204     os.environ["LD_LIBRARY_PATH"] = ld_library_path
 205     os.environ["FAILED_DIR"] = "/tmp/vpp-failed-unittests/"
 206     if not os.environ.get("TEST_JOBS"):
 207         os.environ["TEST_JOBS"] = "1"
 208
 209
 210 # Runs a test inside a spawned QEMU VM
 211 # If a kernel image is not provided, a linux-image-kvm image is
 212 # downloaded to the test_data_dir
 213 def vm_test_runner(test_name, kernel_image, test_data_dir, cpu_mask, mem, jobs="auto"):
 214     script = os.path.join(test_dir, "scripts", "run_vpp_in_vm.sh")
 215     os.environ["TEST_JOBS"] = str(jobs)
 216     p = Popen(
 217         [script, test_name, kernel_image, test_data_dir, cpu_mask, mem],
 218         stdout=PIPE,
 219         cwd=ws_root,
 220     )
 221     # Show only the test result without clobbering the stdout.
 222     # The VM console displays VPP stderr & Linux IPv6 netdev change
 223     # messages, which is logged by default and can be excluded.
 224     exclude_pattern = r"vpp\[\d+\]:|ADDRCONF\(NETDEV_CHANGE\):"
 225     show_progress(p.stdout, exclude_pattern)
 226     post_vm_test_run()
 227
 228
 229 def post_vm_test_run():
 230     # Revert the ownership of certain directories from root to the
 231     # original user after running in QEMU
 232     print("Running post test cleanup tasks")
 233     dirs = ["/tmp/vpp-failed-unittests", os.path.join(ws_root, "test", "__pycache__")]
 234     dirs.extend(glob.glob("/tmp/vpp-unittest-*"))
 235     dirs.extend(glob.glob("/tmp/api_post_mortem.*"))
 236     user = os.getlogin()
 237     for dir in dirs:
 238         if os.path.exists(dir) and Path(dir).owner() != user:
 239             cmd = ["sudo", "chown", "-R", "{0}:{0}".format(user), dir]
 240             p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
 241             show_progress(p.stdout)
 242
 243
 244 def build_venv():
 245     # Builds a virtual env containing all the required packages and patches
 246     # for running VPP unit tests
 247     if not os.path.exists(venv_install_done):
 248         env_builder = ExtendedEnvBuilder(clear=True, with_pip=True)
 249         print("Creating a vEnv for running VPP unit tests in {}".format(venv_dir))
 250         env_builder.create(venv_dir)
 251         # Write state to the venv run dir
 252         Path(venv_run_dir).mkdir(exist_ok=True)
 253         Path(venv_install_done).touch()
 254
 255
 256 def expand_mix_string(s):
 257     # Returns an expanded string computed from a mixrange string (s)
 258     # E.g: If param s = '5-8,10,11' returns '5,6,7,8,10,11'
 259     result = []
 260     for val in s.split(","):
 261         if "-" in val:
 262             start, end = val.split("-")
 263             result.extend(list(range(int(start), int(end) + 1)))
 264         else:
 265             result.append(int(val))
 266     return ",".join(str(i) for i in set(result))
 267
 268
 269 def set_logging(test_data_dir, test_name):
 270     Path(test_data_dir).mkdir(exist_ok=True)
 271     log_file = "vm_{0}_{1}.log".format(test_name, str(time.time())[-5:])
 272     filename = "{0}/{1}".format(test_data_dir, log_file)
 273     Path(filename).touch()
 274     logging.basicConfig(filename=filename, level=logging.DEBUG)
 275
 276
 277 def run_tests_in_venv(
 278     test,
 279     jobs,
 280     log_dir,
 281     socket_dir="",
 282     running_vpp=False,
 283     extended=False,
 284 ):
 285     """Runs tests in the virtual environment set by venv_dir.
 286
 287     Arguments:
 288     test: Name of the test to run
 289     jobs: Maximum concurrent test jobs
 290     log_dir: Directory location for storing log files
 291     socket_dir: Use running VPP's socket files
 292     running_vpp: True if tests are run against a running VPP
 293     extended: Run extended tests
 294     """
 295     script = os.path.join(test_dir, "scripts", "run.sh")
 296     args = [
 297         f"--venv-dir={venv_dir}",
 298         f"--vpp-ws-dir={ws_root}",
 299         f"--socket-dir={socket_dir}",
 300         f"--filter={test}",
 301         f"--jobs={jobs}",
 302         f"--log-dir={log_dir}",
 303         f"--tmp-dir={log_dir}",
 304         f"--cache-vpp-output",
 305     ]
 306     if running_vpp:
 307         args = args + [f"--use-running-vpp"]
 308     if extended:
 309         args = args + [f"--extended"]
 310     print(f"Running script: {script} " f"{' '.join(args)}")
 311     process_args = [script] + args
 312     call(process_args)
 313
 314
 315 if __name__ == "__main__":
 316     # Build a Virtual Environment for running tests on host & QEMU
 317     # (TODO): Create a single config object by merging the below args with
 318     # config.py after gathering dev use-cases.
 319     parser = argparse.ArgumentParser(
 320         description="Run VPP Unit Tests", formatter_class=argparse.RawTextHelpFormatter
 321     )
 322     parser.add_argument(
 323         "--vm",
 324         dest="vm",
 325         required=False,
 326         action="store_true",
 327         help="Run Test Inside a QEMU VM",
 328     )
 329     parser.add_argument(
 330         "--debug",
 331         dest="debug",
 332         required=False,
 333         default=True,
 334         action="store_true",
 335         help="Run Tests on Debug Build",
 336     )
 337     parser.add_argument(
 338         "--release",
 339         dest="release",
 340         required=False,
 341         default=False,
 342         action="store_true",
 343         help="Run Tests on release Build",
 344     )
 345     parser.add_argument(
 346         "-t",
 347         "--test",
 348         dest="test_name",
 349         required=False,
 350         action="store",
 351         default="",
 352         help="Test Name or Test filter",
 353     )
 354     parser.add_argument(
 355         "--vm-kernel-image",
 356         dest="kernel_image",
 357         required=False,
 358         action="store",
 359         default="",
 360         help="Kernel Image Selection to Boot",
 361     )
 362     parser.add_argument(
 363         "--vm-cpu-list",
 364         dest="vm_cpu_list",
 365         required=False,
 366         action="store",
 367         default="5-8",
 368         help="Set CPU Affinity\n"
 369         "E.g. 5-7,10 will schedule on processors "
 370         "#5, #6, #7 and #10. (Default: 5-8)",
 371     )
 372     parser.add_argument(
 373         "--vm-mem",
 374         dest="vm_mem",
 375         required=False,
 376         action="store",
 377         default="2",
 378         help="Guest Memory in Gibibytes\n" "E.g. 4 (Default: 2)",
 379     )
 380     parser.add_argument(
 381         "--log-dir",
 382         action="store",
 383         default=os.path.abspath(f"./test-run-{datetime.date.today()}"),
 384         help="directory where to store directories "
 385         "containing log files (default: ./test-run-YYYY-MM-DD)",
 386     )
 387     parser.add_argument(
 388         "--jobs",
 389         action="store",
 390         default="auto",
 391         help="maximum concurrent test jobs",
 392     )
 393     parser.add_argument(
 394         "-r",
 395         "--use-running-vpp",
 396         dest="running_vpp",
 397         required=False,
 398         action="store_true",
 399         default=False,
 400         help="Runs tests against a running VPP.",
 401     )
 402     parser.add_argument(
 403         "-d",
 404         "--socket-dir",
 405         dest="socket_dir",
 406         required=False,
 407         action="store",
 408         default="",
 409         help="Relative or absolute path of running VPP's socket directory "
 410         "containing api.sock & stats.sock files.\n"
 411         "Default: /var/run/vpp if VPP is started as the root user, else "
 412         "/var/run/user/${uid}/vpp.",
 413     )
 414     parser.add_argument(
 415         "-e",
 416         "--extended",
 417         dest="extended",
 418         required=False,
 419         action="store_true",
 420         default=False,
 421         help="Run extended tests.",
 422     )
 423     args = parser.parse_args()
 424     vm_tests = False
 425     # Enable VM tests
 426     if args.vm and args.test_name:
 427         test_data_dir = "/tmp/vpp-vm-tests"
 428         set_logging(test_data_dir, args.test_name)
 429         vm_tests = True
 430     elif args.vm and not args.test_name:
 431         print("Error: The --test argument must be set for running VM tests")
 432         sys.exit(1)
 433     build_venv()
 434     # Build VPP release or debug binaries
 435     debug = False if args.release else True
 436     build_vpp(debug, args.release)
 437     set_environ()
 438     if args.running_vpp:
 439         print("Tests will be run against a running VPP..")
 440     elif not vm_tests:
 441         print("Tests will be run by spawning a new VPP instance..")
 442     # Run tests against a running VPP or a new instance of VPP
 443     if not vm_tests:
 444         run_tests_in_venv(
 445             test=args.test_name,
 446             jobs=args.jobs,
 447             log_dir=args.log_dir,
 448             socket_dir=args.socket_dir,
 449             running_vpp=args.running_vpp,
 450             extended=args.extended,
 451         )
 452     # Run tests against a VPP inside a VM
 453     else:
 454         print("Running VPP unit test(s):{0} inside a QEMU VM".format(args.test_name))
 455         # Check Available CPUs & Usable Memory
 456         cpus = expand_mix_string(args.vm_cpu_list)
 457         num_cpus, usable_cpus = (len(cpus.split(",")), len(os.sched_getaffinity(0)))
 458         if num_cpus > usable_cpus:
 459             print(f"Error:# of CPUs:{num_cpus} > Avail CPUs:{usable_cpus}")
 460             sys.exit(1)
 461         avail_mem = int(os.popen("free -t -g").readlines()[-1].split()[-1])
 462         if int(args.vm_mem) > avail_mem:
 463             print(f"Error: Mem Size:{args.vm_mem}G > Avail Mem:{avail_mem}G")
 464             sys.exit(1)
 465         vm_test_runner(
 466             args.test_name,
 467             args.kernel_image,
 468             test_data_dir,
 469             cpus,
 470             f"{args.vm_mem}G",
 471             args.jobs,
 472         )