test/run.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (c) 2022 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 #
  16 # Build the Virtual Environment & run VPP unit tests
  17
  18 import argparse
  19 import glob
  20 import logging
  21 import os
  22 from pathlib import Path
  23 import signal
  24 from subprocess import Popen, PIPE, STDOUT, call
  25 import sys
  26 import time
  27 import venv
  28 import datetime
  29 import re
  30
  31
  32 # Required Std. Path Variables
  33 test_dir = os.path.dirname(os.path.realpath(__file__))
  34 ws_root = os.path.dirname(test_dir)
  35 build_root = os.path.join(ws_root, "build-root")
  36 venv_dir = os.path.join(build_root, "test", "venv")
  37 venv_bin_dir = os.path.join(venv_dir, "bin")
  38 venv_lib_dir = os.path.join(venv_dir, "lib")
  39 venv_run_dir = os.path.join(venv_dir, "run")
  40 venv_install_done = os.path.join(venv_run_dir, "venv_install.done")
  41 papi_python_src_dir = os.path.join(ws_root, "src", "vpp-api", "python")
  42
  43 # Path Variables Set after VPP Build/Install
  44 vpp_build_dir = vpp_install_path = vpp_bin = vpp_lib = vpp_lib64 = None
  45 vpp_plugin_path = vpp_test_plugin_path = ld_library_path = None
  46
  47 # Pip version pinning
  48 pip_version = "22.0.4"
  49 pip_tools_version = "6.6.0"
  50
  51 # Test requirement files
  52 test_requirements_file = os.path.join(test_dir, "requirements.txt")
  53 # Auto-generated requirement file
  54 pip_compiled_requirements_file = os.path.join(test_dir, "requirements-3.txt")
  55
  56
  57 # Gracefully exit after executing cleanup scripts
  58 # upon receiving a SIGINT or SIGTERM
  59 def handler(signum, frame):
  60     print("Received Signal {0}".format(signum))
  61     post_vm_test_run()
  62
  63
  64 signal.signal(signal.SIGINT, handler)
  65 signal.signal(signal.SIGTERM, handler)
  66
  67
  68 def show_progress(stream, exclude_pattern=None):
  69     """
  70     Read lines from a subprocess stdout/stderr streams and write
  71     to sys.stdout & the logfile
  72
  73     arguments:
  74     stream - subprocess stdout or stderr data stream
  75     exclude_pattern - lines matching this reg-ex will be excluded
  76                       from stdout.
  77     """
  78     while True:
  79         s = stream.readline()
  80         if not s:
  81             break
  82         data = s.decode("utf-8")
  83         # Filter the annoying SIGTERM signal from the output when VPP is
  84         # terminated after a test run
  85         if "SIGTERM" not in data:
  86             if exclude_pattern is not None:
  87                 if bool(re.search(exclude_pattern, data)) is False:
  88                     sys.stdout.write(data)
  89             else:
  90                 sys.stdout.write(data)
  91             logging.debug(data)
  92         sys.stdout.flush()
  93     stream.close()
  94
  95
  96 class ExtendedEnvBuilder(venv.EnvBuilder):
  97     """
  98     1. Builds a Virtual Environment for running VPP unit tests
  99     2. Installs all necessary scripts, pkgs & patches into the vEnv
 100          - python3, pip, pip-tools, papi, scapy patches &
 101            test-requirement pkgs
 102     """
 103
 104     def __init__(self, *args, **kwargs):
 105         super().__init__(*args, **kwargs)
 106
 107     def post_setup(self, context):
 108         """
 109         Setup all packages that need to be pre-installed into the venv
 110         prior to running VPP unit tests.
 111
 112         :param context: The context of the virtual environment creation
 113                         request being processed.
 114         """
 115         os.environ["VIRTUAL_ENV"] = context.env_dir
 116         os.environ[
 117             "CUSTOM_COMPILE_COMMAND"
 118         ] = "make test-refresh-deps (or update requirements.txt)"
 119         # Cleanup previously auto-generated pip req. file
 120         try:
 121             os.unlink(pip_compiled_requirements_file)
 122         except OSError:
 123             pass
 124         # Set the venv python executable & binary install path
 125         env_exe = context.env_exe
 126         bin_path = context.bin_path
 127         # Packages/requirements to be installed in the venv
 128         # [python-module, cmdline-args, package-name_or_requirements-file-name]
 129         test_req = [
 130             ["pip", "install", "pip===%s" % pip_version],
 131             ["pip", "install", "pip-tools===%s" % pip_tools_version],
 132             [
 133                 "piptools",
 134                 "compile",
 135                 "-q",
 136                 "--generate-hashes",
 137                 test_requirements_file,
 138                 "--output-file",
 139                 pip_compiled_requirements_file,
 140             ],
 141             ["piptools", "sync", pip_compiled_requirements_file],
 142             ["pip", "install", "-e", papi_python_src_dir],
 143         ]
 144         for req in test_req:
 145             args = [env_exe, "-m"]
 146             args.extend(req)
 147             print(args)
 148             p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=bin_path)
 149             show_progress(p.stdout)
 150         self.pip_patch()
 151
 152     def pip_patch(self):
 153         """
 154         Apply scapy patch files
 155         """
 156         scapy_patch_dir = Path(os.path.join(test_dir, "patches", "scapy-2.4.3"))
 157         scapy_source_dir = glob.glob(
 158             os.path.join(venv_lib_dir, "python3.*", "site-packages")
 159         )[0]
 160         for f in scapy_patch_dir.iterdir():
 161             print("Applying patch: {}".format(os.path.basename(str(f))))
 162             args = ["patch", "--forward", "-p1", "-d", scapy_source_dir, "-i", str(f)]
 163             print(args)
 164             p = Popen(args, stdout=PIPE, stderr=STDOUT)
 165             show_progress(p.stdout)
 166
 167
 168 # Build VPP Release/Debug binaries
 169 def build_vpp(debug=True, release=False):
 170     """
 171     Install VPP Release(if release=True) or Debug(if debug=True) Binaries.
 172
 173     Default is to build the debug binaries.
 174     """
 175     global vpp_build_dir, vpp_install_path, vpp_bin, vpp_lib, vpp_lib64
 176     global vpp_plugin_path, vpp_test_plugin_path, ld_library_path
 177     if debug:
 178         print("Building VPP debug binaries")
 179         args = ["make", "build"]
 180         build = "build-vpp_debug-native"
 181         install = "install-vpp_debug-native"
 182     elif release:
 183         print("Building VPP release binaries")
 184         args = ["make", "build-release"]
 185         build = "build-vpp-native"
 186         install = "install-vpp-native"
 187     p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=ws_root)
 188     show_progress(p.stdout)
 189     vpp_build_dir = os.path.join(build_root, build)
 190     vpp_install_path = os.path.join(build_root, install)
 191     vpp_bin = os.path.join(vpp_install_path, "vpp", "bin", "vpp")
 192     vpp_lib = os.path.join(vpp_install_path, "vpp", "lib")
 193     vpp_lib64 = os.path.join(vpp_install_path, "vpp", "lib64")
 194     vpp_plugin_path = (
 195         os.path.join(vpp_lib, "vpp_plugins")
 196         + ":"
 197         + os.path.join(vpp_lib64, "vpp_plugins")
 198     )
 199     vpp_test_plugin_path = (
 200         os.path.join(vpp_lib, "vpp_api_test_plugins")
 201         + ":"
 202         + os.path.join(vpp_lib64, "vpp_api_test_plugins")
 203     )
 204     ld_library_path = os.path.join(vpp_lib) + ":" + os.path.join(vpp_lib64)
 205
 206
 207 # Environment Vars required by the test framework,
 208 # papi_provider & unittests
 209 def set_environ():
 210     os.environ["WS_ROOT"] = ws_root
 211     os.environ["BR"] = build_root
 212     os.environ["VENV_PATH"] = venv_dir
 213     os.environ["VENV_BIN"] = venv_bin_dir
 214     os.environ["RND_SEED"] = str(time.time())
 215     os.environ["VPP_BUILD_DIR"] = vpp_build_dir
 216     os.environ["VPP_BIN"] = vpp_bin
 217     os.environ["VPP_PLUGIN_PATH"] = vpp_plugin_path
 218     os.environ["VPP_TEST_PLUGIN_PATH"] = vpp_test_plugin_path
 219     os.environ["VPP_INSTALL_PATH"] = vpp_install_path
 220     os.environ["LD_LIBRARY_PATH"] = ld_library_path
 221     os.environ["FAILED_DIR"] = "/tmp/vpp-failed-unittests/"
 222     if not os.environ.get("TEST_JOBS"):
 223         os.environ["TEST_JOBS"] = "1"
 224
 225
 226 # Runs a test inside a spawned QEMU VM
 227 # If a kernel image is not provided, a linux-image-kvm image is
 228 # downloaded to the test_data_dir
 229 def vm_test_runner(test_name, kernel_image, test_data_dir, cpu_mask, mem, jobs="auto"):
 230     script = os.path.join(test_dir, "scripts", "run_vpp_in_vm.sh")
 231     os.environ["TEST_JOBS"] = str(jobs)
 232     p = Popen(
 233         [script, test_name, kernel_image, test_data_dir, cpu_mask, mem],
 234         stdout=PIPE,
 235         cwd=ws_root,
 236     )
 237     # Show only the test result without clobbering the stdout.
 238     # The VM console displays VPP stderr & Linux IPv6 netdev change
 239     # messages, which is logged by default and can be excluded.
 240     exclude_pattern = r"vpp\[\d+\]:|ADDRCONF\(NETDEV_CHANGE\):"
 241     show_progress(p.stdout, exclude_pattern)
 242     post_vm_test_run()
 243
 244
 245 def post_vm_test_run():
 246     # Revert the ownership of certain directories from root to the
 247     # original user after running in QEMU
 248     print("Running post test cleanup tasks")
 249     dirs = ["/tmp/vpp-failed-unittests", os.path.join(ws_root, "test", "__pycache__")]
 250     dirs.extend(glob.glob("/tmp/vpp-unittest-*"))
 251     dirs.extend(glob.glob("/tmp/api_post_mortem.*"))
 252     user = os.getlogin()
 253     for dir in dirs:
 254         if os.path.exists(dir) and Path(dir).owner() != user:
 255             cmd = ["sudo", "chown", "-R", "{0}:{0}".format(user), dir]
 256             p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
 257             show_progress(p.stdout)
 258
 259
 260 def build_venv():
 261     # Builds a virtual env containing all the required packages and patches
 262     # for running VPP unit tests
 263     if not os.path.exists(venv_install_done):
 264         env_builder = ExtendedEnvBuilder(clear=True, with_pip=True)
 265         print("Creating a vEnv for running VPP unit tests in {}".format(venv_dir))
 266         env_builder.create(venv_dir)
 267         # Write state to the venv run dir
 268         Path(venv_run_dir).mkdir(exist_ok=True)
 269         Path(venv_install_done).touch()
 270
 271
 272 def expand_mix_string(s):
 273     # Returns an expanded string computed from a mixrange string (s)
 274     # E.g: If param s = '5-8,10,11' returns '5,6,7,8,10,11'
 275     result = []
 276     for val in s.split(","):
 277         if "-" in val:
 278             start, end = val.split("-")
 279             result.extend(list(range(int(start), int(end) + 1)))
 280         else:
 281             result.append(int(val))
 282     return ",".join(str(i) for i in set(result))
 283
 284
 285 def set_logging(test_data_dir, test_name):
 286     Path(test_data_dir).mkdir(exist_ok=True)
 287     log_file = "vm_{0}_{1}.log".format(test_name, str(time.time())[-5:])
 288     filename = "{0}/{1}".format(test_data_dir, log_file)
 289     Path(filename).touch()
 290     logging.basicConfig(filename=filename, level=logging.DEBUG)
 291
 292
 293 def run_tests_in_venv(
 294     test,
 295     jobs,
 296     log_dir,
 297     socket_dir="",
 298     running_vpp=False,
 299     extended=False,
 300 ):
 301     """Runs tests in the virtual environment set by venv_dir.
 302
 303     Arguments:
 304     test: Name of the test to run
 305     jobs: Maximum concurrent test jobs
 306     log_dir: Directory location for storing log files
 307     socket_dir: Use running VPP's socket files
 308     running_vpp: True if tests are run against a running VPP
 309     extended: Run extended tests
 310     """
 311     script = os.path.join(test_dir, "scripts", "run.sh")
 312     args = [
 313         f"--venv-dir={venv_dir}",
 314         f"--vpp-ws-dir={ws_root}",
 315         f"--socket-dir={socket_dir}",
 316         f"--filter={test}",
 317         f"--jobs={jobs}",
 318         f"--log-dir={log_dir}",
 319         f"--tmp-dir={log_dir}",
 320         f"--cache-vpp-output",
 321     ]
 322     if running_vpp:
 323         args = args + [f"--use-running-vpp"]
 324     if extended:
 325         args = args + [f"--extended"]
 326     print(f"Running script: {script} " f"{' '.join(args)}")
 327     process_args = [script] + args
 328     call(process_args)
 329
 330
 331 if __name__ == "__main__":
 332     # Build a Virtual Environment for running tests on host & QEMU
 333     # (TODO): Create a single config object by merging the below args with
 334     # config.py after gathering dev use-cases.
 335     parser = argparse.ArgumentParser(
 336         description="Run VPP Unit Tests", formatter_class=argparse.RawTextHelpFormatter
 337     )
 338     parser.add_argument(
 339         "--vm",
 340         dest="vm",
 341         required=False,
 342         action="store_true",
 343         help="Run Test Inside a QEMU VM",
 344     )
 345     parser.add_argument(
 346         "--debug",
 347         dest="debug",
 348         required=False,
 349         default=True,
 350         action="store_true",
 351         help="Run Tests on Debug Build",
 352     )
 353     parser.add_argument(
 354         "--release",
 355         dest="release",
 356         required=False,
 357         default=False,
 358         action="store_true",
 359         help="Run Tests on release Build",
 360     )
 361     parser.add_argument(
 362         "-t",
 363         "--test",
 364         dest="test_name",
 365         required=False,
 366         action="store",
 367         default="",
 368         help="Test Name or Test filter",
 369     )
 370     parser.add_argument(
 371         "--vm-kernel-image",
 372         dest="kernel_image",
 373         required=False,
 374         action="store",
 375         default="",
 376         help="Kernel Image Selection to Boot",
 377     )
 378     parser.add_argument(
 379         "--vm-cpu-list",
 380         dest="vm_cpu_list",
 381         required=False,
 382         action="store",
 383         default="5-8",
 384         help="Set CPU Affinity\n"
 385         "E.g. 5-7,10 will schedule on processors "
 386         "#5, #6, #7 and #10. (Default: 5-8)",
 387     )
 388     parser.add_argument(
 389         "--vm-mem",
 390         dest="vm_mem",
 391         required=False,
 392         action="store",
 393         default="2",
 394         help="Guest Memory in Gibibytes\n" "E.g. 4 (Default: 2)",
 395     )
 396     parser.add_argument(
 397         "--log-dir",
 398         action="store",
 399         default=os.path.abspath(f"./test-run-{datetime.date.today()}"),
 400         help="directory where to store directories "
 401         "containing log files (default: ./test-run-YYYY-MM-DD)",
 402     )
 403     parser.add_argument(
 404         "--jobs",
 405         action="store",
 406         default="auto",
 407         help="maximum concurrent test jobs",
 408     )
 409     parser.add_argument(
 410         "-r",
 411         "--use-running-vpp",
 412         dest="running_vpp",
 413         required=False,
 414         action="store_true",
 415         default=False,
 416         help="Runs tests against a running VPP.",
 417     )
 418     parser.add_argument(
 419         "-d",
 420         "--socket-dir",
 421         dest="socket_dir",
 422         required=False,
 423         action="store",
 424         default="",
 425         help="Relative or absolute path of running VPP's socket directory "
 426         "containing api.sock & stats.sock files.\n"
 427         "Default: /var/run/vpp if VPP is started as the root user, else "
 428         "/var/run/user/${uid}/vpp.",
 429     )
 430     parser.add_argument(
 431         "-e",
 432         "--extended",
 433         dest="extended",
 434         required=False,
 435         action="store_true",
 436         default=False,
 437         help="Run extended tests.",
 438     )
 439     args = parser.parse_args()
 440     vm_tests = False
 441     # Enable VM tests
 442     if args.vm and args.test_name:
 443         test_data_dir = "/tmp/vpp-vm-tests"
 444         set_logging(test_data_dir, args.test_name)
 445         vm_tests = True
 446     elif args.vm and not args.test_name:
 447         print("Error: The --test argument must be set for running VM tests")
 448         sys.exit(1)
 449     build_venv()
 450     # Build VPP release or debug binaries
 451     debug = False if args.release else True
 452     build_vpp(debug, args.release)
 453     set_environ()
 454     if args.running_vpp:
 455         print("Tests will be run against a running VPP..")
 456     elif not vm_tests:
 457         print("Tests will be run by spawning a new VPP instance..")
 458     # Run tests against a running VPP or a new instance of VPP
 459     if not vm_tests:
 460         run_tests_in_venv(
 461             test=args.test_name,
 462             jobs=args.jobs,
 463             log_dir=args.log_dir,
 464             socket_dir=args.socket_dir,
 465             running_vpp=args.running_vpp,
 466             extended=args.extended,
 467         )
 468     # Run tests against a VPP inside a VM
 469     else:
 470         print("Running VPP unit test(s):{0} inside a QEMU VM".format(args.test_name))
 471         # Check Available CPUs & Usable Memory
 472         cpus = expand_mix_string(args.vm_cpu_list)
 473         num_cpus, usable_cpus = (len(cpus.split(",")), len(os.sched_getaffinity(0)))
 474         if num_cpus > usable_cpus:
 475             print(f"Error:# of CPUs:{num_cpus} > Avail CPUs:{usable_cpus}")
 476             sys.exit(1)
 477         avail_mem = int(os.popen("free -t -g").readlines()[-1].split()[-1])
 478         if int(args.vm_mem) > avail_mem:
 479             print(f"Error: Mem Size:{args.vm_mem}G > Avail Mem:{avail_mem}G")
 480             sys.exit(1)
 481         vm_test_runner(
 482             args.test_name,
 483             args.kernel_image,
 484             test_data_dir,
 485             cpus,
 486             f"{args.vm_mem}G",
 487             args.jobs,
 488         )