test/run.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (c) 2022 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 #
  16 # Build the Virtual Environment & run VPP unit tests
  17
  18 import argparse
  19 import glob
  20 import logging
  21 import os
  22 from pathlib import Path
  23 import signal
  24 from subprocess import Popen, PIPE, STDOUT, call
  25 import sys
  26 import time
  27 import venv
  28 import datetime
  29
  30
  31 # Required Std. Path Variables
  32 test_dir = os.path.dirname(os.path.realpath(__file__))
  33 ws_root = os.path.dirname(test_dir)
  34 build_root = os.path.join(ws_root, "build-root")
  35 venv_dir = os.path.join(build_root, "test", "venv")
  36 venv_bin_dir = os.path.join(venv_dir, "bin")
  37 venv_lib_dir = os.path.join(venv_dir, "lib")
  38 venv_run_dir = os.path.join(venv_dir, "run")
  39 venv_install_done = os.path.join(venv_run_dir, "venv_install.done")
  40 papi_python_src_dir = os.path.join(ws_root, "src", "vpp-api", "python")
  41
  42 # Path Variables Set after VPP Build/Install
  43 vpp_build_dir = vpp_install_path = vpp_bin = vpp_lib = vpp_lib64 = None
  44 vpp_plugin_path = vpp_test_plugin_path = ld_library_path = None
  45
  46 # Pip version pinning
  47 pip_version = "22.0.4"
  48 pip_tools_version = "6.6.0"
  49
  50 # Test requirement files
  51 test_requirements_file = os.path.join(test_dir, "requirements.txt")
  52 # Auto-generated requirement file
  53 pip_compiled_requirements_file = os.path.join(test_dir, "requirements-3.txt")
  54
  55
  56 # Gracefully exit after executing cleanup scripts
  57 # upon receiving a SIGINT or SIGTERM
  58 def handler(signum, frame):
  59     print("Received Signal {0}".format(signum))
  60     post_vm_test_run()
  61
  62
  63 signal.signal(signal.SIGINT, handler)
  64 signal.signal(signal.SIGTERM, handler)
  65
  66
  67 def show_progress(stream):
  68     """
  69     Read lines from a subprocess stdout/stderr streams and write
  70     to sys.stdout & the logfile
  71     """
  72     while True:
  73         s = stream.readline()
  74         if not s:
  75             break
  76         data = s.decode("utf-8")
  77         # Filter the annoying SIGTERM signal from the output when VPP is
  78         # terminated after a test run
  79         if "SIGTERM" not in data:
  80             sys.stdout.write(data)
  81             logging.debug(data)
  82         sys.stdout.flush()
  83     stream.close()
  84
  85
  86 class ExtendedEnvBuilder(venv.EnvBuilder):
  87     """
  88     1. Builds a Virtual Environment for running VPP unit tests
  89     2. Installs all necessary scripts, pkgs & patches into the vEnv
  90          - python3, pip, pip-tools, papi, scapy patches &
  91            test-requirement pkgs
  92     """
  93
  94     def __init__(self, *args, **kwargs):
  95         super().__init__(*args, **kwargs)
  96
  97     def post_setup(self, context):
  98         """
  99         Setup all packages that need to be pre-installed into the venv
 100         prior to running VPP unit tests.
 101
 102         :param context: The context of the virtual environment creation
 103                         request being processed.
 104         """
 105         os.environ["VIRTUAL_ENV"] = context.env_dir
 106         os.environ[
 107             "CUSTOM_COMPILE_COMMAND"
 108         ] = "make test-refresh-deps (or update requirements.txt)"
 109         # Cleanup previously auto-generated pip req. file
 110         try:
 111             os.unlink(pip_compiled_requirements_file)
 112         except OSError:
 113             pass
 114         # Set the venv python executable & binary install path
 115         env_exe = context.env_exe
 116         bin_path = context.bin_path
 117         # Packages/requirements to be installed in the venv
 118         # [python-module, cmdline-args, package-name_or_requirements-file-name]
 119         test_req = [
 120             ["pip", "install", "pip===%s" % pip_version],
 121             ["pip", "install", "pip-tools===%s" % pip_tools_version],
 122             [
 123                 "piptools",
 124                 "compile",
 125                 "-q",
 126                 "--generate-hashes",
 127                 test_requirements_file,
 128                 "--output-file",
 129                 pip_compiled_requirements_file,
 130             ],
 131             ["piptools", "sync", pip_compiled_requirements_file],
 132             ["pip", "install", "-e", papi_python_src_dir],
 133         ]
 134         for req in test_req:
 135             args = [env_exe, "-m"]
 136             args.extend(req)
 137             print(args)
 138             p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=bin_path)
 139             show_progress(p.stdout)
 140         self.pip_patch()
 141
 142     def pip_patch(self):
 143         """
 144         Apply scapy patch files
 145         """
 146         scapy_patch_dir = Path(os.path.join(test_dir, "patches", "scapy-2.4.3"))
 147         scapy_source_dir = glob.glob(
 148             os.path.join(venv_lib_dir, "python3.*", "site-packages")
 149         )[0]
 150         for f in scapy_patch_dir.iterdir():
 151             print("Applying patch: {}".format(os.path.basename(str(f))))
 152             args = ["patch", "--forward", "-p1", "-d", scapy_source_dir, "-i", str(f)]
 153             print(args)
 154             p = Popen(args, stdout=PIPE, stderr=STDOUT)
 155             show_progress(p.stdout)
 156
 157
 158 # Build VPP Release/Debug binaries
 159 def build_vpp(debug=True, release=False):
 160     """
 161     Install VPP Release(if release=True) or Debug(if debug=True) Binaries.
 162
 163     Default is to build the debug binaries.
 164     """
 165     global vpp_build_dir, vpp_install_path, vpp_bin, vpp_lib, vpp_lib64
 166     global vpp_plugin_path, vpp_test_plugin_path, ld_library_path
 167     if debug:
 168         print("Building VPP debug binaries")
 169         args = ["make", "build"]
 170         build = "build-vpp_debug-native"
 171         install = "install-vpp_debug-native"
 172     elif release:
 173         print("Building VPP release binaries")
 174         args = ["make", "build-release"]
 175         build = "build-vpp-native"
 176         install = "install-vpp-native"
 177     p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=ws_root)
 178     show_progress(p.stdout)
 179     vpp_build_dir = os.path.join(build_root, build)
 180     vpp_install_path = os.path.join(build_root, install)
 181     vpp_bin = os.path.join(vpp_install_path, "vpp", "bin", "vpp")
 182     vpp_lib = os.path.join(vpp_install_path, "vpp", "lib")
 183     vpp_lib64 = os.path.join(vpp_install_path, "vpp", "lib64")
 184     vpp_plugin_path = (
 185         os.path.join(vpp_lib, "vpp_plugins")
 186         + ":"
 187         + os.path.join(vpp_lib64, "vpp_plugins")
 188     )
 189     vpp_test_plugin_path = (
 190         os.path.join(vpp_lib, "vpp_api_test_plugins")
 191         + ":"
 192         + os.path.join(vpp_lib64, "vpp_api_test_plugins")
 193     )
 194     ld_library_path = os.path.join(vpp_lib) + ":" + os.path.join(vpp_lib64)
 195
 196
 197 # Environment Vars required by the test framework,
 198 # papi_provider & unittests
 199 def set_environ():
 200     os.environ["WS_ROOT"] = ws_root
 201     os.environ["BR"] = build_root
 202     os.environ["VENV_PATH"] = venv_dir
 203     os.environ["VENV_BIN"] = venv_bin_dir
 204     os.environ["RND_SEED"] = str(time.time())
 205     os.environ["VPP_BUILD_DIR"] = vpp_build_dir
 206     os.environ["VPP_BIN"] = vpp_bin
 207     os.environ["VPP_PLUGIN_PATH"] = vpp_plugin_path
 208     os.environ["VPP_TEST_PLUGIN_PATH"] = vpp_test_plugin_path
 209     os.environ["VPP_INSTALL_PATH"] = vpp_install_path
 210     os.environ["LD_LIBRARY_PATH"] = ld_library_path
 211     os.environ["FAILED_DIR"] = "/tmp/vpp-failed-unittests/"
 212     if not os.environ.get("TEST_JOBS"):
 213         os.environ["TEST_JOBS"] = "1"
 214
 215
 216 # Runs a test inside a spawned QEMU VM
 217 # If a kernel image is not provided, a linux-image-kvm image is
 218 # downloaded to the test_data_dir
 219 def vm_test_runner(test_name, kernel_image, test_data_dir, cpu_mask, mem, jobs="auto"):
 220     script = os.path.join(test_dir, "scripts", "run_vpp_in_vm.sh")
 221     os.environ["TEST_JOBS"] = str(jobs)
 222     p = Popen(
 223         [script, test_name, kernel_image, test_data_dir, cpu_mask, mem],
 224         stdout=PIPE,
 225         stderr=STDOUT,
 226         cwd=ws_root,
 227     )
 228     show_progress(p.stdout)
 229     post_vm_test_run()
 230
 231
 232 def post_vm_test_run():
 233     # Revert the ownership of certain directories from root to the
 234     # original user after running in QEMU
 235     print("Running post test cleanup tasks")
 236     dirs = ["/tmp/vpp-failed-unittests", os.path.join(ws_root, "test", "__pycache__")]
 237     dirs.extend(glob.glob("/tmp/vpp-unittest-*"))
 238     dirs.extend(glob.glob("/tmp/api_post_mortem.*"))
 239     user = os.getlogin()
 240     for dir in dirs:
 241         if os.path.exists(dir) and Path(dir).owner() != user:
 242             cmd = ["sudo", "chown", "-R", "{0}:{0}".format(user), dir]
 243             p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
 244             show_progress(p.stdout)
 245
 246
 247 def build_venv():
 248     # Builds a virtual env containing all the required packages and patches
 249     # for running VPP unit tests
 250     if not os.path.exists(venv_install_done):
 251         env_builder = ExtendedEnvBuilder(clear=True, with_pip=True)
 252         print("Creating a vEnv for running VPP unit tests in {}".format(venv_dir))
 253         env_builder.create(venv_dir)
 254         # Write state to the venv run dir
 255         Path(venv_run_dir).mkdir(exist_ok=True)
 256         Path(venv_install_done).touch()
 257
 258
 259 def expand_mix_string(s):
 260     # Returns an expanded string computed from a mixrange string (s)
 261     # E.g: If param s = '5-8,10,11' returns '5,6,7,8,10,11'
 262     result = []
 263     for val in s.split(","):
 264         if "-" in val:
 265             start, end = val.split("-")
 266             result.extend(list(range(int(start), int(end) + 1)))
 267         else:
 268             result.append(int(val))
 269     return ",".join(str(i) for i in set(result))
 270
 271
 272 def set_logging(test_data_dir, test_name):
 273     Path(test_data_dir).mkdir(exist_ok=True)
 274     log_file = "vm_{0}_{1}.log".format(test_name, str(time.time())[-5:])
 275     filename = "{0}/{1}".format(test_data_dir, log_file)
 276     Path(filename).touch()
 277     logging.basicConfig(filename=filename, level=logging.DEBUG)
 278
 279
 280 def run_tests_in_venv(
 281     test,
 282     jobs,
 283     log_dir,
 284     socket_dir="",
 285     running_vpp=False,
 286 ):
 287     """Runs tests in the virtual environment set by venv_dir.
 288
 289     Arguments:
 290     test: Name of the test to run
 291     jobs: Maximum concurrent test jobs
 292     log_dir: Directory location for storing log files
 293     socket_dir: Use running VPP's socket files
 294     running_vpp: True if tests are run against a running VPP
 295     """
 296     script = os.path.join(test_dir, "scripts", "run.sh")
 297     args = [
 298         f"--venv-dir={venv_dir}",
 299         f"--vpp-ws-dir={ws_root}",
 300         f"--socket-dir={socket_dir}",
 301         f"--filter={test}",
 302         f"--jobs={jobs}",
 303         f"--log-dir={log_dir}",
 304         f"--tmp-dir={log_dir}",
 305     ]
 306     if running_vpp:
 307         args = args + [f"--use-running-vpp"]
 308     print(f"Running script: {script} " f"{' '.join(args)}")
 309     process_args = [script] + args
 310     call(process_args)
 311
 312
 313 if __name__ == "__main__":
 314     # Build a Virtual Environment for running tests on host & QEMU
 315     # (TODO): Create a single config object by merging the below args with
 316     # config.py after gathering dev use-cases.
 317     parser = argparse.ArgumentParser(
 318         description="Run VPP Unit Tests", formatter_class=argparse.RawTextHelpFormatter
 319     )
 320     parser.add_argument(
 321         "--vm",
 322         dest="vm",
 323         required=False,
 324         action="store_true",
 325         help="Run Test Inside a QEMU VM",
 326     )
 327     parser.add_argument(
 328         "--debug",
 329         dest="debug",
 330         required=False,
 331         default=True,
 332         action="store_true",
 333         help="Run Tests on Debug Build",
 334     )
 335     parser.add_argument(
 336         "--release",
 337         dest="release",
 338         required=False,
 339         default=False,
 340         action="store_true",
 341         help="Run Tests on release Build",
 342     )
 343     parser.add_argument(
 344         "-t",
 345         "--test",
 346         dest="test_name",
 347         required=False,
 348         action="store",
 349         default="",
 350         help="Test Name or Test filter",
 351     )
 352     parser.add_argument(
 353         "--vm-kernel-image",
 354         dest="kernel_image",
 355         required=False,
 356         action="store",
 357         default="",
 358         help="Kernel Image Selection to Boot",
 359     )
 360     parser.add_argument(
 361         "--vm-cpu-list",
 362         dest="vm_cpu_list",
 363         required=False,
 364         action="store",
 365         default="5-8",
 366         help="Set CPU Affinity\n"
 367         "E.g. 5-7,10 will schedule on processors "
 368         "#5, #6, #7 and #10. (Default: 5-8)",
 369     )
 370     parser.add_argument(
 371         "--vm-mem",
 372         dest="vm_mem",
 373         required=False,
 374         action="store",
 375         default="2",
 376         help="Guest Memory in Gibibytes\n" "E.g. 4 (Default: 2)",
 377     )
 378     parser.add_argument(
 379         "--log-dir",
 380         action="store",
 381         default=os.path.abspath(f"./test-run-{datetime.date.today()}"),
 382         help="directory where to store directories "
 383         "containing log files (default: ./test-run-YYYY-MM-DD)",
 384     )
 385     parser.add_argument(
 386         "--jobs",
 387         action="store",
 388         default="auto",
 389         help="maximum concurrent test jobs",
 390     )
 391     parser.add_argument(
 392         "-r",
 393         "--use-running-vpp",
 394         dest="running_vpp",
 395         required=False,
 396         action="store_true",
 397         default=False,
 398         help="Runs tests against a running VPP.",
 399     )
 400     parser.add_argument(
 401         "-d",
 402         "--socket-dir",
 403         dest="socket_dir",
 404         required=False,
 405         action="store",
 406         default="",
 407         help="Relative or absolute path of running VPP's socket directory "
 408         "containing api.sock & stats.sock files.\n"
 409         "Default: /var/run/vpp if VPP is started as the root user, else "
 410         "/var/run/user/${uid}/vpp.",
 411     )
 412     args = parser.parse_args()
 413     vm_tests = False
 414     # Enable VM tests
 415     if args.vm and args.test_name:
 416         test_data_dir = "/tmp/vpp-vm-tests"
 417         set_logging(test_data_dir, args.test_name)
 418         vm_tests = True
 419     elif args.vm and not args.test_name:
 420         print("Error: The --test argument must be set for running VM tests")
 421         sys.exit(1)
 422     build_venv()
 423     # Build VPP release or debug binaries
 424     debug = False if args.release else True
 425     build_vpp(debug, args.release)
 426     set_environ()
 427     if args.running_vpp:
 428         print("Tests will be run against a running VPP..")
 429     elif not vm_tests:
 430         print("Tests will be run by spawning a new VPP instance..")
 431     # Run tests against a running VPP or a new instance of VPP
 432     if not vm_tests:
 433         run_tests_in_venv(
 434             test=args.test_name,
 435             jobs=args.jobs,
 436             log_dir=args.log_dir,
 437             socket_dir=args.socket_dir,
 438             running_vpp=args.running_vpp,
 439         )
 440     # Run tests against a VPP inside a VM
 441     else:
 442         print("Running VPP unit test(s):{0} inside a QEMU VM".format(args.test_name))
 443         # Check Available CPUs & Usable Memory
 444         cpus = expand_mix_string(args.vm_cpu_list)
 445         num_cpus, usable_cpus = (len(cpus.split(",")), len(os.sched_getaffinity(0)))
 446         if num_cpus > usable_cpus:
 447             print(f"Error:# of CPUs:{num_cpus} > Avail CPUs:{usable_cpus}")
 448             sys.exit(1)
 449         avail_mem = int(os.popen("free -t -g").readlines()[-1].split()[-1])
 450         if int(args.vm_mem) > avail_mem:
 451             print(f"Error: Mem Size:{args.vm_mem}G > Avail Mem:{avail_mem}G")
 452             sys.exit(1)
 453         vm_test_runner(
 454             args.test_name,
 455             args.kernel_image,
 456             test_data_dir,
 457             cpus,
 458             f"{args.vm_mem}G",
 459             args.jobs,
 460         )