test/run.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (c) 2022 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 #
  16 # Build the Virtual Environment & run VPP unit tests
  17
  18 import argparse
  19 import glob
  20 import logging
  21 import os
  22 from pathlib import Path
  23 import signal
  24 from subprocess import Popen, PIPE, STDOUT, call
  25 import sys
  26 import time
  27 import venv
  28 import datetime
  29
  30
  31 # Required Std. Path Variables
  32 test_dir = os.path.dirname(os.path.realpath(__file__))
  33 ws_root = os.path.dirname(test_dir)
  34 build_root = os.path.join(ws_root, "build-root")
  35 venv_dir = os.path.join(build_root, "test", "venv")
  36 venv_bin_dir = os.path.join(venv_dir, "bin")
  37 venv_lib_dir = os.path.join(venv_dir, "lib")
  38 venv_run_dir = os.path.join(venv_dir, "run")
  39 venv_install_done = os.path.join(venv_run_dir, "venv_install.done")
  40 papi_python_src_dir = os.path.join(ws_root, "src", "vpp-api", "python")
  41
  42 # Path Variables Set after VPP Build/Install
  43 vpp_build_dir = vpp_install_path = vpp_bin = vpp_lib = vpp_lib64 = None
  44 vpp_plugin_path = vpp_test_plugin_path = ld_library_path = None
  45
  46 # Pip version pinning
  47 pip_version = "22.0.4"
  48 pip_tools_version = "6.6.0"
  49
  50 # Test requirement files
  51 test_requirements_file = os.path.join(test_dir, "requirements.txt")
  52 # Auto-generated requirement file
  53 pip_compiled_requirements_file = os.path.join(test_dir, "requirements-3.txt")
  54
  55
  56 # Gracefully exit after executing cleanup scripts
  57 # upon receiving a SIGINT or SIGTERM
  58 def handler(signum, frame):
  59     print("Received Signal {0}".format(signum))
  60     post_vm_test_run()
  61
  62
  63 signal.signal(signal.SIGINT, handler)
  64 signal.signal(signal.SIGTERM, handler)
  65
  66
  67 def show_progress(stream):
  68     """
  69     Read lines from a subprocess stdout/stderr streams and write
  70     to sys.stdout & the logfile
  71     """
  72     while True:
  73         s = stream.readline()
  74         if not s:
  75             break
  76         data = s.decode("utf-8")
  77         # Filter the annoying SIGTERM signal from the output when VPP is
  78         # terminated after a test run
  79         if "SIGTERM" not in data:
  80             sys.stdout.write(data)
  81             logging.debug(data)
  82         sys.stdout.flush()
  83     stream.close()
  84
  85
  86 class ExtendedEnvBuilder(venv.EnvBuilder):
  87     """
  88     1. Builds a Virtual Environment for running VPP unit tests
  89     2. Installs all necessary scripts, pkgs & patches into the vEnv
  90          - python3, pip, pip-tools, papi, scapy patches &
  91            test-requirement pkgs
  92     """
  93
  94     def __init__(self, *args, **kwargs):
  95         super().__init__(*args, **kwargs)
  96
  97     def post_setup(self, context):
  98         """
  99         Setup all packages that need to be pre-installed into the venv
 100         prior to running VPP unit tests.
 101
 102         :param context: The context of the virtual environment creation
 103                         request being processed.
 104         """
 105         os.environ["VIRTUAL_ENV"] = context.env_dir
 106         os.environ[
 107             "CUSTOM_COMPILE_COMMAND"
 108         ] = "make test-refresh-deps (or update requirements.txt)"
 109         # Cleanup previously auto-generated pip req. file
 110         try:
 111             os.unlink(pip_compiled_requirements_file)
 112         except OSError:
 113             pass
 114         # Set the venv python executable & binary install path
 115         env_exe = context.env_exe
 116         bin_path = context.bin_path
 117         # Packages/requirements to be installed in the venv
 118         # [python-module, cmdline-args, package-name_or_requirements-file-name]
 119         test_req = [
 120             ["pip", "install", "pip===%s" % pip_version],
 121             ["pip", "install", "pip-tools===%s" % pip_tools_version],
 122             [
 123                 "piptools",
 124                 "compile",
 125                 "-q",
 126                 "--generate-hashes",
 127                 test_requirements_file,
 128                 "--output-file",
 129                 pip_compiled_requirements_file,
 130             ],
 131             ["piptools", "sync", pip_compiled_requirements_file],
 132             ["pip", "install", "-e", papi_python_src_dir],
 133         ]
 134         for req in test_req:
 135             args = [env_exe, "-m"]
 136             args.extend(req)
 137             print(args)
 138             p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=bin_path)
 139             show_progress(p.stdout)
 140         self.pip_patch()
 141
 142     def pip_patch(self):
 143         """
 144         Apply scapy patch files
 145         """
 146         scapy_patch_dir = Path(os.path.join(test_dir, "patches", "scapy-2.4.3"))
 147         scapy_source_dir = glob.glob(
 148             os.path.join(venv_lib_dir, "python3.*", "site-packages")
 149         )[0]
 150         for f in scapy_patch_dir.iterdir():
 151             print("Applying patch: {}".format(os.path.basename(str(f))))
 152             args = ["patch", "--forward", "-p1", "-d", scapy_source_dir, "-i", str(f)]
 153             print(args)
 154             p = Popen(args, stdout=PIPE, stderr=STDOUT)
 155             show_progress(p.stdout)
 156
 157
 158 # Build VPP Release/Debug binaries
 159 def build_vpp(debug=True, release=False):
 160     """
 161     Install VPP Release(if release=True) or Debug(if debug=True) Binaries.
 162
 163     Default is to build the debug binaries.
 164     """
 165     global vpp_build_dir, vpp_install_path, vpp_bin, vpp_lib, vpp_lib64
 166     global vpp_plugin_path, vpp_test_plugin_path, ld_library_path
 167     if debug:
 168         print("Building VPP debug binaries")
 169         args = ["make", "build"]
 170         build = "build-vpp_debug-native"
 171         install = "install-vpp_debug-native"
 172     elif release:
 173         print("Building VPP release binaries")
 174         args = ["make", "build-release"]
 175         build = "build-vpp-native"
 176         install = "install-vpp-native"
 177     p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=ws_root)
 178     show_progress(p.stdout)
 179     vpp_build_dir = os.path.join(build_root, build)
 180     vpp_install_path = os.path.join(build_root, install)
 181     vpp_bin = os.path.join(vpp_install_path, "vpp", "bin", "vpp")
 182     vpp_lib = os.path.join(vpp_install_path, "vpp", "lib")
 183     vpp_lib64 = os.path.join(vpp_install_path, "vpp", "lib64")
 184     vpp_plugin_path = (
 185         os.path.join(vpp_lib, "vpp_plugins")
 186         + ":"
 187         + os.path.join(vpp_lib64, "vpp_plugins")
 188     )
 189     vpp_test_plugin_path = (
 190         os.path.join(vpp_lib, "vpp_api_test_plugins")
 191         + ":"
 192         + os.path.join(vpp_lib64, "vpp_api_test_plugins")
 193     )
 194     ld_library_path = os.path.join(vpp_lib) + ":" + os.path.join(vpp_lib64)
 195
 196
 197 # Environment Vars required by the test framework,
 198 # papi_provider & unittests
 199 def set_environ():
 200     os.environ["WS_ROOT"] = ws_root
 201     os.environ["BR"] = build_root
 202     os.environ["VENV_PATH"] = venv_dir
 203     os.environ["VENV_BIN"] = venv_bin_dir
 204     os.environ["RND_SEED"] = str(time.time())
 205     os.environ["VPP_BUILD_DIR"] = vpp_build_dir
 206     os.environ["VPP_BIN"] = vpp_bin
 207     os.environ["VPP_PLUGIN_PATH"] = vpp_plugin_path
 208     os.environ["VPP_TEST_PLUGIN_PATH"] = vpp_test_plugin_path
 209     os.environ["VPP_INSTALL_PATH"] = vpp_install_path
 210     os.environ["LD_LIBRARY_PATH"] = ld_library_path
 211     os.environ["FAILED_DIR"] = "/tmp/vpp-failed-unittests/"
 212     if not os.environ.get("TEST_JOBS"):
 213         os.environ["TEST_JOBS"] = "1"
 214
 215
 216 # Runs a test inside a spawned QEMU VM
 217 # If a kernel image is not provided, a linux-image-kvm image is
 218 # downloaded to the test_data_dir
 219 def vm_test_runner(test_name, kernel_image, test_data_dir, cpu_mask, mem, jobs="auto"):
 220     script = os.path.join(test_dir, "scripts", "run_vpp_in_vm.sh")
 221     os.environ["TEST_JOBS"] = str(jobs)
 222     p = Popen(
 223         [script, test_name, kernel_image, test_data_dir, cpu_mask, mem],
 224         stdout=PIPE,
 225         stderr=STDOUT,
 226         cwd=ws_root,
 227     )
 228     show_progress(p.stdout)
 229     post_vm_test_run()
 230
 231
 232 def post_vm_test_run():
 233     # Revert the ownership of certain directories from root to the
 234     # original user after running in QEMU
 235     print("Running post test cleanup tasks")
 236     dirs = ["/tmp/vpp-failed-unittests", os.path.join(ws_root, "test", "__pycache__")]
 237     dirs.extend(glob.glob("/tmp/vpp-unittest-*"))
 238     dirs.extend(glob.glob("/tmp/api_post_mortem.*"))
 239     user = os.getlogin()
 240     for dir in dirs:
 241         if os.path.exists(dir) and Path(dir).owner() != user:
 242             cmd = ["sudo", "chown", "-R", "{0}:{0}".format(user), dir]
 243             p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
 244             show_progress(p.stdout)
 245
 246
 247 def build_venv():
 248     # Builds a virtual env containing all the required packages and patches
 249     # for running VPP unit tests
 250     if not os.path.exists(venv_install_done):
 251         env_builder = ExtendedEnvBuilder(clear=True, with_pip=True)
 252         print("Creating a vEnv for running VPP unit tests in {}".format(venv_dir))
 253         env_builder.create(venv_dir)
 254         # Write state to the venv run dir
 255         Path(venv_run_dir).mkdir(exist_ok=True)
 256         Path(venv_install_done).touch()
 257
 258
 259 def expand_mix_string(s):
 260     # Returns an expanded string computed from a mixrange string (s)
 261     # E.g: If param s = '5-8,10,11' returns '5,6,7,8,10,11'
 262     result = []
 263     for val in s.split(","):
 264         if "-" in val:
 265             start, end = val.split("-")
 266             result.extend(list(range(int(start), int(end) + 1)))
 267         else:
 268             result.append(int(val))
 269     return ",".join(str(i) for i in set(result))
 270
 271
 272 def set_logging(test_data_dir, test_name):
 273     Path(test_data_dir).mkdir(exist_ok=True)
 274     log_file = "vm_{0}_{1}.log".format(test_name, str(time.time())[-5:])
 275     filename = "{0}/{1}".format(test_data_dir, log_file)
 276     Path(filename).touch()
 277     logging.basicConfig(filename=filename, level=logging.DEBUG)
 278
 279
 280 def run_tests_in_venv(
 281     test,
 282     jobs,
 283     log_dir,
 284     socket_dir="",
 285     running_vpp=False,
 286     extended=False,
 287 ):
 288     """Runs tests in the virtual environment set by venv_dir.
 289
 290     Arguments:
 291     test: Name of the test to run
 292     jobs: Maximum concurrent test jobs
 293     log_dir: Directory location for storing log files
 294     socket_dir: Use running VPP's socket files
 295     running_vpp: True if tests are run against a running VPP
 296     extended: Run extended tests
 297     """
 298     script = os.path.join(test_dir, "scripts", "run.sh")
 299     args = [
 300         f"--venv-dir={venv_dir}",
 301         f"--vpp-ws-dir={ws_root}",
 302         f"--socket-dir={socket_dir}",
 303         f"--filter={test}",
 304         f"--jobs={jobs}",
 305         f"--log-dir={log_dir}",
 306         f"--tmp-dir={log_dir}",
 307     ]
 308     if running_vpp:
 309         args = args + [f"--use-running-vpp"]
 310     if extended:
 311         args = args + [f"--extended"]
 312     print(f"Running script: {script} " f"{' '.join(args)}")
 313     process_args = [script] + args
 314     call(process_args)
 315
 316
 317 if __name__ == "__main__":
 318     # Build a Virtual Environment for running tests on host & QEMU
 319     # (TODO): Create a single config object by merging the below args with
 320     # config.py after gathering dev use-cases.
 321     parser = argparse.ArgumentParser(
 322         description="Run VPP Unit Tests", formatter_class=argparse.RawTextHelpFormatter
 323     )
 324     parser.add_argument(
 325         "--vm",
 326         dest="vm",
 327         required=False,
 328         action="store_true",
 329         help="Run Test Inside a QEMU VM",
 330     )
 331     parser.add_argument(
 332         "--debug",
 333         dest="debug",
 334         required=False,
 335         default=True,
 336         action="store_true",
 337         help="Run Tests on Debug Build",
 338     )
 339     parser.add_argument(
 340         "--release",
 341         dest="release",
 342         required=False,
 343         default=False,
 344         action="store_true",
 345         help="Run Tests on release Build",
 346     )
 347     parser.add_argument(
 348         "-t",
 349         "--test",
 350         dest="test_name",
 351         required=False,
 352         action="store",
 353         default="",
 354         help="Test Name or Test filter",
 355     )
 356     parser.add_argument(
 357         "--vm-kernel-image",
 358         dest="kernel_image",
 359         required=False,
 360         action="store",
 361         default="",
 362         help="Kernel Image Selection to Boot",
 363     )
 364     parser.add_argument(
 365         "--vm-cpu-list",
 366         dest="vm_cpu_list",
 367         required=False,
 368         action="store",
 369         default="5-8",
 370         help="Set CPU Affinity\n"
 371         "E.g. 5-7,10 will schedule on processors "
 372         "#5, #6, #7 and #10. (Default: 5-8)",
 373     )
 374     parser.add_argument(
 375         "--vm-mem",
 376         dest="vm_mem",
 377         required=False,
 378         action="store",
 379         default="2",
 380         help="Guest Memory in Gibibytes\n" "E.g. 4 (Default: 2)",
 381     )
 382     parser.add_argument(
 383         "--log-dir",
 384         action="store",
 385         default=os.path.abspath(f"./test-run-{datetime.date.today()}"),
 386         help="directory where to store directories "
 387         "containing log files (default: ./test-run-YYYY-MM-DD)",
 388     )
 389     parser.add_argument(
 390         "--jobs",
 391         action="store",
 392         default="auto",
 393         help="maximum concurrent test jobs",
 394     )
 395     parser.add_argument(
 396         "-r",
 397         "--use-running-vpp",
 398         dest="running_vpp",
 399         required=False,
 400         action="store_true",
 401         default=False,
 402         help="Runs tests against a running VPP.",
 403     )
 404     parser.add_argument(
 405         "-d",
 406         "--socket-dir",
 407         dest="socket_dir",
 408         required=False,
 409         action="store",
 410         default="",
 411         help="Relative or absolute path of running VPP's socket directory "
 412         "containing api.sock & stats.sock files.\n"
 413         "Default: /var/run/vpp if VPP is started as the root user, else "
 414         "/var/run/user/${uid}/vpp.",
 415     )
 416     parser.add_argument(
 417         "-e",
 418         "--extended",
 419         dest="extended",
 420         required=False,
 421         action="store_true",
 422         default=False,
 423         help="Run extended tests.",
 424     )
 425     args = parser.parse_args()
 426     vm_tests = False
 427     # Enable VM tests
 428     if args.vm and args.test_name:
 429         test_data_dir = "/tmp/vpp-vm-tests"
 430         set_logging(test_data_dir, args.test_name)
 431         vm_tests = True
 432     elif args.vm and not args.test_name:
 433         print("Error: The --test argument must be set for running VM tests")
 434         sys.exit(1)
 435     build_venv()
 436     # Build VPP release or debug binaries
 437     debug = False if args.release else True
 438     build_vpp(debug, args.release)
 439     set_environ()
 440     if args.running_vpp:
 441         print("Tests will be run against a running VPP..")
 442     elif not vm_tests:
 443         print("Tests will be run by spawning a new VPP instance..")
 444     # Run tests against a running VPP or a new instance of VPP
 445     if not vm_tests:
 446         run_tests_in_venv(
 447             test=args.test_name,
 448             jobs=args.jobs,
 449             log_dir=args.log_dir,
 450             socket_dir=args.socket_dir,
 451             running_vpp=args.running_vpp,
 452             extended=args.extended,
 453         )
 454     # Run tests against a VPP inside a VM
 455     else:
 456         print("Running VPP unit test(s):{0} inside a QEMU VM".format(args.test_name))
 457         # Check Available CPUs & Usable Memory
 458         cpus = expand_mix_string(args.vm_cpu_list)
 459         num_cpus, usable_cpus = (len(cpus.split(",")), len(os.sched_getaffinity(0)))
 460         if num_cpus > usable_cpus:
 461             print(f"Error:# of CPUs:{num_cpus} > Avail CPUs:{usable_cpus}")
 462             sys.exit(1)
 463         avail_mem = int(os.popen("free -t -g").readlines()[-1].split()[-1])
 464         if int(args.vm_mem) > avail_mem:
 465             print(f"Error: Mem Size:{args.vm_mem}G > Avail Mem:{avail_mem}G")
 466             sys.exit(1)
 467         vm_test_runner(
 468             args.test_name,
 469             args.kernel_image,
 470             test_data_dir,
 471             cpus,
 472             f"{args.vm_mem}G",
 473             args.jobs,
 474         )