test/run.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (c) 2022 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 #
  16 # Build the Virtual Environment & run VPP unit tests
  17
  18 import argparse
  19 import glob
  20 import logging
  21 import os
  22 from pathlib import Path
  23 import signal
  24 from subprocess import Popen, PIPE, STDOUT, call
  25 import sys
  26 import time
  27 import venv
  28
  29
  30 # Required Std. Path Variables
  31 test_dir = os.path.dirname(os.path.realpath(__file__))
  32 ws_root = os.path.dirname(test_dir)
  33 build_root = os.path.join(ws_root, "build-root")
  34 venv_dir = os.path.join(build_root, "test", "venv")
  35 venv_bin_dir = os.path.join(venv_dir, "bin")
  36 venv_lib_dir = os.path.join(venv_dir, "lib")
  37 venv_run_dir = os.path.join(venv_dir, "run")
  38 venv_install_done = os.path.join(venv_run_dir, "venv_install.done")
  39 papi_python_src_dir = os.path.join(ws_root, "src", "vpp-api", "python")
  40
  41 # Path Variables Set after VPP Build/Install
  42 vpp_build_dir = vpp_install_path = vpp_bin = vpp_lib = vpp_lib64 = None
  43 vpp_plugin_path = vpp_test_plugin_path = ld_library_path = None
  44
  45 # Pip version pinning
  46 pip_version = "22.0.4"
  47 pip_tools_version = "6.6.0"
  48
  49 # Test requirement files
  50 test_requirements_file = os.path.join(test_dir, "requirements.txt")
  51 # Auto-generated requirement file
  52 pip_compiled_requirements_file = os.path.join(test_dir, "requirements-3.txt")
  53
  54
  55 # Gracefully exit after executing cleanup scripts
  56 # upon receiving a SIGINT or SIGTERM
  57 def handler(signum, frame):
  58     print("Received Signal {0}".format(signum))
  59     post_vm_test_run()
  60
  61
  62 signal.signal(signal.SIGINT, handler)
  63 signal.signal(signal.SIGTERM, handler)
  64
  65
  66 def show_progress(stream):
  67     """
  68     Read lines from a subprocess stdout/stderr streams and write
  69     to sys.stdout & the logfile
  70     """
  71     while True:
  72         s = stream.readline()
  73         if not s:
  74             break
  75         data = s.decode("utf-8")
  76         # Filter the annoying SIGTERM signal from the output when VPP is
  77         # terminated after a test run
  78         if "SIGTERM" not in data:
  79             sys.stdout.write(data)
  80             logging.debug(data)
  81         sys.stdout.flush()
  82     stream.close()
  83
  84
  85 class ExtendedEnvBuilder(venv.EnvBuilder):
  86     """
  87     1. Builds a Virtual Environment for running VPP unit tests
  88     2. Installs all necessary scripts, pkgs & patches into the vEnv
  89          - python3, pip, pip-tools, papi, scapy patches &
  90            test-requirement pkgs
  91     """
  92
  93     def __init__(self, *args, **kwargs):
  94         super().__init__(*args, **kwargs)
  95
  96     def post_setup(self, context):
  97         """
  98         Setup all packages that need to be pre-installed into the venv
  99         prior to running VPP unit tests.
 100
 101         :param context: The context of the virtual environment creation
 102                         request being processed.
 103         """
 104         os.environ["VIRTUAL_ENV"] = context.env_dir
 105         os.environ[
 106             "CUSTOM_COMPILE_COMMAND"
 107         ] = "make test-refresh-deps (or update requirements.txt)"
 108         # Cleanup previously auto-generated pip req. file
 109         try:
 110             os.unlink(pip_compiled_requirements_file)
 111         except OSError:
 112             pass
 113         # Set the venv python executable & binary install path
 114         env_exe = context.env_exe
 115         bin_path = context.bin_path
 116         # Packages/requirements to be installed in the venv
 117         # [python-module, cmdline-args, package-name_or_requirements-file-name]
 118         test_req = [
 119             ["pip", "install", "pip===%s" % pip_version],
 120             ["pip", "install", "pip-tools===%s" % pip_tools_version],
 121             [
 122                 "piptools",
 123                 "compile",
 124                 "-q",
 125                 "--generate-hashes",
 126                 test_requirements_file,
 127                 "--output-file",
 128                 pip_compiled_requirements_file,
 129             ],
 130             ["piptools", "sync", pip_compiled_requirements_file],
 131             ["pip", "install", "-e", papi_python_src_dir],
 132         ]
 133         for req in test_req:
 134             args = [env_exe, "-m"]
 135             args.extend(req)
 136             print(args)
 137             p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=bin_path)
 138             show_progress(p.stdout)
 139         self.pip_patch()
 140
 141     def pip_patch(self):
 142         """
 143         Apply scapy patch files
 144         """
 145         scapy_patch_dir = Path(os.path.join(test_dir, "patches", "scapy-2.4.3"))
 146         scapy_source_dir = glob.glob(
 147             os.path.join(venv_lib_dir, "python3.*", "site-packages")
 148         )[0]
 149         for f in scapy_patch_dir.iterdir():
 150             print("Applying patch: {}".format(os.path.basename(str(f))))
 151             args = ["patch", "--forward", "-p1", "-d", scapy_source_dir, "-i", str(f)]
 152             print(args)
 153             p = Popen(args, stdout=PIPE, stderr=STDOUT)
 154             show_progress(p.stdout)
 155
 156
 157 # Build VPP Release/Debug binaries
 158 def build_vpp(debug=True, release=False):
 159     """
 160     Install VPP Release(if release=True) or Debug(if debug=True) Binaries.
 161
 162     Default is to build the debug binaries.
 163     """
 164     global vpp_build_dir, vpp_install_path, vpp_bin, vpp_lib, vpp_lib64
 165     global vpp_plugin_path, vpp_test_plugin_path, ld_library_path
 166     if debug:
 167         print("Building VPP debug binaries")
 168         args = ["make", "build"]
 169         build = "build-vpp_debug-native"
 170         install = "install-vpp_debug-native"
 171     elif release:
 172         print("Building VPP release binaries")
 173         args = ["make", "build-release"]
 174         build = "build-vpp-native"
 175         install = "install-vpp-native"
 176     p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=ws_root)
 177     show_progress(p.stdout)
 178     vpp_build_dir = os.path.join(build_root, build)
 179     vpp_install_path = os.path.join(build_root, install)
 180     vpp_bin = os.path.join(vpp_install_path, "vpp", "bin", "vpp")
 181     vpp_lib = os.path.join(vpp_install_path, "vpp", "lib")
 182     vpp_lib64 = os.path.join(vpp_install_path, "vpp", "lib64")
 183     vpp_plugin_path = (
 184         os.path.join(vpp_lib, "vpp_plugins")
 185         + ":"
 186         + os.path.join(vpp_lib64, "vpp_plugins")
 187     )
 188     vpp_test_plugin_path = (
 189         os.path.join(vpp_lib, "vpp_api_test_plugins")
 190         + ":"
 191         + os.path.join(vpp_lib64, "vpp_api_test_plugins")
 192     )
 193     ld_library_path = os.path.join(vpp_lib) + ":" + os.path.join(vpp_lib64)
 194
 195
 196 # Environment Vars required by the test framework,
 197 # papi_provider & unittests
 198 def set_environ():
 199     os.environ["WS_ROOT"] = ws_root
 200     os.environ["BR"] = build_root
 201     os.environ["VENV_PATH"] = venv_dir
 202     os.environ["VENV_BIN"] = venv_bin_dir
 203     os.environ["RND_SEED"] = str(time.time())
 204     os.environ["VPP_BUILD_DIR"] = vpp_build_dir
 205     os.environ["VPP_BIN"] = vpp_bin
 206     os.environ["VPP_PLUGIN_PATH"] = vpp_plugin_path
 207     os.environ["VPP_TEST_PLUGIN_PATH"] = vpp_test_plugin_path
 208     os.environ["VPP_INSTALL_PATH"] = vpp_install_path
 209     os.environ["LD_LIBRARY_PATH"] = ld_library_path
 210     os.environ["FAILED_DIR"] = "/tmp/vpp-failed-unittests/"
 211     if not os.environ.get("TEST_JOBS"):
 212         os.environ["TEST_JOBS"] = "1"
 213
 214
 215 # Runs a test inside a spawned QEMU VM
 216 # If a kernel image is not provided, a linux-image-kvm image is
 217 # downloaded to the test_data_dir
 218 def vm_test_runner(test_name, kernel_image, test_data_dir, cpu_mask, mem, jobs="auto"):
 219     script = os.path.join(test_dir, "scripts", "run_vpp_in_vm.sh")
 220     os.environ["TEST_JOBS"] = str(jobs)
 221     p = Popen(
 222         [script, test_name, kernel_image, test_data_dir, cpu_mask, mem],
 223         stdout=PIPE,
 224         stderr=STDOUT,
 225         cwd=ws_root,
 226     )
 227     show_progress(p.stdout)
 228     post_vm_test_run()
 229
 230
 231 def post_vm_test_run():
 232     # Revert the ownership of certain directories from root to the
 233     # original user after running in QEMU
 234     print("Running post test cleanup tasks")
 235     dirs = ["/tmp/vpp-failed-unittests", os.path.join(ws_root, "test", "__pycache__")]
 236     dirs.extend(glob.glob("/tmp/vpp-unittest-*"))
 237     dirs.extend(glob.glob("/tmp/api_post_mortem.*"))
 238     user = os.getlogin()
 239     for dir in dirs:
 240         if os.path.exists(dir) and Path(dir).owner() != user:
 241             cmd = ["sudo", "chown", "-R", "{0}:{0}".format(user), dir]
 242             p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
 243             show_progress(p.stdout)
 244
 245
 246 def build_venv():
 247     # Builds a virtual env containing all the required packages and patches
 248     # for running VPP unit tests
 249     if not os.path.exists(venv_install_done):
 250         env_builder = ExtendedEnvBuilder(clear=True, with_pip=True)
 251         print("Creating a vEnv for running VPP unit tests in {}".format(venv_dir))
 252         env_builder.create(venv_dir)
 253         # Write state to the venv run dir
 254         Path(venv_run_dir).mkdir(exist_ok=True)
 255         Path(venv_install_done).touch()
 256
 257
 258 def expand_mix_string(s):
 259     # Returns an expanded string computed from a mixrange string (s)
 260     # E.g: If param s = '5-8,10,11' returns '5,6,7,8,10,11'
 261     result = []
 262     for val in s.split(","):
 263         if "-" in val:
 264             start, end = val.split("-")
 265             result.extend(list(range(int(start), int(end) + 1)))
 266         else:
 267             result.append(int(val))
 268     return ",".join(str(i) for i in set(result))
 269
 270
 271 def set_logging(test_data_dir, test_name):
 272     Path(test_data_dir).mkdir(exist_ok=True)
 273     log_file = "vm_{0}_{1}.log".format(test_name, str(time.time())[-5:])
 274     filename = "{0}/{1}".format(test_data_dir, log_file)
 275     Path(filename).touch()
 276     logging.basicConfig(filename=filename, level=logging.DEBUG)
 277
 278
 279 def run_tests_in_venv(
 280     test,
 281     jobs,
 282     log_dir,
 283     socket_dir="",
 284     running_vpp=False,
 285 ):
 286     """Runs tests in the virtual environment set by venv_dir.
 287
 288     Arguments:
 289     test: Name of the test to run
 290     jobs: Maximum concurrent test jobs
 291     log_dir: Directory location for storing log files
 292     socket_dir: Use running VPP's socket files
 293     running_vpp: True if tests are run against a running VPP
 294     """
 295     script = os.path.join(test_dir, "scripts", "run.sh")
 296     args = [
 297         f"--venv-dir={venv_dir}",
 298         f"--vpp-ws-dir={ws_root}",
 299         f"--socket-dir={socket_dir}",
 300         f"--filter={test}",
 301         f"--jobs={jobs}",
 302         f"--log-dir={log_dir}",
 303     ]
 304     if running_vpp:
 305         args = args + [f"--use-running-vpp"]
 306     print(f"Running script: {script} " f"{' '.join(args)}")
 307     process_args = [script] + args
 308     call(process_args)
 309
 310
 311 if __name__ == "__main__":
 312     # Build a Virtual Environment for running tests on host & QEMU
 313     # (TODO): Create a single config object by merging the below args with
 314     # config.py after gathering dev use-cases.
 315     parser = argparse.ArgumentParser(
 316         description="Run VPP Unit Tests", formatter_class=argparse.RawTextHelpFormatter
 317     )
 318     parser.add_argument(
 319         "--vm",
 320         dest="vm",
 321         required=False,
 322         action="store_true",
 323         help="Run Test Inside a QEMU VM",
 324     )
 325     parser.add_argument(
 326         "--debug",
 327         dest="debug",
 328         required=False,
 329         default=True,
 330         action="store_true",
 331         help="Run Tests on Debug Build",
 332     )
 333     parser.add_argument(
 334         "--release",
 335         dest="release",
 336         required=False,
 337         default=False,
 338         action="store_true",
 339         help="Run Tests on release Build",
 340     )
 341     parser.add_argument(
 342         "-t",
 343         "--test",
 344         dest="test_name",
 345         required=False,
 346         action="store",
 347         default="",
 348         help="Test Name or Test filter",
 349     )
 350     parser.add_argument(
 351         "--vm-kernel-image",
 352         dest="kernel_image",
 353         required=False,
 354         action="store",
 355         default="",
 356         help="Kernel Image Selection to Boot",
 357     )
 358     parser.add_argument(
 359         "--vm-cpu-list",
 360         dest="vm_cpu_list",
 361         required=False,
 362         action="store",
 363         default="5-8",
 364         help="Set CPU Affinity\n"
 365         "E.g. 5-7,10 will schedule on processors "
 366         "#5, #6, #7 and #10. (Default: 5-8)",
 367     )
 368     parser.add_argument(
 369         "--vm-mem",
 370         dest="vm_mem",
 371         required=False,
 372         action="store",
 373         default="2",
 374         help="Guest Memory in Gibibytes\n" "E.g. 4 (Default: 2)",
 375     )
 376     parser.add_argument(
 377         "--log-dir",
 378         action="store",
 379         default="/tmp",
 380         help="directory where to store directories "
 381         "containing log files (default: /tmp)",
 382     )
 383     parser.add_argument(
 384         "--jobs",
 385         action="store",
 386         default="auto",
 387         help="maximum concurrent test jobs",
 388     )
 389     parser.add_argument(
 390         "-r",
 391         "--use-running-vpp",
 392         dest="running_vpp",
 393         required=False,
 394         action="store_true",
 395         default=False,
 396         help="Runs tests against a running VPP.",
 397     )
 398     parser.add_argument(
 399         "-d",
 400         "--socket-dir",
 401         dest="socket_dir",
 402         required=False,
 403         action="store",
 404         default="",
 405         help="Relative or absolute path of running VPP's socket directory "
 406         "containing api.sock & stats.sock files.\n"
 407         "Default: /var/run/vpp if VPP is started as the root user, else "
 408         "/var/run/user/${uid}/vpp.",
 409     )
 410     args = parser.parse_args()
 411     vm_tests = False
 412     # Enable VM tests
 413     if args.vm and args.test_name:
 414         test_data_dir = "/tmp/vpp-vm-tests"
 415         set_logging(test_data_dir, args.test_name)
 416         vm_tests = True
 417     elif args.vm and not args.test_name:
 418         print("Error: The --test argument must be set for running VM tests")
 419         sys.exit(1)
 420     build_venv()
 421     # Build VPP release or debug binaries
 422     debug = False if args.release else True
 423     build_vpp(debug, args.release)
 424     set_environ()
 425     if args.running_vpp:
 426         print("Tests will be run against a running VPP..")
 427     elif not vm_tests:
 428         print("Tests will be run by spawning a new VPP instance..")
 429     # Run tests against a running VPP or a new instance of VPP
 430     if not vm_tests:
 431         run_tests_in_venv(
 432             test=args.test_name,
 433             jobs=args.jobs,
 434             log_dir=args.log_dir,
 435             socket_dir=args.socket_dir,
 436             running_vpp=args.running_vpp,
 437         )
 438     # Run tests against a VPP inside a VM
 439     else:
 440         print("Running VPP unit test(s):{0} inside a QEMU VM".format(args.test_name))
 441         # Check Available CPUs & Usable Memory
 442         cpus = expand_mix_string(args.vm_cpu_list)
 443         num_cpus, usable_cpus = (len(cpus.split(",")), len(os.sched_getaffinity(0)))
 444         if num_cpus > usable_cpus:
 445             print(f"Error:# of CPUs:{num_cpus} > Avail CPUs:{usable_cpus}")
 446             sys.exit(1)
 447         avail_mem = int(os.popen("free -t -g").readlines()[-1].split()[-1])
 448         if int(args.vm_mem) > avail_mem:
 449             print(f"Error: Mem Size:{args.vm_mem}G > Avail Mem:{avail_mem}G")
 450             sys.exit(1)
 451         vm_test_runner(
 452             args.test_name,
 453             args.kernel_image,
 454             test_data_dir,
 455             cpus,
 456             f"{args.vm_mem}G",
 457             args.jobs,
 458         )