test/run.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (c) 2022 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 #
  16 # Build the Virtual Environment & run VPP unit tests
  17
  18 import argparse
  19 import glob
  20 import logging
  21 import os
  22 from pathlib import Path
  23 import signal
  24 from subprocess import Popen, PIPE, STDOUT
  25 import sys
  26 import time
  27 import venv
  28
  29
  30 # Required Std. Path Variables
  31 test_dir = os.path.dirname(os.path.realpath(__file__))
  32 ws_root = os.path.dirname(test_dir)
  33 build_root = os.path.join(ws_root, "build-root")
  34 venv_dir = os.path.join(test_dir, "venv")
  35 venv_bin_dir = os.path.join(venv_dir, "bin")
  36 venv_lib_dir = os.path.join(venv_dir, "lib")
  37 venv_run_dir = os.path.join(venv_dir, "run")
  38 venv_install_done = os.path.join(venv_run_dir, "venv_install.done")
  39 papi_python_src_dir = os.path.join(ws_root, "src", "vpp-api", "python")
  40
  41 # Path Variables Set after VPP Build/Install
  42 vpp_build_dir = vpp_install_path = vpp_bin = vpp_lib = vpp_lib64 = None
  43 vpp_plugin_path = vpp_test_plugin_path = ld_library_path = None
  44
  45 # Pip version pinning
  46 pip_version = "22.0.4"
  47 pip_tools_version = "6.6.0"
  48
  49 # Test requirement files
  50 test_requirements_file = os.path.join(test_dir, "requirements.txt")
  51 # Auto-generated requirement file
  52 pip_compiled_requirements_file = os.path.join(test_dir, "requirements-3.txt")
  53
  54
  55 # Gracefully exit after executing cleanup scripts
  56 # upon receiving a SIGINT or SIGTERM
  57 def handler(signum, frame):
  58     print("Received Signal {0}".format(signum))
  59     post_vm_test_run()
  60
  61
  62 signal.signal(signal.SIGINT, handler)
  63 signal.signal(signal.SIGTERM, handler)
  64
  65
  66 def show_progress(stream):
  67     """
  68     Read lines from a subprocess stdout/stderr streams and write
  69     to sys.stdout & the logfile
  70     """
  71     while True:
  72         s = stream.readline()
  73         if not s:
  74             break
  75         data = s.decode("utf-8")
  76         # Filter the annoying SIGTERM signal from the output when VPP is
  77         # terminated after a test run
  78         if "SIGTERM" not in data:
  79             sys.stdout.write(data)
  80             logging.debug(data)
  81         sys.stdout.flush()
  82     stream.close()
  83
  84
  85 class ExtendedEnvBuilder(venv.EnvBuilder):
  86     """
  87     1. Builds a Virtual Environment for running VPP unit tests
  88     2. Installs all necessary scripts, pkgs & patches into the vEnv
  89          - python3, pip, pip-tools, papi, scapy patches &
  90            test-requirement pkgs
  91     """
  92
  93     def __init__(self, *args, **kwargs):
  94         super().__init__(*args, **kwargs)
  95
  96     def post_setup(self, context):
  97         """
  98         Setup all packages that need to be pre-installed into the venv
  99         prior to running VPP unit tests.
 100
 101         :param context: The context of the virtual environment creation
 102                         request being processed.
 103         """
 104         os.environ["VIRTUAL_ENV"] = context.env_dir
 105         os.environ[
 106             "CUSTOM_COMPILE_COMMAND"
 107         ] = "make test-refresh-deps (or update requirements.txt)"
 108         # Cleanup previously auto-generated pip req. file
 109         try:
 110             os.unlink(pip_compiled_requirements_file)
 111         except OSError:
 112             pass
 113         # Set the venv python executable & binary install path
 114         env_exe = context.env_exe
 115         bin_path = context.bin_path
 116         # Packages/requirements to be installed in the venv
 117         # [python-module, cmdline-args, package-name_or_requirements-file-name]
 118         test_req = [
 119             ["pip", "install", "pip===%s" % pip_version],
 120             ["pip", "install", "pip-tools===%s" % pip_tools_version],
 121             [
 122                 "piptools",
 123                 "compile",
 124                 "-q",
 125                 "--generate-hashes",
 126                 test_requirements_file,
 127                 "--output-file",
 128                 pip_compiled_requirements_file,
 129             ],
 130             ["piptools", "sync", pip_compiled_requirements_file],
 131             ["pip", "install", "-e", papi_python_src_dir],
 132         ]
 133         for req in test_req:
 134             args = [env_exe, "-m"]
 135             args.extend(req)
 136             print(args)
 137             p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=bin_path)
 138             show_progress(p.stdout)
 139         self.pip_patch()
 140
 141     def pip_patch(self):
 142         """
 143         Apply scapy patch files
 144         """
 145         scapy_patch_dir = Path(os.path.join(test_dir, "patches", "scapy-2.4.3"))
 146         scapy_source_dir = glob.glob(
 147             os.path.join(venv_lib_dir, "python3.*", "site-packages")
 148         )[0]
 149         for f in scapy_patch_dir.iterdir():
 150             print("Applying patch: {}".format(os.path.basename(str(f))))
 151             args = ["patch", "--forward", "-p1", "-d", scapy_source_dir, "-i", str(f)]
 152             print(args)
 153             p = Popen(args, stdout=PIPE, stderr=STDOUT)
 154             show_progress(p.stdout)
 155
 156
 157 # Build VPP Release/Debug binaries
 158 def build_vpp(debug=True, release=False):
 159     """
 160     Install VPP Release(if release=True) or Debug(if debug=True) Binaries.
 161
 162     Default is to build the debug binaries.
 163     """
 164     global vpp_build_dir, vpp_install_path, vpp_bin, vpp_lib, vpp_lib64
 165     global vpp_plugin_path, vpp_test_plugin_path, ld_library_path
 166     if debug:
 167         print("Building VPP debug binaries")
 168         args = ["make", "build"]
 169         build = "build-vpp_debug-native"
 170         install = "install-vpp_debug-native"
 171     elif release:
 172         print("Building VPP release binaries")
 173         args = ["make", "build-release"]
 174         build = "build-vpp-native"
 175         install = "install-vpp-native"
 176     p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=ws_root)
 177     show_progress(p.stdout)
 178     vpp_build_dir = os.path.join(build_root, build)
 179     vpp_install_path = os.path.join(build_root, install)
 180     vpp_bin = os.path.join(vpp_install_path, "vpp", "bin", "vpp")
 181     vpp_lib = os.path.join(vpp_install_path, "vpp", "lib")
 182     vpp_lib64 = os.path.join(vpp_install_path, "vpp", "lib64")
 183     vpp_plugin_path = (
 184         os.path.join(vpp_lib, "vpp_plugins")
 185         + ":"
 186         + os.path.join(vpp_lib64, "vpp_plugins")
 187     )
 188     vpp_test_plugin_path = (
 189         os.path.join(vpp_lib, "vpp_api_test_plugins")
 190         + ":"
 191         + os.path.join(vpp_lib64, "vpp_api_test_plugins")
 192     )
 193     ld_library_path = os.path.join(vpp_lib) + ":" + os.path.join(vpp_lib64)
 194
 195
 196 # Environment Vars required by the test framework,
 197 # papi_provider & unittests
 198 def set_environ():
 199     os.environ["WS_ROOT"] = ws_root
 200     os.environ["BR"] = build_root
 201     os.environ["VENV_PATH"] = venv_dir
 202     os.environ["VENV_BIN"] = venv_bin_dir
 203     os.environ["RND_SEED"] = str(time.time())
 204     os.environ["VPP_BUILD_DIR"] = vpp_build_dir
 205     os.environ["VPP_BIN"] = vpp_bin
 206     os.environ["VPP_PLUGIN_PATH"] = vpp_plugin_path
 207     os.environ["VPP_TEST_PLUGIN_PATH"] = vpp_test_plugin_path
 208     os.environ["VPP_INSTALL_PATH"] = vpp_install_path
 209     os.environ["LD_LIBRARY_PATH"] = ld_library_path
 210     os.environ["FAILED_DIR"] = "/tmp/vpp-failed-unittests/"
 211     if not os.environ.get("TEST_JOBS"):
 212         os.environ["TEST_JOBS"] = "1"
 213
 214
 215 # Runs a test inside a spawned QEMU VM
 216 # If a kernel image is not provided, a linux-image-kvm image is
 217 # downloaded to the test_data_dir
 218 def vm_test_runner(test_name, kernel_image, test_data_dir, cpu_mask, mem):
 219     script = os.path.join(test_dir, "scripts", "run_vpp_in_vm.sh")
 220     p = Popen(
 221         [script, test_name, kernel_image, test_data_dir, cpu_mask, mem],
 222         stdout=PIPE,
 223         stderr=STDOUT,
 224         cwd=ws_root,
 225     )
 226     show_progress(p.stdout)
 227     post_vm_test_run()
 228
 229
 230 def post_vm_test_run():
 231     # Revert the ownership of certain directories from root to the
 232     # original user after running in QEMU
 233     print("Running post test cleanup tasks")
 234     dirs = ["/tmp/vpp-failed-unittests", os.path.join(ws_root, "test", "__pycache__")]
 235     dirs.extend(glob.glob("/tmp/vpp-unittest-*"))
 236     dirs.extend(glob.glob("/tmp/api_post_mortem.*"))
 237     user = os.getlogin()
 238     for dir in dirs:
 239         if os.path.exists(dir) and Path(dir).owner() != user:
 240             cmd = ["sudo", "chown", "-R", "{0}:{0}".format(user), dir]
 241             p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
 242             show_progress(p.stdout)
 243
 244
 245 def build_venv():
 246     # Builds a virtual env containing all the required packages and patches
 247     # for running VPP unit tests
 248     if not os.path.exists(venv_install_done):
 249         env_builder = ExtendedEnvBuilder(clear=True, with_pip=True)
 250         print("Creating a vEnv for running VPP unit tests in {}".format(venv_dir))
 251         env_builder.create(venv_dir)
 252         # Write state to the venv run dir
 253         Path(venv_run_dir).mkdir(exist_ok=True)
 254         Path(venv_install_done).touch()
 255
 256
 257 def expand_mix_string(s):
 258     # Returns an expanded string computed from a mixrange string (s)
 259     # E.g: If param s = '5-8,10,11' returns '5,6,7,8,10,11'
 260     result = []
 261     for val in s.split(","):
 262         if "-" in val:
 263             start, end = val.split("-")
 264             result.extend(list(range(int(start), int(end) + 1)))
 265         else:
 266             result.append(int(val))
 267     return ",".join(str(i) for i in set(result))
 268
 269
 270 def set_logging(test_data_dir, test_name):
 271     Path(test_data_dir).mkdir(exist_ok=True)
 272     log_file = "vm_{0}_{1}.log".format(test_name, str(time.time())[-5:])
 273     filename = "{0}/{1}".format(test_data_dir, log_file)
 274     Path(filename).touch()
 275     logging.basicConfig(filename=filename, level=logging.DEBUG)
 276
 277
 278 if __name__ == "__main__":
 279     # Build a Virtual Environment for running tests on host & QEMU
 280     parser = argparse.ArgumentParser(
 281         description="Run VPP Unit Tests", formatter_class=argparse.RawTextHelpFormatter
 282     )
 283     parser.add_argument(
 284         "--vm",
 285         dest="vm",
 286         required=True,
 287         action="store_true",
 288         help="Run Test Inside a QEMU VM",
 289     )
 290     parser.add_argument(
 291         "-d",
 292         "--debug",
 293         dest="debug",
 294         required=False,
 295         default=True,
 296         action="store_true",
 297         help="Run Tests on Debug Build",
 298     )
 299     parser.add_argument(
 300         "-r",
 301         "--release",
 302         dest="release",
 303         required=False,
 304         default=False,
 305         action="store_true",
 306         help="Run Tests on release Build",
 307     )
 308     parser.add_argument(
 309         "--test",
 310         dest="test_name",
 311         required=False,
 312         action="store",
 313         default="",
 314         help="Tests to Run",
 315     )
 316     parser.add_argument(
 317         "--vm-kernel-image",
 318         dest="kernel_image",
 319         required=False,
 320         action="store",
 321         default="",
 322         help="Kernel Image Selection to Boot",
 323     )
 324     parser.add_argument(
 325         "--vm-cpu-list",
 326         dest="vm_cpu_list",
 327         required=False,
 328         action="store",
 329         default="5-8",
 330         help="Set CPU Affinity\n"
 331         "E.g. 5-7,10 will schedule on processors "
 332         "#5, #6, #7 and #10. (Default: 5-8)",
 333     )
 334     parser.add_argument(
 335         "--vm-mem",
 336         dest="vm_mem",
 337         required=False,
 338         action="store",
 339         default="2",
 340         help="Guest Memory in Gibibytes\n" "E.g. 4 (Default: 2)",
 341     )
 342     args = parser.parse_args()
 343     # Enable VM tests
 344     if args.vm and args.test_name:
 345         test_data_dir = "/tmp/vpp-vm-tests"
 346         set_logging(test_data_dir, args.test_name)
 347         vm_tests = True
 348     elif args.vm and not args.test_name:
 349         print("Error: The --test argument must be set for running VM tests")
 350         sys.exit(1)
 351     build_venv()
 352     # Build VPP release or debug binaries
 353     debug = False if args.release else True
 354     build_vpp(debug, args.release)
 355     set_environ()
 356     if vm_tests:
 357         print("Running VPP unit test(s):{0} inside a QEMU VM".format(args.test_name))
 358         # Check Available CPUs & Usable Memory
 359         cpus = expand_mix_string(args.vm_cpu_list)
 360         num_cpus, usable_cpus = (len(cpus.split(",")), len(os.sched_getaffinity(0)))
 361         if num_cpus > usable_cpus:
 362             print(f"Error:# of CPUs:{num_cpus} > Avail CPUs:{usable_cpus}")
 363             sys.exit(1)
 364         avail_mem = int(os.popen("free -t -g").readlines()[-1].split()[-1])
 365         if int(args.vm_mem) > avail_mem:
 366             print(f"Error: Mem Size:{args.vm_mem}G > Avail Mem:{avail_mem}G")
 367             sys.exit(1)
 368         vm_test_runner(
 369             args.test_name, args.kernel_image, test_data_dir, cpus, f"{args.vm_mem}G"
 370         )