3 # Copyright (c) 2022 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # Build the Virtual Environment & run VPP unit tests
22 from pathlib import Path
24 from subprocess import Popen, PIPE, STDOUT, call
32 # Required Std. Path Variables
33 test_dir = os.path.dirname(os.path.realpath(__file__))
34 ws_root = os.path.dirname(test_dir)
35 build_root = os.path.join(ws_root, "build-root")
36 venv_dir = os.path.join(build_root, "test", "venv")
37 venv_bin_dir = os.path.join(venv_dir, "bin")
38 venv_lib_dir = os.path.join(venv_dir, "lib")
39 venv_run_dir = os.path.join(venv_dir, "run")
40 venv_install_done = os.path.join(venv_run_dir, "venv_install.done")
41 papi_python_src_dir = os.path.join(ws_root, "src", "vpp-api", "python")
43 # Path Variables Set after VPP Build/Install
44 vpp_build_dir = vpp_install_path = vpp_bin = vpp_lib = vpp_lib64 = None
45 vpp_plugin_path = vpp_test_plugin_path = ld_library_path = None
48 pip_version = "22.0.4"
49 pip_tools_version = "6.6.0"
51 # Test requirement files
52 test_requirements_file = os.path.join(test_dir, "requirements.txt")
53 # Auto-generated requirement file
54 pip_compiled_requirements_file = os.path.join(test_dir, "requirements-3.txt")
57 # Gracefully exit after executing cleanup scripts
58 # upon receiving a SIGINT or SIGTERM
59 def handler(signum, frame):
60 print("Received Signal {0}".format(signum))
64 signal.signal(signal.SIGINT, handler)
65 signal.signal(signal.SIGTERM, handler)
68 def show_progress(stream, exclude_pattern=None):
70 Read lines from a subprocess stdout/stderr streams and write
71 to sys.stdout & the logfile
74 stream - subprocess stdout or stderr data stream
75 exclude_pattern - lines matching this reg-ex will be excluded
82 data = s.decode("utf-8")
83 # Filter the annoying SIGTERM signal from the output when VPP is
84 # terminated after a test run
85 if "SIGTERM" not in data:
86 if exclude_pattern is not None:
87 if bool(re.search(exclude_pattern, data)) is False:
88 sys.stdout.write(data)
90 sys.stdout.write(data)
96 class ExtendedEnvBuilder(venv.EnvBuilder):
98 1. Builds a Virtual Environment for running VPP unit tests
99 2. Installs all necessary scripts, pkgs & patches into the vEnv
100 - python3, pip, pip-tools, papi, scapy patches &
101 test-requirement pkgs
104 def __init__(self, *args, **kwargs):
105 super().__init__(*args, **kwargs)
107 def post_setup(self, context):
109 Setup all packages that need to be pre-installed into the venv
110 prior to running VPP unit tests.
112 :param context: The context of the virtual environment creation
113 request being processed.
115 os.environ["VIRTUAL_ENV"] = context.env_dir
117 "CUSTOM_COMPILE_COMMAND"
118 ] = "make test-refresh-deps (or update requirements.txt)"
119 # Cleanup previously auto-generated pip req. file
121 os.unlink(pip_compiled_requirements_file)
124 # Set the venv python executable & binary install path
125 env_exe = context.env_exe
126 bin_path = context.bin_path
127 # Packages/requirements to be installed in the venv
128 # [python-module, cmdline-args, package-name_or_requirements-file-name]
130 ["pip", "install", "pip===%s" % pip_version],
131 ["pip", "install", "pip-tools===%s" % pip_tools_version],
137 test_requirements_file,
139 pip_compiled_requirements_file,
141 ["piptools", "sync", pip_compiled_requirements_file],
142 ["pip", "install", "-e", papi_python_src_dir],
145 args = [env_exe, "-m"]
148 p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=bin_path)
149 show_progress(p.stdout)
154 Apply scapy patch files
156 scapy_patch_dir = Path(os.path.join(test_dir, "patches", "scapy-2.4.3"))
157 scapy_source_dir = glob.glob(
158 os.path.join(venv_lib_dir, "python3.*", "site-packages")
160 for f in scapy_patch_dir.iterdir():
161 print("Applying patch: {}".format(os.path.basename(str(f))))
162 args = ["patch", "--forward", "-p1", "-d", scapy_source_dir, "-i", str(f)]
164 p = Popen(args, stdout=PIPE, stderr=STDOUT)
165 show_progress(p.stdout)
168 # Build VPP Release/Debug binaries
169 def build_vpp(debug=True, release=False):
171 Install VPP Release(if release=True) or Debug(if debug=True) Binaries.
173 Default is to build the debug binaries.
175 global vpp_build_dir, vpp_install_path, vpp_bin, vpp_lib, vpp_lib64
176 global vpp_plugin_path, vpp_test_plugin_path, ld_library_path
178 print("Building VPP debug binaries")
179 args = ["make", "build"]
180 build = "build-vpp_debug-native"
181 install = "install-vpp_debug-native"
183 print("Building VPP release binaries")
184 args = ["make", "build-release"]
185 build = "build-vpp-native"
186 install = "install-vpp-native"
187 p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=ws_root)
188 show_progress(p.stdout)
189 vpp_build_dir = os.path.join(build_root, build)
190 vpp_install_path = os.path.join(build_root, install)
191 vpp_bin = os.path.join(vpp_install_path, "vpp", "bin", "vpp")
192 vpp_lib = os.path.join(vpp_install_path, "vpp", "lib")
193 vpp_lib64 = os.path.join(vpp_install_path, "vpp", "lib64")
195 os.path.join(vpp_lib, "vpp_plugins")
197 + os.path.join(vpp_lib64, "vpp_plugins")
199 vpp_test_plugin_path = (
200 os.path.join(vpp_lib, "vpp_api_test_plugins")
202 + os.path.join(vpp_lib64, "vpp_api_test_plugins")
204 ld_library_path = os.path.join(vpp_lib) + ":" + os.path.join(vpp_lib64)
207 # Environment Vars required by the test framework,
208 # papi_provider & unittests
210 os.environ["WS_ROOT"] = ws_root
211 os.environ["BR"] = build_root
212 os.environ["VENV_PATH"] = venv_dir
213 os.environ["VENV_BIN"] = venv_bin_dir
214 os.environ["RND_SEED"] = str(time.time())
215 os.environ["VPP_BUILD_DIR"] = vpp_build_dir
216 os.environ["VPP_BIN"] = vpp_bin
217 os.environ["VPP_PLUGIN_PATH"] = vpp_plugin_path
218 os.environ["VPP_TEST_PLUGIN_PATH"] = vpp_test_plugin_path
219 os.environ["VPP_INSTALL_PATH"] = vpp_install_path
220 os.environ["LD_LIBRARY_PATH"] = ld_library_path
221 os.environ["FAILED_DIR"] = "/tmp/vpp-failed-unittests/"
222 if not os.environ.get("TEST_JOBS"):
223 os.environ["TEST_JOBS"] = "1"
226 # Runs a test inside a spawned QEMU VM
227 # If a kernel image is not provided, a linux-image-kvm image is
228 # downloaded to the test_data_dir
229 def vm_test_runner(test_name, kernel_image, test_data_dir, cpu_mask, mem, jobs="auto"):
230 script = os.path.join(test_dir, "scripts", "run_vpp_in_vm.sh")
231 os.environ["TEST_JOBS"] = str(jobs)
233 [script, test_name, kernel_image, test_data_dir, cpu_mask, mem],
237 # Show only the test result without clobbering the stdout.
238 # The VM console displays VPP stderr & Linux IPv6 netdev change
239 # messages, which is logged by default and can be excluded.
240 exclude_pattern = r"vpp\[\d+\]:|ADDRCONF\(NETDEV_CHANGE\):"
241 show_progress(p.stdout, exclude_pattern)
245 def post_vm_test_run():
246 # Revert the ownership of certain directories from root to the
247 # original user after running in QEMU
248 print("Running post test cleanup tasks")
249 dirs = ["/tmp/vpp-failed-unittests", os.path.join(ws_root, "test", "__pycache__")]
250 dirs.extend(glob.glob("/tmp/vpp-unittest-*"))
251 dirs.extend(glob.glob("/tmp/api_post_mortem.*"))
254 if os.path.exists(dir) and Path(dir).owner() != user:
255 cmd = ["sudo", "chown", "-R", "{0}:{0}".format(user), dir]
256 p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
257 show_progress(p.stdout)
261 # Builds a virtual env containing all the required packages and patches
262 # for running VPP unit tests
263 if not os.path.exists(venv_install_done):
264 env_builder = ExtendedEnvBuilder(clear=True, with_pip=True)
265 print("Creating a vEnv for running VPP unit tests in {}".format(venv_dir))
266 env_builder.create(venv_dir)
267 # Write state to the venv run dir
268 Path(venv_run_dir).mkdir(exist_ok=True)
269 Path(venv_install_done).touch()
272 def expand_mix_string(s):
273 # Returns an expanded string computed from a mixrange string (s)
274 # E.g: If param s = '5-8,10,11' returns '5,6,7,8,10,11'
276 for val in s.split(","):
278 start, end = val.split("-")
279 result.extend(list(range(int(start), int(end) + 1)))
281 result.append(int(val))
282 return ",".join(str(i) for i in set(result))
285 def set_logging(test_data_dir, test_name):
286 Path(test_data_dir).mkdir(exist_ok=True)
287 log_file = "vm_{0}_{1}.log".format(test_name, str(time.time())[-5:])
288 filename = "{0}/{1}".format(test_data_dir, log_file)
289 Path(filename).touch()
290 logging.basicConfig(filename=filename, level=logging.DEBUG)
293 def run_tests_in_venv(
301 """Runs tests in the virtual environment set by venv_dir.
304 test: Name of the test to run
305 jobs: Maximum concurrent test jobs
306 log_dir: Directory location for storing log files
307 socket_dir: Use running VPP's socket files
308 running_vpp: True if tests are run against a running VPP
309 extended: Run extended tests
311 script = os.path.join(test_dir, "scripts", "run.sh")
313 f"--venv-dir={venv_dir}",
314 f"--vpp-ws-dir={ws_root}",
315 f"--socket-dir={socket_dir}",
318 f"--log-dir={log_dir}",
319 f"--tmp-dir={log_dir}",
320 f"--cache-vpp-output",
323 args = args + [f"--use-running-vpp"]
325 args = args + [f"--extended"]
326 print(f"Running script: {script} " f"{' '.join(args)}")
327 process_args = [script] + args
331 if __name__ == "__main__":
332 # Build a Virtual Environment for running tests on host & QEMU
333 # (TODO): Create a single config object by merging the below args with
334 # config.py after gathering dev use-cases.
335 parser = argparse.ArgumentParser(
336 description="Run VPP Unit Tests", formatter_class=argparse.RawTextHelpFormatter
343 help="Run Test Inside a QEMU VM",
351 help="Run Tests on Debug Build",
359 help="Run Tests on release Build",
368 help="Test Name or Test filter",
376 help="Kernel Image Selection to Boot",
384 help="Set CPU Affinity\n"
385 "E.g. 5-7,10 will schedule on processors "
386 "#5, #6, #7 and #10. (Default: 5-8)",
394 help="Guest Memory in Gibibytes\n" "E.g. 4 (Default: 2)",
399 default=os.path.abspath(f"./test-run-{datetime.date.today()}"),
400 help="directory where to store directories "
401 "containing log files (default: ./test-run-YYYY-MM-DD)",
407 help="maximum concurrent test jobs",
416 help="Runs tests against a running VPP.",
425 help="Relative or absolute path of running VPP's socket directory "
426 "containing api.sock & stats.sock files.\n"
427 "Default: /var/run/vpp if VPP is started as the root user, else "
428 "/var/run/user/${uid}/vpp.",
437 help="Run extended tests.",
439 args = parser.parse_args()
442 if args.vm and args.test_name:
443 test_data_dir = "/tmp/vpp-vm-tests"
444 set_logging(test_data_dir, args.test_name)
446 elif args.vm and not args.test_name:
447 print("Error: The --test argument must be set for running VM tests")
450 # Build VPP release or debug binaries
451 debug = False if args.release else True
452 build_vpp(debug, args.release)
455 print("Tests will be run against a running VPP..")
457 print("Tests will be run by spawning a new VPP instance..")
458 # Run tests against a running VPP or a new instance of VPP
463 log_dir=args.log_dir,
464 socket_dir=args.socket_dir,
465 running_vpp=args.running_vpp,
466 extended=args.extended,
468 # Run tests against a VPP inside a VM
470 print("Running VPP unit test(s):{0} inside a QEMU VM".format(args.test_name))
471 # Check Available CPUs & Usable Memory
472 cpus = expand_mix_string(args.vm_cpu_list)
473 num_cpus, usable_cpus = (len(cpus.split(",")), len(os.sched_getaffinity(0)))
474 if num_cpus > usable_cpus:
475 print(f"Error:# of CPUs:{num_cpus} > Avail CPUs:{usable_cpus}")
477 avail_mem = int(os.popen("free -t -g").readlines()[-1].split()[-1])
478 if int(args.vm_mem) > avail_mem:
479 print(f"Error: Mem Size:{args.vm_mem}G > Avail Mem:{avail_mem}G")