test/Makefile

   1 ASAN_OPTIONS?=verify_asan_link_order=0:detect_leaks=0:abort_on_error=1:unmap_shadow_on_exit=1:disable_coredump=0
   2 export ASAN_OPTIONS
   3
   4 .PHONY: verify-env
   5 verify-env:
   6 ifndef WS_ROOT
   7         $(error WS_ROOT is not set)
   8 endif
   9 ifndef BR
  10         $(error BR is not set)
  11 endif
  12 ifndef TEST_DIR
  13         $(error TEST_DIR is not set)
  14 endif
  15
  16 export TEST_BR = $(BR)/build-test
  17 export TEST_DOC_BR = $(TEST_BR)/doc
  18 export BUILD_TEST_SRC = $(TEST_BR)/src
  19 FAILED_DIR=/tmp/vpp-failed-unittests/
  20 PLUGIN_TEST_DIRS=$(shell find $(PLUGIN_SRC_DIR) -type d -name test -exec echo -n " -d {}" \;)
  21 CORE_TEST_DIRS=$(shell find $(WS_ROOT)/src -not \( -path $(INTERN_PLUGIN_SRC_DIR) -prune \) -type d -name test -exec echo -n " -d {}" \;)
  22 VPP_TEST_DIRS=$(shell ls -d $(TEST_DIR)$(PLUGIN_TEST_DIRS)$(CORE_TEST_DIRS) $(EXTERN_TESTS))
  23 VPP_TEST_SRC=$(shell for dir in $(VPP_TEST_DIRS) ; do ls $$dir/*.py; done)
  24
  25 .PHONY: verify-no-running-vpp
  26
  27 ifdef VPP_ZOMBIE_NOCHECK
  28 VPP_PIDS=
  29 else
  30 VPP_PIDS=$(shell pgrep -d, -x vpp_main)
  31 endif
  32
  33 ifeq ($(DEBUG),gdb)
  34 FORCE_FOREGROUND=1
  35 else ifeq ($(DEBUG),gdbserver)
  36 FORCE_FOREGROUND=1
  37 else ifeq ($(DEBUG),gdb-all)
  38 FORCE_FOREGROUND=1
  39 else ifeq ($(DEBUG),gdbserver-all)
  40 FORCE_FOREGROUND=1
  41 else ifeq ($(DEBUG),core)
  42 FORCE_FOREGROUND=1
  43 else ifeq ($(STEP),yes)
  44 FORCE_FOREGROUND=1
  45 else ifeq ($(STEP),y)
  46 FORCE_FOREGROUND=1
  47 else ifeq ($(STEP),1)
  48 FORCE_FOREGROUND=1
  49 else
  50 FORCE_FOREGROUND=0
  51 endif
  52
  53 ifdef PROFILE_OUTPUT
  54 PROFILE_OUTPUT_OPTS=-o $(PROFILE_OUTPUT)
  55 endif
  56
  57 ifndef PROFILE_SORT_BY
  58 PROFILE_SORT_BY=cumtime
  59 endif
  60
  61 ifeq ($(PROFILE),1)
  62 PYTHON_PROFILE_OPTS=-m cProfile $(PROFILE_OUTPUT_OPTS) -s $(PROFILE_SORT_BY)
  63 FORCE_FOREGROUND=1
  64 endif
  65
  66 verify-no-running-vpp:
  67         @if [ "$(VPP_PIDS)" != "" ]; then \
  68                 echo; \
  69                 echo "*** Existing vpp processes detected (PID(s): $(VPP_PIDS)). Running tests under these conditions is not supported. ***"; \
  70                 echo; \
  71                 ps -fp $(VPP_PIDS);\
  72                 echo; \
  73                 false; \
  74         fi
  75
  76 UNITTEST_EXTRA_OPTS=
  77 UNITTEST_FAILFAST_OPTS=
  78
  79 ifeq ($(FAILFAST),1)
  80 UNITTEST_EXTRA_OPTS=-f
  81 endif
  82
  83 ifneq ($(EXTERN_TESTS),)
  84 UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS)
  85 endif
  86
  87 VENV_PATH=$(TEST_BR)/venv
  88
  89 ifeq ($(TEST_DEBUG),1)
  90 VENV_RUN_DIR:=$(VENV_PATH)/run-debug
  91 else
  92 VENV_RUN_DIR:=$(VENV_PATH)/run
  93 endif
  94
  95 ifeq ($(PYTHON),)
  96 PYTHON_INTERP=python3
  97 else
  98 PYTHON_INTERP=$(PYTHON)
  99 endif
 100
 101 PYTHON_VERSION=$(shell $(PYTHON_INTERP) -c 'import sys; print(sys.version_info.major)')
 102 PIP_VERSION=20.1.1
 103 # Keep in sync with requirements.txt
 104 PIP_TOOLS_VERSION=5.1.2
 105 PYTHON_DEPENDS=requirements-$(PYTHON_VERSION).txt
 106 SCAPY_SOURCE=$(shell find $(VENV_PATH)/lib/python* -name site-packages)
 107 BUILD_COV_DIR=$(TEST_BR)/coverage
 108
 109 PIP_TOOLS_INSTALL_DONE=$(VENV_RUN_DIR)/pip-tools-install-$(PYTHON_VERSION)-$(PIP_TOOLS_VERSION).done
 110 PIP_INSTALL_DONE=$(VENV_RUN_DIR)/pip-install-$(PYTHON_VERSION)-$(PIP_VERSION).done
 111 PIP_PATCH_DONE=$(VENV_RUN_DIR)/pip-patch-$(PYTHON_VERSION).done
 112 PAPI_INSTALL_DONE=$(VENV_RUN_DIR)/papi-install-$(PYTHON_VERSION).done
 113 PAPI_PYTHON_SRC_DIR=$(WS_ROOT)/src/vpp-api/python
 114 PAPI_WIPE_DIST=$(WS_ROOT)/src/vpp-api/vapi/__pycache__ \
 115         $(PAPI_PYTHON_SRC_DIR)/build \
 116         $(PAPI_PYTHON_SRC_DIR)/vpp_papi.egg-info \
 117         $(PAPI_PYTHON_SRC_DIR)/vpp_papi/__pycache__
 118
 119 PAPI_INSTALL_FLAGS=$(PIP_INSTALL_DONE) $(PIP_PATCH_DONE) $(PAPI_INSTALL_DONE)
 120
 121 $(PIP_TOOLS_INSTALL_DONE):
 122         @rm -rf $(VENV_PATH)
 123         @mkdir -p $(VENV_RUN_DIR)
 124         @virtualenv $(VENV_PATH) -p $(PYTHON_INTERP)
 125         # pip version pinning
 126         @bash -c "source $(VENV_PATH)/bin/activate && \
 127                   $(PYTHON_INTERP) -m pip install pip===$(PIP_VERSION)"
 128         @bash -c "source $(VENV_PATH)/bin/activate && \
 129                   $(PYTHON_INTERP) -m pip install pip-tools===$(PIP_TOOLS_VERSION)"
 130         @touch $@
 131
 132 $(PYTHON_DEPENDS): $(PIP_TOOLS_INSTALL_DONE) requirements.txt
 133         @bash -c "source $(VENV_PATH)/bin/activate && \
 134                   CUSTOM_COMPILE_COMMAND='make test-refresh-deps (or update requirements.txt)' \
 135                   $(PYTHON_INTERP) -m piptools compile -q --generate-hashes requirements.txt --output-file $@"
 136
 137 $(PIP_INSTALL_DONE): $(PYTHON_DEPENDS)
 138         @bash -c "source $(VENV_PATH)/bin/activate && \
 139                   $(PYTHON_INTERP) -m piptools sync $(PYTHON_DEPENDS)"
 140         @touch $@
 141
 142 $(PIP_PATCH_DONE): $(PIP_INSTALL_DONE)
 143         @echo --- patching ---
 144         @sleep 1 # Ensure python recompiles patched *.py files -> *.pyc
 145         for f in $(CURDIR)/patches/scapy-2.4.3/*.patch ; do \
 146                 echo Applying patch: $$(basename $$f) ; \
 147                 patch --forward -p1 -d $(SCAPY_SOURCE) < $$f ; \
 148                 retCode=$$?; \
 149                 [ $$retCode -gt 0 ] && exit $$retCode; \
 150         done; \
 151         touch $@
 152
 153 $(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE)
 154         @bash -c "source $(VENV_PATH)/bin/activate && $(PYTHON_INTERP) -m pip install -e $(PAPI_PYTHON_SRC_DIR)"
 155         @touch $@
 156
 157 .PHONY: refresh-deps
 158 refresh-deps: clean-deps $(PYTHON_DEPENDS)
 159
 160 .PHONY: clean-deps
 161 clean-deps:
 162         @rm -f $(PYTHON_DEPENDS)
 163
 164 INTERN_PLUGIN_SRC_DIR=$(WS_ROOT)/src/plugins
 165 ifneq ($(EXTERN_PLUGIN_SRC_DIR),)
 166 PLUGIN_SRC_DIR=$(EXTERN_PLUGIN_SRC_DIR)
 167 else
 168 PLUGIN_SRC_DIR=$(INTERN_PLUGIN_SRC_DIR)
 169 endif
 170
 171 define retest-func
 172 @env FORCE_FOREGROUND=$(FORCE_FOREGROUND) FAILED_DIR=$(FAILED_DIR) VENV_PATH=$(VENV_PATH) scripts/setsid_wrapper.sh $(FORCE_FOREGROUND) $(VENV_PATH)/bin/activate $(PYTHON_INTERP) $(PYTHON_PROFILE_OPTS) $(BUILD_TEST_SRC)/run_tests.py -d $(BUILD_TEST_SRC) $(UNITTEST_EXTRA_OPTS) || env FAILED_DIR=$(FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh
 173 endef
 174
 175 .PHONY: sanity
 176
 177 ifeq ($(SANITY),no)
 178 SANITY_IMPORT_VPP_PAPI_CMD=true
 179 SANITY_RUN_VPP_CMD=true
 180 else
 181 SANITY_IMPORT_VPP_PAPI_CMD=source $(VENV_PATH)/bin/activate && $(PYTHON_INTERP) $(BUILD_TEST_SRC)/sanity_import_vpp_papi.py
 182 SANITY_RUN_VPP_CMD=source $(VENV_PATH)/bin/activate && PYTHONPATH=$(BUILD_TEST_SRC) $(PYTHON_INTERP) $(BUILD_TEST_SRC)/sanity_run_vpp.py
 183 endif
 184
 185 ifndef TEST_JOBS
 186 PARALLEL_ILLEGAL=0
 187 else ifeq ($(FORCE_FOREGROUND),0)
 188 PARALLEL_ILLEGAL=0
 189 else ifeq ($(TEST_JOBS),auto)
 190 PARALLEL_ILLEGAL=0
 191 else ifeq ($(TEST_JOBS),1)
 192 PARALLEL_ILLEGAL=0
 193 else
 194 PARALLEL_ILLEGAL=1
 195 endif
 196
 197 sanity: test-dep verify-no-running-vpp
 198         @sys_req/dev_shm_size.sh
 199         @bash -c "test $(PARALLEL_ILLEGAL) -eq 0 ||\
 200             (echo \"*******************************************************************\" &&\
 201                  echo \"* Sanity check failed, TEST_JOBS is not 1 or 'auto' and DEBUG, STEP or PROFILE is set\" &&\
 202                  echo \"*******************************************************************\" &&\
 203                  false)"
 204         @bash -c "$(SANITY_IMPORT_VPP_PAPI_CMD) ||\
 205                 (echo \"*******************************************************************\" &&\
 206                  echo \"* Sanity check failed, cannot import vpp_papi\" &&\
 207                  echo \"* to debug: \" &&\
 208                  echo \"* 1. enter test shell:   make test-shell\" &&\
 209                  echo \"* 2. execute debugger:   gdb python -ex 'run sanity_import_vpp_papi.py'\" &&\
 210                  echo \"*******************************************************************\" &&\
 211                  false)"
 212         @bash -c "$(SANITY_RUN_VPP_CMD) ||\
 213                 (echo \"*******************************************************************\" &&\
 214                  echo \"* Sanity check failed, cannot run vpp\" &&\
 215                  echo \"*******************************************************************\" &&\
 216                  false)"
 217
 218 .PHONY: ext-test-apps
 219 ext-test-apps:
 220         make -C ext test-apps
 221
 222 $(BUILD_TEST_SRC): verify-env
 223         @mkdir -p $@
 224         @for file in $(VPP_TEST_SRC); do if [ ! -e $(BUILD_TEST_SRC)/$$(basename $$file) ] ; then ln -s $$file $(BUILD_TEST_SRC) ; fi ; done
 225
 226 $(FAILED_DIR): reset
 227         @mkdir -p $@
 228
 229 .PHONY: test-dep
 230 test-dep: $(BUILD_TEST_SRC) $(PAPI_INSTALL_DONE) $(FAILED_DIR)
 231
 232 .PHONY: test
 233 test: test-dep ext-test-apps sanity
 234         $(call retest-func)
 235
 236 .PHONY: retest
 237 retest: verify-env sanity $(FAILED_DIR)
 238         $(call retest-func)
 239
 240 .PHONY: shell
 241 shell: test-dep
 242         @echo "source $(VENV_PATH)/bin/activate;\
 243                 cd $(BUILD_TEST_SRC);\
 244                 export PYTHONPATH=$(BUILD_TEST_SRC);\
 245                 export RND_SEED=$(RND_SEED);\
 246                 echo '***';\
 247                 echo PYTHONPATH=$(BUILD_TEST_SRC);\
 248                 echo RND_SEED=$(RND_SEED);\
 249                 echo VPP_BUILD_DIR=$(VPP_BUILD_DIR);\
 250                 echo VPP_BIN=$(VPP_BIN);\
 251                 echo VPP_PLUGIN_PATH=$(VPP_PLUGIN_PATH);\
 252                 echo VPP_TEST_PLUGIN_PATH=$(VPP_TEST_PLUGIN_PATH);\
 253                 echo VPP_INSTALL_PATH=$(VPP_INSTALL_PATH);\
 254                 echo EXTERN_TESTS=$(EXTERN_TESTS);\
 255                 echo EXTERN_PLUGINS=$(EXTERN_PLUGINS);\
 256                 echo EXTERN_COV_DIR=$(EXTERN_COV_DIR);\
 257                 echo LD_LIBRARY_PATH=$(LD_LIBRARY_PATH);\
 258                 echo '***';\
 259                 exec </dev/tty" | bash -i
 260
 261 .PHONY: reset
 262 reset:
 263         @rm -f /dev/shm/vpp-unittest-*
 264         @rm -rf /tmp/vpp-unittest-*
 265         @rm -f /tmp/api_post_mortem.*
 266         @rm -rf $(FAILED_DIR)
 267
 268 .PHONY: wipe
 269 wipe: reset
 270         @make -C ext clean
 271         @rm -rf $(VENV_PATH)
 272         @rm -rf $(patsubst %,%/__pycache__, $(VPP_TEST_DIRS) $(BUILD_TEST_SRC))
 273
 274 $(TEST_DOC_BR): $(PIP_INSTALL_DONE)
 275         @mkdir -p $@
 276         @bash -c "source $(VENV_PATH)/bin/activate && make -C doc html"
 277
 278 .PHONY: doc
 279 doc: $(BUILD_TEST_SRC) $(PIP_PATCH_DONE) $(TEST_DOC_BR)
 280         @echo
 281         @echo "Test Documentation URL: $(TEST_DOC_BR)/html/index.html"
 282         @echo "Run 'make test-wipe-doc test-doc' to rebuild the test docs"
 283         @echo
 284
 285 .PHONY: wipe-doc
 286 wipe-doc:
 287         @rm -rf $(TEST_DOC_BR)
 288
 289 $(BUILD_COV_DIR):
 290         @mkdir -p $@
 291
 292 .PHONY: cov
 293 cov: wipe-cov test-dep ext $(BUILD_COV_DIR)
 294         @lcov --zerocounters --directory $(VPP_BUILD_DIR)
 295         @test -z "$(EXTERN_COV_DIR)" || lcov --zerocounters --directory $(EXTERN_COV_DIR)
 296         $(call retest-func)
 297         @lcov --capture --directory $(VPP_BUILD_DIR) --output-file $(BUILD_COV_DIR)/coverage.info
 298         @test -z "$(EXTERN_COV_DIR)" || lcov --capture --directory $(EXTERN_COV_DIR) --output-file $(BUILD_COV_DIR)/extern-coverage.info
 299         @genhtml $(BUILD_COV_DIR)/coverage.info --output-directory $(BUILD_COV_DIR)/html
 300         @test -z "$(EXTERN_COV_DIR)" || genhtml $(BUILD_COV_DIR)/extern-coverage.info --output-directory $(BUILD_COV_DIR)/extern-html
 301         @echo
 302         @echo "Build finished. Code coverage report is in $(BUILD_COV_DIR)/html/index.html"
 303         @test -z "$(EXTERN_COV_DIR)" || echo "Code coverage report for out-of-tree objects is in $(BUILD_COV_DIR)/extern-html/index.html"
 304
 305 .PHONY: wipe-cov
 306 wipe-cov: wipe
 307         @rm -rf $(BUILD_COV_DIR)
 308
 309 .PHONY: wipe-papi
 310 wipe-papi:
 311         @rm -rf $(PAPI_INSTALL_DONE) $(PAPI_WIPE_DIST)
 312
 313 .PHONY: wipe-all
 314 wipe-all: wipe wipe-papi wipe-doc wipe-cov
 315         @rm -rf $(TEST_BR)
 316
 317 .PHONY: checkstyle
 318 checkstyle: $(BUILD_TEST_SRC) $(PIP_INSTALL_DONE)
 319         @bash -c "source $(VENV_PATH)/bin/activate &&\
 320                   $(PYTHON_INTERP) -m pip install pycodestyle"
 321         @bash -c "source $(VENV_PATH)/bin/activate &&\
 322                 pycodestyle --show-source --ignore=W504,E126,E241,E226,E305,E704,E741,E722 -v $(BUILD_TEST_SRC)/*.py ||\
 323                 (echo \"*******************************************************************\" &&\
 324                  echo \"* Test framework PEP8 compliance check FAILED \" &&\
 325                  echo \"*******************************************************************\" &&\
 326                  false)"
 327         @echo "*******************************************************************"
 328         @echo "* Test framework PEP8 compliance check passed"
 329         @echo "*******************************************************************"
 330
 331 .PHONY: help
 332 help:
 333         @echo "Running tests:"
 334         @echo ""
 335         @echo " test                   - build and run (basic) functional tests"
 336         @echo " test-debug             - build and run (basic) functional tests (debug build)"
 337         @echo " test-all               - build and run functional and extended tests"
 338         @echo " test-all-debug         - build and run functional and extended tests (debug build)"
 339         @echo " retest                 - run functional tests"
 340         @echo " retest-debug           - run functional tests (debug build)"
 341         @echo " retest-all             - run functional and extended tests"
 342         @echo " retest-all-debug       - run functional and extended tests (debug build)"
 343         @echo " test-cov               - generate code coverage report for test framework"
 344         @echo " test-gcov                      - build and run functional tests (gcov build)"
 345
 346         @echo " test-wipe              - wipe (temporary) files generated by unit tests"
 347         @echo " test-wipe-cov          - wipe code coverage report for test framework"
 348         @echo " test-wipe-doc          - wipe documentation for test framework"
 349         @echo " test-wipe-papi         - rebuild vpp_papi sources"
 350         @echo " test-wipe-all          - wipe (temporary) files generated by unit tests, docs, and coverage"
 351         @echo " test-shell             - enter shell with test environment"
 352         @echo " test-shell-debug       - enter shell with test environment (debug build)"
 353         @echo " test-checkstyle      - check PEP8 compliance for test framework"
 354         @echo " test-refresh-deps    - refresh the Python dependencies for the tests"
 355         @echo ""
 356         @echo "Arguments controlling test runs:"
 357         @echo " V=[0|1|2]              - set test verbosity level"
 358         @echo "                          0=ERROR, 1=INFO, 2=DEBUG"
 359         @echo " TEST_JOBS=[<n>|auto]   - use <n> parallel processes for test execution or automatic discovery of maximum acceptable processes (default: 1)"
 360         @echo " CACHE_OUTPUT=[0|1]     - cache VPP stdout/stderr and log as one block after test finishes (default: 1)"
 361         @echo " FAILFAST=[0|1]         - fail fast if 1, complete all tests if 0"
 362         @echo " TIMEOUT=<timeout>      - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)"
 363         @echo " RETRIES=<n>            - retry failed tests <n> times"
 364         @echo " DEBUG=<type>           - set VPP debugging kind"
 365         @echo "    DEBUG=core          - detect coredump and load it in gdb on crash"
 366         @echo "    DEBUG=gdb           - allow easy debugging by printing VPP PID"
 367         @echo "                          and waiting for user input before running"
 368         @echo "                          and tearing down a testcase"
 369         @echo "    DEBUG=gdbserver     - run gdb inside a gdb server, otherwise"
 370         @echo "                          same as above"
 371         @echo " STEP=[yes|no]          - ease debugging by stepping through a testcase"
 372         @echo " SANITY=[yes|no]        - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)"
 373         @echo " EXTENDED_TESTS=[1|y]   - used by '[re]test-all' & '[re]test-all-debug' to run extended tests"
 374         @echo " TEST=<filter>          - filter the set of tests:"
 375         @echo "    by file-name        - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py"
 376         @echo "    by file-suffix      - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py"
 377         @echo "    by wildcard         - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'"
 378         @echo "                          e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name"
 379         @echo "                               TEST='bfd.*.*' is equivalent to above example of filter by file-suffix"
 380         @echo "                               TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class"
 381         @echo "                               TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase"
 382         @echo "                               TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes"
 383         @echo ""
 384         @echo " VARIANT=<variant>      - specify which march node variant to unit test"
 385         @echo "                          e.g. VARIANT=avx2 test the avx2 march variants"
 386         @echo "                          e.g. VARIANT=avx512 test the avx512 march variants"
 387         @echo ""
 388         @echo " VPP_ZOMBIE_NOCHECK=1   - skip checking for vpp (zombie) processes (CAUTION)"
 389         @echo " COREDUMP_SIZE=<size>   - pass <size> as unix { coredump-size <size> } argument to vpp"
 390         @echo "                          e.g. COREDUMP_SIZE=4g"
 391         @echo "                               COREDUMP_SIZE=unlimited"
 392         @echo " COREDUMP_COMPRESS=1    - compress core files if not debugging them"
 393         @echo " EXTERN_TESTS=<path>    - path to out-of-tree test_<name>.py files containing test cases"
 394         @echo " EXTERN_PLUGINS=<path>  - path to out-of-tree plugins to be loaded by vpp under test"
 395         @echo " EXTERN_COV_DIR=<path>  - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report"
 396         @echo ""
 397         @echo " PROFILE=1              - enable profiling of test framework via cProfile module"
 398         @echo " PROFILE_SORT_BY=opt    - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)"
 399         @echo " PROFILE_OUTPUT=file    - output profiling info to file - use absolute path (default: stdout)"
 400         @echo ""
 401         @echo " TEST_DEBUG=1           - turn on debugging of the test framework itself (expert)"
 402         @echo ""
 403         @echo " SKIP_AARCH64=1         - skip tests that are failing on the ARM platorm in FD.io CI"
 404         @echo ""
 405         @echo " SOCKET=1               - Communicate with VPP over Unix domain socket instead of SHM"
 406         @echo ""
 407         @echo " RND_SEED=seed          - Seed RND with given seed"
 408         @echo ""
 409         @echo "Creating test documentation"
 410         @echo " test-doc               - generate documentation for test framework"
 411         @echo " test-wipe-doc          - wipe documentation for test framework"
 412         @echo ""
 413         @echo "Creating test code coverage report"
 414         @echo " test-cov               - generate code coverage report for test framework"
 415         @echo " test-wipe-cov          - wipe code coverage report for test framework"
 416         @echo ""
 417         @echo "Verifying code-style"
 418         @echo " test-checkstyle        - check PEP8 compliance"
 419         @echo ""