test/Makefile

   1 ASAN_OPTIONS?=verify_asan_link_order=0:detect_leaks=0:abort_on_error=1:unmap_shadow_on_exit=1:disable_coredump=0
   2 export ASAN_OPTIONS
   3
   4 .PHONY: verify-env
   5 verify-env:
   6 ifndef WS_ROOT
   7         $(error WS_ROOT is not set)
   8 endif
   9 ifndef BR
  10         $(error BR is not set)
  11 endif
  12 ifndef TEST_DIR
  13         $(error TEST_DIR is not set)
  14 endif
  15
  16 export TEST_BR = $(BR)/build-test
  17 export TEST_DOC_BR = $(TEST_BR)/doc
  18 export BUILD_TEST_SRC = $(TEST_BR)/src
  19 FAILED_DIR=/tmp/vpp-failed-unittests/
  20 PLUGIN_TEST_DIRS=$(shell find $(PLUGIN_SRC_DIR) -type d -name test -exec echo -n " -d {}" \;)
  21 CORE_TEST_DIRS=$(shell find $(WS_ROOT)/src -not \( -path $(INTERN_PLUGIN_SRC_DIR) -prune \) -type d -name test -exec echo -n " -d {}" \;)
  22 VPP_TEST_DIRS=$(shell ls -d $(TEST_DIR)$(PLUGIN_TEST_DIRS)$(CORE_TEST_DIRS) $(EXTERN_TESTS))
  23 VPP_TEST_SRC=$(shell for dir in $(VPP_TEST_DIRS) ; do ls $$dir/*.py; done)
  24
  25 .PHONY: verify-no-running-vpp
  26
  27 ifdef VPP_ZOMBIE_NOCHECK
  28 VPP_PIDS=
  29 else
  30 VPP_PIDS=$(shell pgrep -d, -x vpp_main)
  31 endif
  32
  33 ifeq ($(DEBUG),gdb)
  34 FORCE_FOREGROUND=1
  35 else ifeq ($(DEBUG),gdbserver)
  36 FORCE_FOREGROUND=1
  37 else ifeq ($(DEBUG),gdb-all)
  38 FORCE_FOREGROUND=1
  39 else ifeq ($(DEBUG),gdbserver-all)
  40 FORCE_FOREGROUND=1
  41 else ifeq ($(DEBUG),core)
  42 FORCE_FOREGROUND=1
  43 else ifeq ($(STEP),yes)
  44 FORCE_FOREGROUND=1
  45 else ifeq ($(STEP),y)
  46 FORCE_FOREGROUND=1
  47 else ifeq ($(STEP),1)
  48 FORCE_FOREGROUND=1
  49 else
  50 FORCE_FOREGROUND=0
  51 endif
  52
  53 ifdef PROFILE_OUTPUT
  54 PROFILE_OUTPUT_OPTS=-o $(PROFILE_OUTPUT)
  55 endif
  56
  57 ifndef PROFILE_SORT_BY
  58 PROFILE_SORT_BY=cumtime
  59 endif
  60
  61 ifeq ($(PROFILE),1)
  62 PYTHON_PROFILE_OPTS=-m cProfile $(PROFILE_OUTPUT_OPTS) -s $(PROFILE_SORT_BY)
  63 FORCE_FOREGROUND=1
  64 endif
  65
  66 verify-no-running-vpp:
  67         @if [ "$(VPP_PIDS)" != "" ]; then \
  68                 echo; \
  69                 echo "*** Existing vpp processes detected (PID(s): $(VPP_PIDS)). Running tests under these conditions is not supported. ***"; \
  70                 echo; \
  71                 ps -fp $(VPP_PIDS);\
  72                 echo; \
  73                 false; \
  74         fi
  75
  76 UNITTEST_EXTRA_OPTS=
  77 UNITTEST_FAILFAST_OPTS=
  78
  79 ifeq ($(FAILFAST),1)
  80 UNITTEST_EXTRA_OPTS=-f
  81 endif
  82
  83 ifneq ($(EXTERN_TESTS),)
  84 UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS)
  85 endif
  86
  87 VENV_PATH=$(TEST_BR)/venv
  88
  89 ifeq ($(TEST_DEBUG),1)
  90 VENV_RUN_DIR:=$(VENV_PATH)/run-debug
  91 else
  92 VENV_RUN_DIR:=$(VENV_PATH)/run
  93 endif
  94
  95 ifeq ($(PYTHON),)
  96 PYTHON_INTERP=python3
  97 else
  98 PYTHON_INTERP=$(PYTHON)
  99 endif
 100
 101 PYTHON_VERSION=$(shell $(PYTHON_INTERP) -c 'import sys; print(sys.version_info.major)')
 102 PIP_VERSION=19.1.1
 103 PIP_TOOLS_VERSION=3.8.0   # Keep in sync with requirements.txt
 104 PYTHON_DEPENDS=requirements-$(PYTHON_VERSION).txt
 105 SCAPY_SOURCE=$(shell find $(VENV_PATH)/lib/python* -name site-packages)
 106 BUILD_COV_DIR=$(TEST_BR)/coverage
 107
 108 PIP_TOOLS_INSTALL_DONE=$(VENV_RUN_DIR)/pip-tools-install-$(PYTHON_VERSION).done
 109 PIP_INSTALL_DONE=$(VENV_RUN_DIR)/pip-install-$(PYTHON_VERSION).done
 110 PIP_PATCH_DONE=$(VENV_RUN_DIR)/pip-patch-$(PYTHON_VERSION).done
 111 PAPI_INSTALL_DONE=$(VENV_RUN_DIR)/papi-install-$(PYTHON_VERSION).done
 112 PAPI_PYTHON_SRC_DIR=$(WS_ROOT)/src/vpp-api/python
 113 PAPI_WIPE_DIST=$(WS_ROOT)/src/vpp-api/vapi/__pycache__ \
 114         $(PAPI_PYTHON_SRC_DIR)/build \
 115         $(PAPI_PYTHON_SRC_DIR)/vpp_papi.egg-info \
 116         $(PAPI_PYTHON_SRC_DIR)/vpp_papi/__pycache__
 117
 118 PAPI_INSTALL_FLAGS=$(PIP_INSTALL_DONE) $(PIP_PATCH_DONE) $(PAPI_INSTALL_DONE)
 119
 120 $(PIP_TOOLS_INSTALL_DONE):
 121         @rm -rf $(VENV_PATH)
 122         @mkdir -p $(VENV_RUN_DIR)
 123         @virtualenv $(VENV_PATH) -p $(PYTHON_INTERP)
 124         # pip version pinning
 125         @bash -c "source $(VENV_PATH)/bin/activate && \
 126                   $(PYTHON_INTERP) -m pip install pip===$(PIP_VERSION)"
 127         @bash -c "source $(VENV_PATH)/bin/activate && \
 128                   $(PYTHON_INTERP) -m pip install pip-tools===$(PIP_TOOLS_VERSION)"
 129         @touch $@
 130
 131 $(PYTHON_DEPENDS): $(PIP_TOOLS_INSTALL_DONE) requirements.txt
 132         @bash -c "source $(VENV_PATH)/bin/activate && \
 133                   CUSTOM_COMPILE_COMMAND='make test-refresh-deps (or update requirements.txt)' \
 134                   $(PYTHON_INTERP) -m piptools compile -q --generate-hashes requirements.txt --output-file $@"
 135
 136 $(PIP_INSTALL_DONE): $(PYTHON_DEPENDS)
 137         @bash -c "source $(VENV_PATH)/bin/activate && \
 138                   $(PYTHON_INTERP) -m piptools sync $(PYTHON_DEPENDS)"
 139         @touch $@
 140
 141 $(PIP_PATCH_DONE): $(PIP_INSTALL_DONE)
 142         @echo --- patching ---
 143         @sleep 1 # Ensure python recompiles patched *.py files -> *.pyc
 144         for f in $(CURDIR)/patches/scapy-2.4.3/*.patch ; do \
 145                 echo Applying patch: $$(basename $$f) ; \
 146                 patch --forward -p1 -d $(SCAPY_SOURCE) < $$f ; \
 147                 retCode=$$?; \
 148                 [ $$retCode -gt 0 ] && exit $$retCode; \
 149         done; \
 150         touch $@
 151
 152 $(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE)
 153         @bash -c "source $(VENV_PATH)/bin/activate && $(PYTHON_INTERP) -m pip install -e $(PAPI_PYTHON_SRC_DIR)"
 154         @touch $@
 155
 156 .PHONY: refresh-deps
 157 refresh-deps: clean-deps $(PYTHON_DEPENDS)
 158
 159 .PHONY: clean-deps
 160 clean-deps:
 161         @rm -f $(PYTHON_DEPENDS)
 162
 163 INTERN_PLUGIN_SRC_DIR=$(WS_ROOT)/src/plugins
 164 ifneq ($(EXTERN_PLUGIN_SRC_DIR),)
 165 PLUGIN_SRC_DIR=$(EXTERN_PLUGIN_SRC_DIR)
 166 else
 167 PLUGIN_SRC_DIR=$(INTERN_PLUGIN_SRC_DIR)
 168 endif
 169
 170 define retest-func
 171 @env FORCE_FOREGROUND=$(FORCE_FOREGROUND) FAILED_DIR=$(FAILED_DIR) VENV_PATH=$(VENV_PATH) scripts/setsid_wrapper.sh $(FORCE_FOREGROUND) $(VENV_PATH)/bin/activate $(PYTHON_INTERP) $(PYTHON_PROFILE_OPTS) $(BUILD_TEST_SRC)/run_tests.py -d $(BUILD_TEST_SRC) $(UNITTEST_EXTRA_OPTS) || env FAILED_DIR=$(FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh
 172 endef
 173
 174 .PHONY: sanity
 175
 176 ifeq ($(SANITY),no)
 177 SANITY_IMPORT_VPP_PAPI_CMD=true
 178 SANITY_RUN_VPP_CMD=true
 179 else
 180 SANITY_IMPORT_VPP_PAPI_CMD=source $(VENV_PATH)/bin/activate && $(PYTHON_INTERP) $(BUILD_TEST_SRC)/sanity_import_vpp_papi.py
 181 SANITY_RUN_VPP_CMD=source $(VENV_PATH)/bin/activate && PYTHONPATH=$(BUILD_TEST_SRC) $(PYTHON_INTERP) $(BUILD_TEST_SRC)/sanity_run_vpp.py
 182 endif
 183
 184 ifndef TEST_JOBS
 185 PARALLEL_ILLEGAL=0
 186 else ifeq ($(FORCE_FOREGROUND),0)
 187 PARALLEL_ILLEGAL=0
 188 else ifeq ($(TEST_JOBS),auto)
 189 PARALLEL_ILLEGAL=0
 190 else ifeq ($(TEST_JOBS),1)
 191 PARALLEL_ILLEGAL=0
 192 else
 193 PARALLEL_ILLEGAL=1
 194 endif
 195
 196 sanity: test-dep verify-no-running-vpp
 197         @sys_req/dev_shm_size.sh
 198         @bash -c "test $(PARALLEL_ILLEGAL) -eq 0 ||\
 199             (echo \"*******************************************************************\" &&\
 200                  echo \"* Sanity check failed, TEST_JOBS is not 1 or 'auto' and DEBUG, STEP or PROFILE is set\" &&\
 201                  echo \"*******************************************************************\" &&\
 202                  false)"
 203         @bash -c "$(SANITY_IMPORT_VPP_PAPI_CMD) ||\
 204                 (echo \"*******************************************************************\" &&\
 205                  echo \"* Sanity check failed, cannot import vpp_papi\" &&\
 206                  echo \"* to debug: \" &&\
 207                  echo \"* 1. enter test shell:   make test-shell\" &&\
 208                  echo \"* 2. execute debugger:   gdb python -ex 'run sanity_import_vpp_papi.py'\" &&\
 209                  echo \"*******************************************************************\" &&\
 210                  false)"
 211         @bash -c "$(SANITY_RUN_VPP_CMD) ||\
 212                 (echo \"*******************************************************************\" &&\
 213                  echo \"* Sanity check failed, cannot run vpp\" &&\
 214                  echo \"*******************************************************************\" &&\
 215                  false)"
 216
 217 .PHONY: ext-test-apps
 218 ext-test-apps:
 219         make -C ext test-apps
 220
 221 $(BUILD_TEST_SRC): verify-env
 222         @mkdir -p $@
 223         @for file in $(VPP_TEST_SRC); do if [ ! -e $(BUILD_TEST_SRC)/$$(basename $$file) ] ; then ln -s $$file $(BUILD_TEST_SRC) ; fi ; done
 224
 225 $(FAILED_DIR): reset
 226         @mkdir -p $@
 227
 228 .PHONY: test-dep
 229 test-dep: $(BUILD_TEST_SRC) $(PAPI_INSTALL_DONE) $(FAILED_DIR)
 230
 231 .PHONY: test
 232 test: test-dep ext-test-apps sanity
 233         $(call retest-func)
 234
 235 .PHONY: retest
 236 retest: verify-env sanity $(FAILED_DIR)
 237         $(call retest-func)
 238
 239 .PHONY: shell
 240 shell: test-dep
 241         @echo "source $(VENV_PATH)/bin/activate;\
 242                 cd $(BUILD_TEST_SRC);\
 243                 export PYTHONPATH=$(BUILD_TEST_SRC);\
 244                 export RND_SEED=$(RND_SEED);\
 245                 echo '***';\
 246                 echo PYTHONPATH=$(BUILD_TEST_SRC);\
 247                 echo RND_SEED=$(RND_SEED);\
 248                 echo VPP_BUILD_DIR=$(VPP_BUILD_DIR);\
 249                 echo VPP_BIN=$(VPP_BIN);\
 250                 echo VPP_PLUGIN_PATH=$(VPP_PLUGIN_PATH);\
 251                 echo VPP_TEST_PLUGIN_PATH=$(VPP_TEST_PLUGIN_PATH);\
 252                 echo VPP_INSTALL_PATH=$(VPP_INSTALL_PATH);\
 253                 echo EXTERN_TESTS=$(EXTERN_TESTS);\
 254                 echo EXTERN_PLUGINS=$(EXTERN_PLUGINS);\
 255                 echo EXTERN_COV_DIR=$(EXTERN_COV_DIR);\
 256                 echo LD_LIBRARY_PATH=$(LD_LIBRARY_PATH);\
 257                 echo '***';\
 258                 exec </dev/tty" | bash -i
 259
 260 .PHONY: reset
 261 reset:
 262         @rm -f /dev/shm/vpp-unittest-*
 263         @rm -rf /tmp/vpp-unittest-*
 264         @rm -f /tmp/api_post_mortem.*
 265         @rm -rf $(FAILED_DIR)
 266
 267 .PHONY: wipe
 268 wipe: reset
 269         @make -C ext clean
 270         @rm -rf $(VENV_PATH)
 271         @rm -rf $(patsubst %,%/__pycache__, $(VPP_TEST_DIRS) $(BUILD_TEST_SRC))
 272
 273 $(TEST_DOC_BR):
 274         @mkdir -p $@
 275         @bash -c "source $(VENV_PATH)/bin/activate && \
 276                   $(PYTHON_INTERP) -m pip install sphinx sphinx-rtd-theme"
 277         @bash -c "source $(VENV_PATH)/bin/activate && make -C doc html"
 278
 279 .PHONY: doc
 280 doc: $(BUILD_TEST_SRC) $(PIP_PATCH_DONE) $(TEST_DOC_BR)
 281         @echo
 282         @echo "Test Documentation URL: $(TEST_DOC_BR)/html/index.html"
 283         @echo "Run 'make test-wipe-doc test-doc' to rebuild the test docs"
 284         @echo
 285
 286 .PHONY: wipe-doc
 287 wipe-doc:
 288         @rm -rf $(TEST_DOC_BR)
 289
 290 $(BUILD_COV_DIR):
 291         @mkdir -p $@
 292
 293 .PHONY: cov
 294 cov: wipe-cov test-dep ext $(BUILD_COV_DIR)
 295         @lcov --zerocounters --directory $(VPP_BUILD_DIR)
 296         @test -z "$(EXTERN_COV_DIR)" || lcov --zerocounters --directory $(EXTERN_COV_DIR)
 297         $(call retest-func)
 298         @lcov --capture --directory $(VPP_BUILD_DIR) --output-file $(BUILD_COV_DIR)/coverage.info
 299         @test -z "$(EXTERN_COV_DIR)" || lcov --capture --directory $(EXTERN_COV_DIR) --output-file $(BUILD_COV_DIR)/extern-coverage.info
 300         @genhtml $(BUILD_COV_DIR)/coverage.info --output-directory $(BUILD_COV_DIR)/html
 301         @test -z "$(EXTERN_COV_DIR)" || genhtml $(BUILD_COV_DIR)/extern-coverage.info --output-directory $(BUILD_COV_DIR)/extern-html
 302         @echo
 303         @echo "Build finished. Code coverage report is in $(BUILD_COV_DIR)/html/index.html"
 304         @test -z "$(EXTERN_COV_DIR)" || echo "Code coverage report for out-of-tree objects is in $(BUILD_COV_DIR)/extern-html/index.html"
 305
 306 .PHONY: wipe-cov
 307 wipe-cov: wipe
 308         @rm -rf $(BUILD_COV_DIR)
 309
 310 .PHONY: wipe-papi
 311 wipe-papi:
 312         @rm -rf $(PAPI_INSTALL_DONE) $(PAPI_WIPE_DIST)
 313
 314 .PHONY: wipe-all
 315 wipe-all: wipe wipe-papi wipe-doc wipe-cov
 316         @rm -rf $(TEST_BR)
 317
 318 .PHONY: checkstyle
 319 checkstyle: $(BUILD_TEST_SRC) $(PIP_INSTALL_DONE)
 320         @bash -c "source $(VENV_PATH)/bin/activate &&\
 321                   $(PYTHON_INTERP) -m pip install pycodestyle"
 322         @bash -c "source $(VENV_PATH)/bin/activate &&\
 323                 pycodestyle --show-source --ignore=W504,E126,E241,E226,E305,E704,E741,E722 -v $(BUILD_TEST_SRC)/*.py ||\
 324                 (echo \"*******************************************************************\" &&\
 325                  echo \"* Test framework PEP8 compliance check FAILED \" &&\
 326                  echo \"*******************************************************************\" &&\
 327                  false)"
 328         @echo "*******************************************************************"
 329         @echo "* Test framework PEP8 compliance check passed"
 330         @echo "*******************************************************************"
 331
 332 .PHONY: help
 333 help:
 334         @echo "Running tests:"
 335         @echo ""
 336         @echo " test                   - build and run (basic) functional tests"
 337         @echo " test-debug             - build and run (basic) functional tests (debug build)"
 338         @echo " test-all               - build and run functional and extended tests"
 339         @echo " test-all-debug         - build and run functional and extended tests (debug build)"
 340         @echo " retest                 - run functional tests"
 341         @echo " retest-debug           - run functional tests (debug build)"
 342         @echo " retest-all             - run functional and extended tests"
 343         @echo " retest-all-debug       - run functional and extended tests (debug build)"
 344         @echo " test-cov               - generate code coverage report for test framework"
 345         @echo " test-gcov                      - build and run functional tests (gcov build)"
 346
 347         @echo " test-wipe              - wipe (temporary) files generated by unit tests"
 348         @echo " test-wipe-cov          - wipe code coverage report for test framework"
 349         @echo " test-wipe-doc          - wipe documentation for test framework"
 350         @echo " test-wipe-papi         - rebuild vpp_papi sources"
 351         @echo " test-wipe-all          - wipe (temporary) files generated by unit tests, docs, and coverage"
 352         @echo " test-shell             - enter shell with test environment"
 353         @echo " test-shell-debug       - enter shell with test environment (debug build)"
 354         @echo " test-checkstyle      - check PEP8 compliance for test framework"
 355         @echo " test-refresh-deps    - refresh the Python dependencies for the tests"
 356         @echo ""
 357         @echo "Arguments controlling test runs:"
 358         @echo " V=[0|1|2]              - set test verbosity level"
 359         @echo "                          0=ERROR, 1=INFO, 2=DEBUG"
 360         @echo " TEST_JOBS=[<n>|auto]   - use <n> parallel processes for test execution or automatic discovery of maximum acceptable processes (default: 1)"
 361         @echo " CACHE_OUTPUT=[0|1]     - cache VPP stdout/stderr and log as one block after test finishes (default: 1)"
 362         @echo " FAILFAST=[0|1]         - fail fast if 1, complete all tests if 0"
 363         @echo " TIMEOUT=<timeout>      - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)"
 364         @echo " RETRIES=<n>            - retry failed tests <n> times"
 365         @echo " DEBUG=<type>           - set VPP debugging kind"
 366         @echo "    DEBUG=core          - detect coredump and load it in gdb on crash"
 367         @echo "    DEBUG=gdb           - allow easy debugging by printing VPP PID"
 368         @echo "                          and waiting for user input before running"
 369         @echo "                          and tearing down a testcase"
 370         @echo "    DEBUG=gdbserver     - run gdb inside a gdb server, otherwise"
 371         @echo "                          same as above"
 372         @echo " STEP=[yes|no]          - ease debugging by stepping through a testcase"
 373         @echo " SANITY=[yes|no]        - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)"
 374         @echo " EXTENDED_TESTS=[1|y]   - used by '[re]test-all' & '[re]test-all-debug' to run extended tests"
 375         @echo " TEST=<filter>          - filter the set of tests:"
 376         @echo "    by file-name        - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py"
 377         @echo "    by file-suffix      - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py"
 378         @echo "    by wildcard         - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'"
 379         @echo "                          e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name"
 380         @echo "                               TEST='bfd.*.*' is equivalent to above example of filter by file-suffix"
 381         @echo "                               TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class"
 382         @echo "                               TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase"
 383         @echo "                               TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes"
 384         @echo ""
 385         @echo " VPP_ZOMBIE_NOCHECK=1   - skip checking for vpp (zombie) processes (CAUTION)"
 386         @echo " COREDUMP_SIZE=<size>   - pass <size> as unix { coredump-size <size> } argument to vpp"
 387         @echo "                          e.g. COREDUMP_SIZE=4g"
 388         @echo "                               COREDUMP_SIZE=unlimited"
 389         @echo " COREDUMP_COMPRESS=1    - compress core files if not debugging them"
 390         @echo " EXTERN_TESTS=<path>    - path to out-of-tree test_<name>.py files containing test cases"
 391         @echo " EXTERN_PLUGINS=<path>  - path to out-of-tree plugins to be loaded by vpp under test"
 392         @echo " EXTERN_COV_DIR=<path>  - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report"
 393         @echo ""
 394         @echo " PROFILE=1              - enable profiling of test framework via cProfile module"
 395         @echo " PROFILE_SORT_BY=opt    - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)"
 396         @echo " PROFILE_OUTPUT=file    - output profiling info to file - use absolute path (default: stdout)"
 397         @echo ""
 398         @echo " TEST_DEBUG=1           - turn on debugging of the test framework itself (expert)"
 399         @echo ""
 400         @echo " SKIP_AARCH64=1         - skip tests that are failing on the ARM platorm in FD.io CI"
 401         @echo ""
 402         @echo " SOCKET=1               - Communicate with VPP over Unix domain socket instead of SHM"
 403         @echo ""
 404         @echo " RND_SEED=seed          - Seed RND with given seed"
 405         @echo ""
 406         @echo "Creating test documentation"
 407         @echo " test-doc               - generate documentation for test framework"
 408         @echo " test-wipe-doc          - wipe documentation for test framework"
 409         @echo ""
 410         @echo "Creating test code coverage report"
 411         @echo " test-cov               - generate code coverage report for test framework"
 412         @echo " test-wipe-cov          - wipe code coverage report for test framework"
 413         @echo ""
 414         @echo "Verifying code-style"
 415         @echo " test-checkstyle        - check PEP8 compliance"
 416         @echo ""