X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=test%2FMakefile;h=bf5f91123c2adda5b0959bc610d150b90fda0350;hb=a8a9a8a569d43343f67219010bedc1e8b51824cb;hp=244822beb0866b2ce2a1ee82c8eb9ff88416926b;hpb=6e6ad64a4cc97f9f777891c0349bbb129ab03013;p=vpp.git

diff --git a/test/Makefile b/test/Makefile
index 244822beb08..bf5f91123c2 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -13,23 +13,11 @@ ifndef TEST_DIR
 	$(error TEST_DIR is not set)
 endif
 
-export TEST_BR = $(BR)/build-test
-export TEST_DOC_BR = $(TEST_BR)/doc
-export BUILD_TEST_SRC = $(TEST_BR)/src
+export TEST_BR = $(TEST_DIR)
 FAILED_DIR=/tmp/vpp-failed-unittests/
-PLUGIN_TEST_DIRS=$(shell find $(PLUGIN_SRC_DIR) -type d -name test -exec echo -n " -d {}" \;)
-CORE_TEST_DIRS=$(shell find $(WS_ROOT)/src -not \( -path $(INTERN_PLUGIN_SRC_DIR) -prune \) -type d -name test -exec echo -n " -d {}" \;)
-VPP_TEST_DIRS=$(shell ls -d $(TEST_DIR)$(PLUGIN_TEST_DIRS)$(CORE_TEST_DIRS) $(EXTERN_TESTS))
-VPP_TEST_SRC=$(shell for dir in $(VPP_TEST_DIRS) ; do ls $$dir/*.py; done)
-
-.PHONY: verify-no-running-vpp
-
-ifdef VPP_ZOMBIE_NOCHECK
-VPP_PIDS=
-else
-VPP_PIDS=$(shell pgrep -d, -x vpp_main)
-endif
+VPP_TEST_DIRS=$(shell ls -d $(TEST_DIR) $(EXTERN_TESTS))
 
+FORCE_NO_WIPE=0
 ifeq ($(DEBUG),gdb)
 FORCE_FOREGROUND=1
 else ifeq ($(DEBUG),gdbserver)
@@ -40,6 +28,9 @@ else ifeq ($(DEBUG),gdbserver-all)
 FORCE_FOREGROUND=1
 else ifeq ($(DEBUG),core)
 FORCE_FOREGROUND=1
+else ifeq ($(DEBUG),attach)
+FORCE_FOREGROUND=1
+FORCE_NO_WIPE=1
 else ifeq ($(STEP),yes)
 FORCE_FOREGROUND=1
 else ifeq ($(STEP),y)
@@ -63,16 +54,6 @@ PYTHON_PROFILE_OPTS=-m cProfile $(PROFILE_OUTPUT_OPTS) -s $(PROFILE_SORT_BY)
 FORCE_FOREGROUND=1
 endif
 
-verify-no-running-vpp:
-	@if [ "$(VPP_PIDS)" != "" ]; then \
-		echo; \
-		echo "*** Existing vpp processes detected (PID(s): $(VPP_PIDS)). Running tests under these conditions is not supported. ***"; \
-		echo; \
-		ps -fp $(VPP_PIDS);\
-		echo; \
-		false; \
-	fi
-
 UNITTEST_EXTRA_OPTS=
 UNITTEST_FAILFAST_OPTS=
 
@@ -84,7 +65,7 @@ ifneq ($(EXTERN_TESTS),)
 UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS)
 endif
 
-VENV_PATH=$(TEST_BR)/venv
+VENV_PATH=$(TEST_DIR)/venv
 
 ifeq ($(TEST_DEBUG),1)
 VENV_RUN_DIR:=$(VENV_PATH)/run-debug
@@ -99,14 +80,15 @@ PYTHON_INTERP=$(PYTHON)
 endif
 
 PYTHON_VERSION=$(shell $(PYTHON_INTERP) -c 'import sys; print(sys.version_info.major)')
-PIP_VERSION=19.1.1
-PIP_TOOLS_VERSION=3.8.0   # Keep in sync with requirements.txt
+PIP_VERSION=21.3.1
+# Keep in sync with requirements.txt
+PIP_TOOLS_VERSION=6.4.0
 PYTHON_DEPENDS=requirements-$(PYTHON_VERSION).txt
 SCAPY_SOURCE=$(shell find $(VENV_PATH)/lib/python* -name site-packages)
 BUILD_COV_DIR=$(TEST_BR)/coverage
 
-PIP_TOOLS_INSTALL_DONE=$(VENV_RUN_DIR)/pip-tools-install-$(PYTHON_VERSION).done
-PIP_INSTALL_DONE=$(VENV_RUN_DIR)/pip-install-$(PYTHON_VERSION).done
+PIP_TOOLS_INSTALL_DONE=$(VENV_RUN_DIR)/pip-tools-install-$(PYTHON_VERSION)-$(PIP_TOOLS_VERSION).done
+PIP_INSTALL_DONE=$(VENV_RUN_DIR)/pip-install-$(PYTHON_VERSION)-$(PIP_VERSION).done
 PIP_PATCH_DONE=$(VENV_RUN_DIR)/pip-patch-$(PYTHON_VERSION).done
 PAPI_INSTALL_DONE=$(VENV_RUN_DIR)/papi-install-$(PYTHON_VERSION).done
 PAPI_PYTHON_SRC_DIR=$(WS_ROOT)/src/vpp-api/python
@@ -115,27 +97,25 @@ PAPI_WIPE_DIST=$(WS_ROOT)/src/vpp-api/vapi/__pycache__ \
 	$(PAPI_PYTHON_SRC_DIR)/vpp_papi.egg-info \
 	$(PAPI_PYTHON_SRC_DIR)/vpp_papi/__pycache__
 
-PAPI_INSTALL_FLAGS=$(PIP_INSTALL_DONE) $(PIP_PATCH_DONE) $(PAPI_INSTALL_DONE)
-
 $(PIP_TOOLS_INSTALL_DONE):
 	@rm -rf $(VENV_PATH)
 	@mkdir -p $(VENV_RUN_DIR)
-	@virtualenv $(VENV_PATH) -p $(PYTHON_INTERP)
+	@$(PYTHON_INTERP) -m venv $(VENV_PATH)
 	# pip version pinning
 	@bash -c "source $(VENV_PATH)/bin/activate && \
-		  $(PYTHON_INTERP) -m pip install pip===$(PIP_VERSION)"
+		  python3 -m pip install pip===$(PIP_VERSION)"
 	@bash -c "source $(VENV_PATH)/bin/activate && \
-		  $(PYTHON_INTERP) -m pip install pip-tools===$(PIP_TOOLS_VERSION)"
+		  python3 -m pip install pip-tools===$(PIP_TOOLS_VERSION)"
 	@touch $@
 
-$(PYTHON_DEPENDS): $(PIP_TOOLS_INSTALL_DONE) requirements.txt
+$(PYTHON_DEPENDS): requirements.txt
 	@bash -c "source $(VENV_PATH)/bin/activate && \
 		  CUSTOM_COMPILE_COMMAND='make test-refresh-deps (or update requirements.txt)' \
-		  $(PYTHON_INTERP) -m piptools compile -q --generate-hashes requirements.txt --output-file $@"
+		  python3 -m piptools compile -q --generate-hashes requirements.txt --output-file $@"
 
-$(PIP_INSTALL_DONE): $(PYTHON_DEPENDS)
+$(PIP_INSTALL_DONE): $(PIP_TOOLS_INSTALL_DONE) $(PYTHON_DEPENDS)
 	@bash -c "source $(VENV_PATH)/bin/activate && \
-		  $(PYTHON_INTERP) -m piptools sync $(PYTHON_DEPENDS)"
+		  python3 -m piptools sync $(PYTHON_DEPENDS)"
 	@touch $@
 
 $(PIP_PATCH_DONE): $(PIP_INSTALL_DONE)
@@ -145,12 +125,12 @@ $(PIP_PATCH_DONE): $(PIP_INSTALL_DONE)
 		echo Applying patch: $$(basename $$f) ; \
 		patch --forward -p1 -d $(SCAPY_SOURCE) < $$f ; \
 		retCode=$$?; \
-		[ $$retCode -gt 0 ] && exit $$retCode; \
+		[ $$retCode -gt 1 ] && exit $$retCode; \
 	done; \
 	touch $@
 
 $(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE)
-	@bash -c "source $(VENV_PATH)/bin/activate && $(PYTHON_INTERP) -m pip install -e $(PAPI_PYTHON_SRC_DIR)"
+	@bash -c "source $(VENV_PATH)/bin/activate && python3 -m pip install -e $(PAPI_PYTHON_SRC_DIR)"
 	@touch $@
 
 .PHONY: refresh-deps
@@ -168,7 +148,7 @@ PLUGIN_SRC_DIR=$(INTERN_PLUGIN_SRC_DIR)
 endif
 
 define retest-func
-@env FORCE_FOREGROUND=$(FORCE_FOREGROUND) FAILED_DIR=$(FAILED_DIR) VENV_PATH=$(VENV_PATH) scripts/setsid_wrapper.sh $(FORCE_FOREGROUND) $(VENV_PATH)/bin/activate $(PYTHON_INTERP) $(PYTHON_PROFILE_OPTS) $(BUILD_TEST_SRC)/run_tests.py -d $(BUILD_TEST_SRC) $(UNITTEST_EXTRA_OPTS) || env FAILED_DIR=$(FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh
+@env VPP_IN_GDB=$(VPP_IN_GDB) FORCE_FOREGROUND=$(FORCE_FOREGROUND) FAILED_DIR=$(FAILED_DIR) VENV_PATH=$(VENV_PATH) scripts/setsid_wrapper.sh $(FORCE_FOREGROUND) $(VENV_PATH)/bin/activate $(PYTHON_INTERP) $(PYTHON_PROFILE_OPTS) run_tests.py -d $(TEST_DIR) $(UNITTEST_EXTRA_OPTS) || env FAILED_DIR=$(FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh
 endef
 
 .PHONY: sanity
@@ -177,8 +157,8 @@ ifeq ($(SANITY),no)
 SANITY_IMPORT_VPP_PAPI_CMD=true
 SANITY_RUN_VPP_CMD=true
 else
-SANITY_IMPORT_VPP_PAPI_CMD=source $(VENV_PATH)/bin/activate && $(PYTHON_INTERP) $(BUILD_TEST_SRC)/sanity_import_vpp_papi.py
-SANITY_RUN_VPP_CMD=source $(VENV_PATH)/bin/activate && PYTHONPATH=$(BUILD_TEST_SRC) $(PYTHON_INTERP) $(BUILD_TEST_SRC)/sanity_run_vpp.py
+SANITY_IMPORT_VPP_PAPI_CMD=source $(VENV_PATH)/bin/activate && python3 sanity_import_vpp_papi.py
+SANITY_RUN_VPP_CMD=source $(VENV_PATH)/bin/activate && python3 sanity_run_vpp.py
 endif
 
 ifndef TEST_JOBS
@@ -193,8 +173,7 @@ else
 PARALLEL_ILLEGAL=1
 endif
 
-sanity: test-dep verify-no-running-vpp
-	@sys_req/dev_shm_size.sh
+sanity: test-dep
 	@bash -c "test $(PARALLEL_ILLEGAL) -eq 0 ||\
 	    (echo \"*******************************************************************\" &&\
 		 echo \"* Sanity check failed, TEST_JOBS is not 1 or 'auto' and DEBUG, STEP or PROFILE is set\" &&\
@@ -214,22 +193,14 @@ sanity: test-dep verify-no-running-vpp
 	         echo \"*******************************************************************\" &&\
 		 false)"
 
-.PHONY: ext-test-apps
-ext-test-apps:
-	make -C ext test-apps
-
-$(BUILD_TEST_SRC): verify-env
-	@mkdir -p $@
-	@for file in $(VPP_TEST_SRC); do if [ ! -e $(BUILD_TEST_SRC)/$$(basename $$file) ] ; then ln -s $$file $(BUILD_TEST_SRC) ; fi ; done
-
 $(FAILED_DIR): reset
 	@mkdir -p $@
 
 .PHONY: test-dep
-test-dep: $(BUILD_TEST_SRC) $(PAPI_INSTALL_DONE) $(FAILED_DIR)
+test-dep: $(PAPI_INSTALL_DONE) $(FAILED_DIR)
 
 .PHONY: test
-test: test-dep ext-test-apps sanity
+test: test-dep sanity
 	$(call retest-func)
 
 .PHONY: retest
@@ -239,11 +210,9 @@ retest: verify-env sanity $(FAILED_DIR)
 .PHONY: shell
 shell: test-dep
 	@echo "source $(VENV_PATH)/bin/activate;\
-		cd $(BUILD_TEST_SRC);\
-		export PYTHONPATH=$(BUILD_TEST_SRC);\
 		export RND_SEED=$(RND_SEED);\
 		echo '***';\
-		echo PYTHONPATH=$(BUILD_TEST_SRC);\
+		echo PYTHONPATH=$(PYTHONPATH);\
 		echo RND_SEED=$(RND_SEED);\
 		echo VPP_BUILD_DIR=$(VPP_BUILD_DIR);\
 		echo VPP_BIN=$(VPP_BIN);\
@@ -260,32 +229,14 @@ shell: test-dep
 .PHONY: reset
 reset:
 	@rm -f /dev/shm/vpp-unittest-*
-	@rm -rf /tmp/vpp-unittest-*
+	@if [ $(FORCE_NO_WIPE) -eq "0" ] ; then rm -rf /tmp/vpp-unittest-*;  fi
 	@rm -f /tmp/api_post_mortem.*
 	@rm -rf $(FAILED_DIR)
 
 .PHONY: wipe
 wipe: reset
-	@make -C ext clean
 	@rm -rf $(VENV_PATH)
-	@rm -rf $(patsubst %,%/__pycache__, $(VPP_TEST_DIRS) $(BUILD_TEST_SRC))
-
-$(TEST_DOC_BR):
-	@mkdir -p $@
-	@bash -c "source $(VENV_PATH)/bin/activate && \
-		  $(PYTHON_INTERP) -m pip install sphinx sphinx-rtd-theme"
-	@bash -c "source $(VENV_PATH)/bin/activate && make -C doc html"
-
-.PHONY: doc
-doc: $(BUILD_TEST_SRC) $(PIP_PATCH_DONE) $(TEST_DOC_BR)
-	@echo
-	@echo "Test Documentation URL: $(TEST_DOC_BR)/html/index.html"
-	@echo "Run 'make test-wipe-doc test-doc' to rebuild the test docs"
-	@echo
-
-.PHONY: wipe-doc
-wipe-doc:
-	@rm -rf $(TEST_DOC_BR)
+	@rm -rf $(patsubst %,%/__pycache__, $(VPP_TEST_DIRS))
 
 $(BUILD_COV_DIR):
 	@mkdir -p $@
@@ -312,21 +263,42 @@ wipe-papi:
 	@rm -rf $(PAPI_INSTALL_DONE) $(PAPI_WIPE_DIST)
 
 .PHONY: wipe-all
-wipe-all: wipe wipe-papi wipe-doc wipe-cov
+wipe-all: wipe wipe-papi wipe-cov
 	@rm -rf $(TEST_BR)
 
+.PHONY: checkstyle-diff
+checkstyle-diff: $(PIP_INSTALL_DONE)
+	@bash -c "source $(VENV_PATH)/bin/activate &&\
+		  python3 -m pip install pycodestyle"
+	@bash -c "source $(VENV_PATH)/bin/activate &&\
+		cd $(WS_ROOT) && git diff --name-only --no-color --relative HEAD~1 ':!*.patch' | grep '.py$$' | xargs -n 1 \
+		pycodestyle --show-source --ignore=W504,E126,E241,E226,E305,E704,E741,E722 -v ||\
+		(echo \"*********************************************************************\" &&\
+		 echo \"* Test framework PEP8 compliance check FAILED (checked changed files)\" &&\
+	         echo \"*********************************************************************\" &&\
+		 false)"
+	@echo "*********************************************************************"
+	@echo "* Test framework PEP8 compliance check passed (checked changed files)"
+	@echo "*********************************************************************"
+
+.PHONY: start-gdb
+start-gdb: sanity
+	$(eval VPP_IN_GDB=1)
+	$(eval FORCE_FOREGROUND=1)
+	$(call retest-func)
+
 .PHONY: checkstyle
-checkstyle: $(BUILD_TEST_SRC) $(PIP_INSTALL_DONE)
+checkstyle: $(PIP_INSTALL_DONE)
 	@bash -c "source $(VENV_PATH)/bin/activate &&\
-		  $(PYTHON_INTERP) -m pip install pycodestyle"
+		  python3 -m pip install pycodestyle"
 	@bash -c "source $(VENV_PATH)/bin/activate &&\
-		pycodestyle --show-source --ignore=W504,E126,E241,E226,E305,E704,E741,E722 -v $(BUILD_TEST_SRC)/*.py ||\
+		pycodestyle --show-source --ignore=W504,E126,E241,E226,E305,E704,E741,E722 -v *.py ||\
 		(echo \"*******************************************************************\" &&\
-		 echo \"* Test framework PEP8 compliance check FAILED \" &&\
+		 echo \"* Test framework PEP8 compliance check FAILED (checked all files)\" &&\
 	         echo \"*******************************************************************\" &&\
 		 false)"
 	@echo "*******************************************************************"
-	@echo "* Test framework PEP8 compliance check passed"
+	@echo "* Test framework PEP8 compliance check passed (checked all files)"
 	@echo "*******************************************************************"
 
 .PHONY: help
@@ -343,21 +315,20 @@ help:
 	@echo " retest-all-debug       - run functional and extended tests (debug build)"
 	@echo " test-cov               - generate code coverage report for test framework"
 	@echo " test-gcov                      - build and run functional tests (gcov build)"
-
 	@echo " test-wipe              - wipe (temporary) files generated by unit tests"
 	@echo " test-wipe-cov          - wipe code coverage report for test framework"
-	@echo " test-wipe-doc          - wipe documentation for test framework"
 	@echo " test-wipe-papi         - rebuild vpp_papi sources"
-	@echo " test-wipe-all          - wipe (temporary) files generated by unit tests, docs, and coverage"
+	@echo " test-wipe-all          - wipe (temporary) files generated by unit tests, and coverage"
 	@echo " test-shell             - enter shell with test environment"
 	@echo " test-shell-debug       - enter shell with test environment (debug build)"
-	@echo " test-checkstyle      - check PEP8 compliance for test framework"
-	@echo " test-refresh-deps    - refresh the Python dependencies for the tests"
+	@echo " test-checkstyle        - check PEP8 compliance for test framework"
+	@echo " test-refresh-deps      - refresh the Python dependencies for the tests"
 	@echo ""
 	@echo "Arguments controlling test runs:"
 	@echo " V=[0|1|2]              - set test verbosity level"
 	@echo "                          0=ERROR, 1=INFO, 2=DEBUG"
-	@echo " TEST_JOBS=[<n>|auto]   - use <n> parallel processes for test execution or automatic discovery of maximum acceptable processes (default: 1)"
+	@echo " TEST_JOBS=[<n>|auto]   - use at most <n> parallel python processes for test execution, if auto, set to number of available cpus (default: 1)"
+	@echo " MAX_VPP_CPUS=[<n>|auto]- use at most <n> cpus for running vpp main and worker threads, if auto, set to number of available cpus (default: auto)"
 	@echo " CACHE_OUTPUT=[0|1]     - cache VPP stdout/stderr and log as one block after test finishes (default: 1)"
 	@echo " FAILFAST=[0|1]         - fail fast if 1, complete all tests if 0"
 	@echo " TIMEOUT=<timeout>      - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)"
@@ -369,6 +340,8 @@ help:
 	@echo "                          and tearing down a testcase"
 	@echo "    DEBUG=gdbserver     - run gdb inside a gdb server, otherwise"
 	@echo "                          same as above"
+	@echo "    DEBUG=attach        - attach test case to already running vpp in gdb (see test-start-vpp-in-gdb)"
+	@echo ""
 	@echo " STEP=[yes|no]          - ease debugging by stepping through a testcase"
 	@echo " SANITY=[yes|no]        - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)"
 	@echo " EXTENDED_TESTS=[1|y]   - used by '[re]test-all' & '[re]test-all-debug' to run extended tests"
@@ -382,7 +355,10 @@ help:
 	@echo "                               TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase"
 	@echo "                               TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes"
 	@echo ""
-	@echo " VPP_ZOMBIE_NOCHECK=1   - skip checking for vpp (zombie) processes (CAUTION)"
+	@echo " VARIANT=<variant>      - specify which march node variant to unit test"
+	@echo "                          e.g. VARIANT=skx test the skx march variants"
+	@echo "                          e.g. VARIANT=icl test the icl march variants"
+	@echo ""
 	@echo " COREDUMP_SIZE=<size>   - pass <size> as unix { coredump-size <size> } argument to vpp"
 	@echo "                          e.g. COREDUMP_SIZE=4g"
 	@echo "                               COREDUMP_SIZE=unlimited"
@@ -399,13 +375,18 @@ help:
 	@echo ""
 	@echo " SKIP_AARCH64=1         - skip tests that are failing on the ARM platorm in FD.io CI"
 	@echo ""
-	@echo " SOCKET=1               - Communicate with VPP over Unix domain socket instead of SHM"
-	@echo ""
 	@echo " RND_SEED=seed          - Seed RND with given seed"
 	@echo ""
-	@echo "Creating test documentation"
-	@echo " test-doc               - generate documentation for test framework"
-	@echo " test-wipe-doc          - wipe documentation for test framework"
+	@echo "Starting VPP in GDB for use with DEBUG=attach:"
+	@echo ""
+	@echo " test-start-vpp-in-gdb       - start VPP in gdb (release)"
+	@echo " test-start-vpp-debug-in-gdb - start VPP in gdb (debug)"
+	@echo ""
+	@echo "Arguments controlling VPP in GDB runs:"
+	@echo " "
+	@echo " VPP_IN_GDB_TMP_DIR     - specify directory to run VPP IN (default: /tmp/unittest-attach-gdb)"
+	@echo " VPP_IN_GDB_NO_RMDIR=0  - don't remove existing tmp dir but fail instead"
+	@echo " VPP_IN_GDB_CMDLINE=1   - add 'interactive' to VPP arguments to run with command line"
 	@echo ""
 	@echo "Creating test code coverage report"
 	@echo " test-cov               - generate code coverage report for test framework"