X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=test%2FMakefile;h=af503dcf59f79ab44cf533bc74141ca9f0e6f596;hb=853cc9f2ad3ee52cbdd891fb09d51c25678baed0;hp=44187408440584d6396d3185a7bbd97c0d1d843c;hpb=08c50e3b7acb932adb1f62002619a1e52207262a;p=vpp.git

diff --git a/test/Makefile b/test/Makefile
index 44187408440..af503dcf59f 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -74,13 +74,13 @@ V=0
 endif
 
 PYTHON_VERSION=$(shell $(PYTHON_INTERP) -c 'import sys; print(sys.version_info.major)')
-PIP_VERSION=22.0.4
+PIP_VERSION=23.3.1
 # Keep in sync with requirements.txt
-PIP_TOOLS_VERSION=6.6.0
-PIP_SETUPTOOLS_VERSION=62.1.0
+PIP_TOOLS_VERSION=7.3.0
+PIP_SETUPTOOLS_VERSION=68.1.0
 PYTHON_DEPENDS=requirements-$(PYTHON_VERSION).txt
 SCAPY_SOURCE=$(shell find $(VENV_PATH)/lib/python* -name site-packages)
-BUILD_COV_DIR=$(TEST_BR)/coverage
+BUILD_COV_DIR=$(BR)/test-coverage
 
 PIP_TOOLS_INSTALL_DONE=$(VENV_RUN_DIR)/pip-tools-install-$(PYTHON_VERSION)-$(PIP_TOOLS_VERSION).done
 PIP_INSTALL_DONE=$(VENV_RUN_DIR)/pip-install-$(PYTHON_VERSION)-$(PIP_VERSION).done
@@ -107,7 +107,7 @@ $(PIP_TOOLS_INSTALL_DONE):
 
 $(PYTHON_DEPENDS): requirements.txt
 	@bash -c "source $(VENV_PATH)/bin/activate && \
-		  CUSTOM_COMPILE_COMMAND='make test-refresh-deps (or update requirements.txt)' \
+		  CUSTOM_COMPILE_COMMAND='$(MAKE) test-refresh-deps (or update requirements.txt)' \
 		  python3 -m piptools compile -q --generate-hashes requirements.txt --output-file $@"
 
 $(PIP_INSTALL_DONE): $(PIP_TOOLS_INSTALL_DONE) $(PYTHON_DEPENDS)
@@ -131,7 +131,7 @@ $(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE)
 	@touch $@
 
 .PHONY: refresh-deps
-refresh-deps: clean-deps $(PYTHON_DEPENDS)
+refresh-deps: clean-deps $(PIP_INSTALL_DONE) $(PYTHON_DEPENDS)
 
 .PHONY: clean-deps
 clean-deps:
@@ -200,7 +200,7 @@ ARG6=--step
 endif
 
 ARG7=
-ifneq ($(findstring $(TESTS_GCOV),1 y yes),)
+ifneq ($(findstring $(TEST_GCOV),1 y yes),)
 ARG7=--gcov
 endif
 
@@ -254,9 +254,28 @@ ifneq ($(EXTERN_APIDIR),)
 ARG17=--extern-apidir=$(EXTERN_APIDIR)
 endif
 
-EXTRA_ARGS=$(ARG0) $(ARG1) $(ARG2) $(ARG3) $(ARG4) $(ARG5) $(ARG6) $(ARG7) $(ARG8) $(ARG9) $(ARG10) $(ARG11) $(ARG12) $(ARG13) $(ARG14) $(ARG15) $(ARG16) $(ARG17)
+ARG18=
+ifneq ($(findstring $(DECODE_PCAPS),1 y yes),)
+ARG18=--decode-pcaps
+endif
+
+ifneq ($(findstring $(API_PRELOAD),1 y yes),)
+ARG19=--api-preload
+else
+ARG19=
+endif
+
+EXC_PLUGINS_ARG=
+ifneq ($(VPP_EXCLUDED_PLUGINS),)
+# convert the comma-separated list into N invocations of the argument to exclude a plugin
+EXC_PLUGINS_ARG=$(shell echo "${VPP_EXCLUDED_PLUGINS}" | sed 's/\([^,]*\)/--excluded-plugin=\1/g; s/,/ /g')
+endif
+
+
+
+EXTRA_ARGS=$(ARG0) $(ARG1) $(ARG2) $(ARG3) $(ARG4) $(ARG5) $(ARG6) $(ARG7) $(ARG8) $(ARG9) $(ARG10) $(ARG11) $(ARG12) $(ARG13) $(ARG14) $(ARG15) $(ARG16) $(ARG17) $(ARG18) $(ARG19)
 
-RUN_TESTS_ARGS=--failed-dir=$(FAILED_DIR) --verbose=$(V) --jobs=$(TEST_JOBS) --filter=$(TEST) --retries=$(RETRIES) --venv-dir=$(VENV_PATH) --vpp-ws-dir=$(WS_ROOT) --vpp-tag=$(TAG) --rnd-seed=$(RND_SEED) --vpp-worker-count="$(VPP_WORKER_COUNT)" --keep-pcaps $(PLUGIN_PATH_ARGS) $(TEST_PLUGIN_PATH_ARGS) $(EXTRA_ARGS)
+RUN_TESTS_ARGS=--failed-dir=$(FAILED_DIR) --verbose=$(V) --jobs=$(TEST_JOBS) --filter=$(TEST) --retries=$(RETRIES) --venv-dir=$(VENV_PATH) --vpp-ws-dir=$(WS_ROOT) --vpp-tag=$(TAG) --rnd-seed=$(RND_SEED) --vpp-worker-count="$(VPP_WORKER_COUNT)" --keep-pcaps $(PLUGIN_PATH_ARGS) $(EXC_PLUGINS_ARG) $(TEST_PLUGIN_PATH_ARGS) $(EXTRA_ARGS)
 RUN_SCRIPT_ARGS=--python-opts=$(PYTHON_OPTS)
 
 define retest-func
@@ -326,19 +345,42 @@ wipe: reset
 $(BUILD_COV_DIR):
 	@mkdir -p $@
 
-.PHONY: cov
-cov: wipe-cov test-dep ext $(BUILD_COV_DIR)
+.PHONY: cov-prep
+cov-prep: test-dep
 	@lcov --zerocounters --directory $(VPP_BUILD_DIR)
 	@test -z "$(EXTERN_COV_DIR)" || lcov --zerocounters --directory $(EXTERN_COV_DIR)
-	$(call retest-func)
-	@lcov --capture --directory $(VPP_BUILD_DIR) --output-file $(BUILD_COV_DIR)/coverage.info
-	@test -z "$(EXTERN_COV_DIR)" || lcov --capture --directory $(EXTERN_COV_DIR) --output-file $(BUILD_COV_DIR)/extern-coverage.info
-	@genhtml $(BUILD_COV_DIR)/coverage.info --output-directory $(BUILD_COV_DIR)/html
-	@test -z "$(EXTERN_COV_DIR)" || genhtml $(BUILD_COV_DIR)/extern-coverage.info --output-directory $(BUILD_COV_DIR)/extern-html
+
+.PHONY: cov-post
+cov-post: wipe-cov $(BUILD_COV_DIR)
+	@lcov --capture \
+		--directory $(VPP_BUILD_DIR) \
+		--output-file $(BUILD_COV_DIR)/coverage.info
+	@test -z "$(EXTERN_COV_DIR)" || \
+		lcov --capture \
+		--directory $(EXTERN_COV_DIR) \
+		--output-file $(BUILD_COV_DIR)/extern-coverage.info
+	@lcov --remove $(BUILD_COV_DIR)/coverage.info \
+		"/usr/include/*" "*/build-root/*" "/opt/*" "/usr/lib/*" \
+		"*_test.*" "*vat*" "*rdma*" "*/vpp-api/client/*" "*/plugins/af_packet/*" \
+		"*/plugins/af_xdp/*" "*/plugins/avf/*" "*/plugins/dma_intel/*" \
+		"*/plugins/hs_apps/*" "*/plugins/vmxnet3/*" "*/vnet/devices/virtio/*" \
+		"*/plugins/perfmon/arm*" "*/plugins/perfmon/intel/*" "*/vlib/vmbus/*" \
+		"*/vnet/dev/*" "*/plugins/dev_ena/*" "*/plugins/builtinurl/*" "*/vnet/flow/*" \
+		"*/plugins/http_static/builtinurl/*" "*/plugins/dev_iavf/*" \
+		-o $(BUILD_COV_DIR)/coverage-filtered.info
+	@genhtml $(BUILD_COV_DIR)/coverage-filtered.info \
+		--output-directory $(BUILD_COV_DIR)/html
+	@test -z "$(EXTERN_COV_DIR)" || \
+		genhtml $(BUILD_COV_DIR)/extern-coverage.info \
+			--output-directory $(BUILD_COV_DIR)/extern-html
 	@echo
 	@echo "Build finished. Code coverage report is in $(BUILD_COV_DIR)/html/index.html"
 	@test -z "$(EXTERN_COV_DIR)" || echo "Code coverage report for out-of-tree objects is in $(BUILD_COV_DIR)/extern-html/index.html"
 
+.PHONY: cov
+cov:
+	$(MAKE) -C . cov-prep test cov-post
+
 .PHONY: wipe-cov
 wipe-cov: wipe
 	@rm -rf $(BUILD_COV_DIR)
@@ -388,14 +430,17 @@ help:
 	@echo ""
 	@echo " test                   - build and run (basic) functional tests"
 	@echo " test-debug             - build and run (basic) functional tests (debug build)"
+	@echo " test-cov               - generate code coverage report for functional tests"
+	@echo " test-cov-prep          - coverage phase #1 : prepare lcov"
+	@echo " test-cov-build         - coverage phase #2 : build gcov image & run tests against it (use TEST=)"
+	@echo " test-cov-post          - coverage phase #3 : generate lcov html report"
 	@echo " test-all               - build and run functional and extended tests"
 	@echo " test-all-debug         - build and run functional and extended tests (debug build)"
+	@echo " test-all-cov           - generate code coverage report for functional and extended tests"
 	@echo " retest                 - run functional tests"
 	@echo " retest-debug           - run functional tests (debug build)"
 	@echo " retest-all             - run functional and extended tests"
 	@echo " retest-all-debug       - run functional and extended tests (debug build)"
-	@echo " test-cov               - generate code coverage report for test framework"
-	@echo " test-gcov                      - build and run functional tests (gcov build)"
 	@echo " test-wipe              - wipe (temporary) files generated by unit tests"
 	@echo " test-wipe-cov          - wipe code coverage report for test framework"
 	@echo " test-wipe-papi         - rebuild vpp_papi sources"
@@ -556,6 +601,10 @@ help:
 	@echo "       enable debugging of the test framework itself (expert)"
 	@echo "       (default: no)"
 	@echo ""
+	@echo "   TEST_GCOV=[1|y|yes]"
+	@echo "       enable tests specifically designed soley for code coverage"
+	@echo "       (default: no)"
+	@echo ""
 	@echo "   API_FUZZ=[1|y|yes]"
 	@echo "       enable VPP api fuzz testing"
 	@echo "       (default: no)"
@@ -569,7 +618,3 @@ help:
 	@echo " test-start-vpp-in-gdb       - start VPP in gdb (release)"
 	@echo " test-start-vpp-debug-in-gdb - start VPP in gdb (debug)"
 	@echo ""
-	@echo "Creating test code coverage report:"
-	@echo ""
-	@echo " test-cov               - generate code coverage report for test framework"
-	@echo " test-wipe-cov          - wipe code coverage report for test framework"