tox: Add job-spec checker

Change-Id: I2bbdc28431b6271d1950ec0e74bf708cae841146
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/libraries/bash/entry/check/job_spec.sh b/resources/libraries/bash/entry/check/job_spec.sh
new file mode 100644 (file)
index 0000000..21013fe
--- /dev/null
+++ b/resources/libraries/bash/entry/check/job_spec.sh
@@ -0,0 +1,60 @@
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+# This file should be executed from tox, as the assumed working directory
+# is different from where this file is located.
+# This file does not have executable flag nor shebang,
+# to dissuade non-tox callers.
+
+# This script checks if the number of tests in a job specification is the same
+# as declared at the beginning of the file.
+# It counts the lines not starting with '#' so it can also detect redundant
+# empty lines, or lines which should not be in the job specification.
+
+# "set -eu" handles failures from the following two lines.
+BASH_CHECKS_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_CHECKS_DIR}/../../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+
+job_spec_dir="docs/job_specs/"
+rm -f "job_spec.log" || die
+violations=0
+
+# Disabling -x: Following lines are doing too much garbage output.
+set +x
+
+for f in $(find ${job_spec_dir} -type f | grep -v perf_tests_job_specs); do
+    declared=$(fgrep "### tests" $f | tr -dc '0-9')
+    present=$(fgrep -v '#' $f | wc -l)
+    if [ "${declared}" != "${present}" ]; then
+        echo "Wrong number of tests detected in ${f}: \
+declared: ${declared} / present: ${present}" | tee -a job_spec.log
+        violations=$((violations+1))
+    fi
+done
+
+set -x
+
+if [ "${violations}" != "0" ]; then
+    warn
+    warn "Number of tests in job spec checker: FAIL"
+    exit 1
+fi
+
+warn
+warn "Number of tests in job spec checker: PASS"

diff --git a/tox.ini b/tox.ini
index b7573f9..ab6a4b0 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -26,8 +26,8 @@
 
 [tox]
 # Fast and brief checkers to front, slow or verbose checkers to back.
-envlist = tc_naming, tc_coverage, copyright_year, gpl_license, new_line_length,
-    line_length, autogen, pylint, doc_verify
+envlist = job_spec, tc_naming, tc_coverage, copyright_year, gpl_license,
+    new_line_length, line_length, autogen, pylint, doc_verify
 
 # The following is needed as tox requires setup.py by default.
 skipsdist = true
@@ -65,6 +65,10 @@ commands = bash {[tox]checker_dir}/doc_verify.sh
 whitelist_externals = /bin/bash
 commands = bash {[tox]checker_dir}/gpl_license.sh
 
+[testenv:job_spec]
+whitelist_externals = /bin/bash
+commands = bash {[tox]checker_dir}/job_spec.sh
+
 [testenv:line_length]
 # Fix all transgressions and remove the ignore_outcome flag.
 ignore_outcome = true