From: pmikus <pmikus@cisco.com>
Date: Fri, 20 Aug 2021 09:27:12 +0000 (+0000)
Subject: FIX: Docs generation
X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=commitdiff_plain;h=aa72d4256fb7409fa01a0cb33aaa763128f0d30b

FIX: Docs generation

+ Tests will be fixed with renamed hoststack tests.
+ Tox is at least working

Signed-off-by: pmikus <pmikus@cisco.com>
Change-Id: Ibee30cd54c78b67d2ef907cdd14a71ae197be59e
---

diff --git a/resources/libraries/bash/entry/check/doc_verify.sh b/resources/libraries/bash/entry/check/doc_verify.sh
index 544586a767..baa9d8aa74 100644
--- a/resources/libraries/bash/entry/check/doc_verify.sh
+++ b/resources/libraries/bash/entry/check/doc_verify.sh
@@ -27,14 +27,9 @@ source "${BASH_FUNCTION_DIR}/common.sh" || {
     echo "Source failed." >&2
     exit 1
 }
-
+source "${BASH_FUNCTION_DIR}/docs.sh" || die "Source failed."
 common_dirs || die
-log_file="$(pwd)/doc_verify.log" || die
-
-# Pre-cleanup.
-rm -f "${log_file}" || die
-rm -f "${DOC_GEN_DIR}/csit.docs.tar.gz" || die
-rm -rf "${DOC_GEN_DIR}/_build" || die
+activate_virtualenv || die
 
 # Documentation generation.
 # Here we do store only stderr to file while stdout (inlcuding Xtrace) is
@@ -43,13 +38,12 @@ rm -rf "${DOC_GEN_DIR}/_build" || die
 # task.
 exec 3>&1 || die
 export BASH_XTRACEFD="3" || die
+log_file="$(pwd)/doc_verify.log" || die
 
-pushd "${DOC_GEN_DIR}" || die
-source ./run_doc.sh ${GERRIT_BRANCH:-local} 2> ${log_file} || true
-popd || die
+generate_docs 2> ${log_file} || die
 
-if [[ ! -f "${log_file}" ]] || [[ -s "${log_file}" ]]; then
-    # Output file not exists or is non empty.
+if [[ "${DOCS_EXIT_STATUS}" != 0 ]]; then
+    # Failed to generate report.
     warn
     warn "Doc verify checker: FAIL"
     exit 1
diff --git a/resources/libraries/bash/function/docs.sh b/resources/libraries/bash/function/docs.sh
index ec5cbefdd0..be3f9861ce 100644
--- a/resources/libraries/bash/function/docs.sh
+++ b/resources/libraries/bash/function/docs.sh
@@ -50,53 +50,68 @@ function generate_docs () {
     WORKING_DIR="tmp"
     BUILD_DIR="_build"
 
+    # Remove the old build:
+    rm -rf ${BUILD_DIR} || true
+    rm -rf ${WORKING_DIR} || true
+
     # Create working directories
-    mkdir "${BUILD_DIR}"
-    mkdir --parents "${WORKING_DIR}"/resources/libraries/python/
-    mkdir --parents "${WORKING_DIR}"/resources/libraries/robot/
-    mkdir --parents "${WORKING_DIR}"/tests/
+    mkdir -p "${BUILD_DIR}" || die "Mkdir failed!"
+    mkdir -p "${WORKING_DIR}"/resources/libraries/python/ || die "Mkdir failed!"
+    mkdir -p "${WORKING_DIR}"/resources/libraries/robot/ || die "Mkdir failed!"
+    mkdir -p "${WORKING_DIR}"/tests/ || die "Mkdir failed!"
 
     # Copy the Sphinx source files:
-    cp -r src/* ${WORKING_DIR}/
+    cp -r src/* ${WORKING_DIR}/ || die "Copy the Sphinx source files failed!"
 
     # Copy the source files to be processed:
-    from_dir="../../../resources/libraries/python/"
+    from_dir="${RESOURCES_DIR}/libraries/python/"
     to_dir="${WORKING_DIR}/resources/libraries/python/"
-    command="rsync -a --include '*/'"
-    ${command} --include '*.py' --exclude '*' "${from_dir}" "${to_dir}"
-    cp ../../../resources/__init__.py ${WORKING_DIR}/resources/
-    cp ../../../resources/libraries/__init__.py ${WORKING_DIR}/resources/libraries/
-    from_dir="../../../resources/libraries/robot/"
-    to_dir="${WORKING_DIR}/resources/libraries/robot/"
-    ${command} --include '*.robot' --exclude '*' "${from_dir}" "${to_dir}"
-    from_dir="../../../tests/"
-    to_dir="${WORKING_DIR}/tests/"
-    ${command} --include '*.robot' --exclude '*' "${from_dir}" "${to_dir}"
+    dirs="${from_dir} ${to_dir}"
+    rsync -ar --include='*/' --include='*.py' --exclude='*' ${dirs} || {
+        die "rSync failed!"
+    }
 
-    python3 gen_rst.py
-    # Remove all rst files from ./${WORKING_DIR}/env directory - we do not need
-    # them
-    find ./${WORKING_DIR}/env -type f -name '*.rst' | xargs rm -f
+    from_dir="${RESOURCES_DIR}/libraries/robot/"
+    to_dir="${WORKING_DIR}/resources/libraries/robot/"
+    dirs="${from_dir} ${to_dir}"
+    rsync -ar --include='*/' --include '*.robot' --exclude '*' ${dirs} || {
+        die "rSync failed!"
+    }
+    touch ${to_dir}/index.robot || {
+        die "Touch index.robot file failed!"
+    }
+
+    # Due to hoststack tests having dots in the name of suite, tests will become
+    # disabled as spihnxdoc cannot properly work with the path. gen_rst
+    # is generating dots scheme. The solution is to rename suites as
+    # having dots is misleading with robot framework naming conventions.
+
+    #from_dir="${CSIT_DIR}/tests/"
+    #to_dir="${WORKING_DIR}/tests/"
+    #dirs="${from_dir} ${to_dir}"
+    #rsync -ar --include='*/' --include '*.robot' --exclude '*' ${dirs} || {
+    #    die "rSync failed!"
+    #}
+
+    find ${WORKING_DIR}/ -type d -exec echo {} \; -exec touch {}/__init__.py \;
+
+    python3 gen_rst.py || die "Generate .rst files failed!"
 
     # Generate the documentation:
-    DATE=$(date -u '+%d-%b-%Y')
+    DATE=$(date -u '+%d-%b-%Y') || die "Get date failed!"
 
     all_options=("-v")
     all_options+=("-c" "${WORKING_DIR}")
     all_options+=("-a")
     all_options+=("-b" "html")
     all_options+=("-E")
-    all_options+=("-D" "release=$1")
-    all_options+=("-D" "version='$1 documentation - $DATE'")
+    all_options+=("-D" "version="${GERRIT_BRANCH:-master}"")
     all_options+=("${WORKING_DIR}" "${BUILD_DIR}/")
 
     set +e
     sphinx-build "${all_options[@]}"
     DOCS_EXIT_STATUS="$?"
     set -e
-
-    find . -type d -name 'env' | xargs rm -rf
-
 }
 
 function generate_report () {
diff --git a/resources/tools/doc_gen/run_doc.sh b/resources/tools/doc_gen/run_doc.sh
deleted file mode 100755
index 10cc3e249d..0000000000
--- a/resources/tools/doc_gen/run_doc.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/bin/bash
-
-WORKING_DIR='tmp'
-BUILD_DIR='_build'
-
-# Clean-up when finished:
-trap 'rm -rf ${WORKING_DIR}; exit' EXIT
-trap 'rm -rf ${WORKING_DIR}; exit' ERR
-
-# Remove the old build:
-rm -rf ${BUILD_DIR} || true
-rm -rf ${WORKING_DIR} || true
-
-# Create working directories
-mkdir ${BUILD_DIR}
-mkdir --parents ${WORKING_DIR}/resources/libraries/python/
-mkdir --parents ${WORKING_DIR}/resources/libraries/robot/
-mkdir --parents ${WORKING_DIR}/tests/
-
-# Copy the Sphinx source files:
-cp -r src/* ${WORKING_DIR}/
-
-# Copy the source files to be processed:
-from_dir="../../../resources/libraries/python/"
-to_dir="${WORKING_DIR}/resources/libraries/python/"
-command="rsync -a --include '*/'"
-${command} --include '*.py' --exclude '*' "${from_dir}" "${to_dir}"
-cp ../../../resources/__init__.py ${WORKING_DIR}/resources/
-cp ../../../resources/libraries/__init__.py ${WORKING_DIR}/resources/libraries/
-from_dir="../../../resources/libraries/robot/"
-to_dir="${WORKING_DIR}/resources/libraries/robot/"
-${command} --include '*.robot' --exclude '*' "${from_dir}" "${to_dir}"
-from_dir="../../../tests/"
-to_dir="${WORKING_DIR}/tests/"
-${command} --include '*.robot' --exclude '*' "${from_dir}" "${to_dir}"
-
-# Create virtual environment:
-virtualenv --python=$(which python3) ${WORKING_DIR}/env
-. ${WORKING_DIR}/env/bin/activate
-
-# Install CSIT requirements:
-pip3 install --upgrade -r ../../../requirements.txt
-
-export PYTHONPATH=`pwd`
-
-# Generate rst files:
-python3 gen_rst.py
-
-# Remove all rst files from ./${WORKING_DIR}/env directory - we do not need them
-find ./${WORKING_DIR}/env -type f -name '*.rst' | xargs rm -f
-
-# Generate the documentation:
-DATE=$(date -u '+%d-%b-%Y')
-command="sphinx-build -v -c '${WORKING_DIR}' -a  -b html -E -D release='$1' -D"
-command+=" version='$1 documentation - $DATE' '${WORKING_DIR}' '${BUILD_DIR}/'"
-${command}
-
-find . -type d -name 'env' | xargs rm -rf
-
-echo Creating csit.doc.tar.gz ...
-tar -czvf ./csit.docs.tar.gz ${BUILD_DIR}
diff --git a/resources/tools/doc_gen/src/conf.py b/resources/tools/doc_gen/src/conf.py
index 9be0baea53..02707665ad 100644
--- a/resources/tools/doc_gen/src/conf.py
+++ b/resources/tools/doc_gen/src/conf.py
@@ -42,20 +42,15 @@ templates_path = [u"_templates"]
 # The suffix(es) of source file names.
 # You can specify multiple suffix as a list of string:
 #
-# source_suffix = [u".rst", u".md"]
-source_suffix = u".rst"
-
-# The encoding of source files.
-#
-# source_encoding = 'utf-8-sig'
+source_suffix = [u".rst", u".md"]
 
 # The master toctree document.
 master_doc = u"index"
 
 # General information about the project.
-project = u"CSIT"
-copyright = u"2018, FD.io"
-author = u"CSIT"
+project = u"FD.io CSIT"
+copyright = u"2021, FD.io"
+author = u'FD.io CSIT'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -71,7 +66,7 @@ author = u"CSIT"
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = u'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -109,12 +104,6 @@ exclude_patterns = [u"_build", u"Thumbs.db", u".DS_Store"]
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = u"sphinx"
 
-# A list of ignored prefixes for module index sorting.
-# modindex_common_prefix = []
-
-# If true, keep warnings as "system message" paragraphs in the built documents.
-# keep_warnings = False
-
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
@@ -131,10 +120,23 @@ html_theme = u"sphinx_rtd_theme"
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
-# html_theme_options = {}
+html_theme_options = {
+    u'canonical_url': u'',
+    u'analytics_id': u'',
+    u'logo_only': False,
+    u'display_version': True,
+    u'prev_next_buttons_location': u'bottom',
+    u'style_external_links': False,
+    # Toc options
+    u'collapse_navigation': True,
+    u'sticky_navigation': True,
+    u'navigation_depth': 3,
+    u'includehidden': True,
+    u'titles_only': False
+}
 
 # Add any paths that contain custom themes here, relative to this directory.
-html_theme_path = [u"env/lib/python2.7/site-packages/sphinx_rtd_theme"]
+html_theme_path = [u'env/lib/python3.8/site-packages/sphinx_rtd_theme']
 
 # The name for this set of Sphinx documents.
 # "<project> v<release> documentation" by default.
@@ -145,11 +147,6 @@ html_title = u"CSIT Documentation"
 #
 html_short_title = u"CSIT"
 
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-#
-html_logo = u"fdio_logo.png"
-
 # The name of an image file (relative to this directory) to use as a favicon of
 # the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
@@ -190,56 +187,17 @@ html_static_path = [u"_static"]
 # html_additional_pages = {}
 
 # If false, no module index is generated.
-#
-# html_domain_indices = True
+html_domain_indices = True
 
 # If false, no index is generated.
-#
-# html_use_index = True
+html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#
 html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#
 html_show_sourcelink = True
 
-# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#
-# html_show_sphinx = True
-
-# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#
-# html_show_copyright = True
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a <link> tag referring to it.  The value of this option must be the
-# base URL from which the finished HTML is served.
-#
-# html_use_opensearch = ''
-
-# This is the file name suffix for HTML files (e.g. ".xhtml").
-# html_file_suffix = None
-
-# Language to be used for generating the HTML full-text search index.
-# Sphinx supports the following languages:
-#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
-#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh'
-#
-# html_search_language = 'en'
-
-# A dictionary with options for the search language support, empty by default.
-# 'ja' uses this config value.
-# 'zh' user can custom change `jieba` dictionary path.
-#
-# html_search_options = {'type': 'default'}
-
-# The name of a javascript file (relative to the configuration directory) that
-# implements a search results scorer. If empty, the default will be used.
-#
-# html_search_scorer = 'scorer.js'
-
 # Output file base name for HTML help builder.
 htmlhelp_basename = u"csitdoc"
 
diff --git a/resources/tools/doc_gen/src/fdio_logo.png b/resources/tools/doc_gen/src/fdio_logo.png
deleted file mode 100644
index 3c362b2786..0000000000
Binary files a/resources/tools/doc_gen/src/fdio_logo.png and /dev/null differ
diff --git a/resources/tools/doc_gen/src/tests.vpp.perf.ip4_tunnels.rst b/resources/tools/doc_gen/src/tests.vpp.perf.ip4_tunnels.rst
deleted file mode 100644
index 1c296e308f..0000000000
--- a/resources/tools/doc_gen/src/tests.vpp.perf.ip4_tunnels.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-IPv4 Tunnels
-============
-
diff --git a/resources/tools/doc_gen/src/tests.vpp.perf.ip6_tunnels.rst b/resources/tools/doc_gen/src/tests.vpp.perf.ip6_tunnels.rst
deleted file mode 100644
index 40dbbc569b..0000000000
--- a/resources/tools/doc_gen/src/tests.vpp.perf.ip6_tunnels.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-IPv6 Tunnels
-============
-
diff --git a/resources/tools/doc_gen/src/tests.vpp.perf.vm_vhost.rst b/resources/tools/doc_gen/src/tests.vpp.perf.vm_vhost.rst
deleted file mode 100644
index c669b232ea..0000000000
--- a/resources/tools/doc_gen/src/tests.vpp.perf.vm_vhost.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-VM VHOST
-========
-