1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13 from os import walk, listdir, scandir, environ
14 from os.path import isfile, isdir, join, getsize
16 # Temporary working directory. It is created and deleted by docs.sh
17 WORKING_DIR = environ.get("WORKING_DIR")
19 # Directory with resources to be documented.
20 RESOURCES_DIR = u"resources"
22 # Directory with libraries (python, robot) to be documented.
23 LIB_DIR = u"libraries"
25 # Directory with tests (func, perf) to be documented.
31 PATH_PY_LIBS = join(WORKING_DIR, RESOURCES_DIR, LIB_DIR, u"python")
32 PATH_RF_LIBS = join(WORKING_DIR, RESOURCES_DIR, LIB_DIR, u"robot")
33 PATH_TESTS = join(WORKING_DIR, TESTS_DIR)
35 # Sections in rst files
47 rst_rf_suite_setup = u"""
52 rst_rf_variables = u"""
57 rst_rf_keywords = u"""
68 def get_files(path, extension):
69 """Generates the list of files to process.
71 :param path: Path to files.
72 :param extension: Extension of files to process. If it is the empty string,
73 all files will be processed.
76 :returns: List of files to process.
81 for root, dirs, files in walk(path):
82 for filename in files:
84 if filename.endswith(extension) and u"__init__" not in filename:
85 file_list.append(join(root, filename))
87 file_list.append(join(root, filename))
92 def create_file_name(path, start):
93 """Create the name of rst file.
98 :param path: Path to a module to be documented.
99 :param start: The first directory in path which is used in the file name.
105 dir_list = path.split(u"/")
106 start_index = dir_list.index(start)
107 return u".".join(dir_list[start_index:-1]) + u".rst"
110 def create_rst_file_names_set(files, start):
111 """Generate a set of unique rst file names.
113 :param files: List of all files to be documented with path beginning in the
115 :param start: The first directory in path which is used in the file name.
118 :returns: Set of unique rst file names.
123 file_names.add(create_file_name(file, start))
127 def add_nested_folders_in_rst_set(file_names, path):
128 """Add RST files from folders where are only folders without tests.
130 :param file_names: List of all files to be documented with path beginning
131 in the working directory.
132 :param path: Path where it starts adding missing RST files.
133 :type file_names: list
137 # When we split directory tree by "/" we don't need to create RST file in
138 # folders in depth <= 5. It's because the WORKING_DIR folder structure i
140 # /tmp/tmp-csitXXX/tests/<subject_of_test>/<type_of_test>/<what_is_tested>
142 # ['', 'tmp', 'tmp-csitXXX', 'tests', 'vpp', 'device', 'container_memif']
143 # We need to generate RST files for folders after <subject_of_test> which
146 for directory in fast_scandir(path):
147 dir_list = directory.split(u"/")
148 if len(dir_list) > 5:
149 # cut ['', 'tmp', 'tmp-csitXXX']
150 dir_rst = u".".join(dir_list[3:]) + u".rst"
151 if dir_rst not in file_names and u"__pycache__" not in dir_rst:
152 file_names.add(dir_rst)
156 """Create a list of files and directories in the given directory.
158 :param path: Path to the directory.
160 :returns: List of directories and list of files sorted in alphabetical
162 :rtype: tuple of two lists
166 items = listdir(path)
168 if isfile(join(path, item)) and u"__init__" not in item:
170 elif isdir(join(path, item)):
172 return sorted(dirs), sorted(files)
175 def write_toc(fh, path, dirs):
176 """Write a table of contents to given rst file.
178 :param fh: File handler of the rst file.
179 :param path: Path to package.
180 :param dirs: List of directories to be included in ToC.
186 for directory in dirs:
187 fh.write(f" {u'.'.join(path)}.{directory}\n")
190 def write_module_title(fh, module_name):
191 """Write the module title to the given rst file. The title will be on the
194 :param fh: File handler of the rst file.
195 :param module_name: The name of module used for title.
197 :type module_name: str
199 title = f"{module_name} suite"
200 fh.write(f"\n{title}\n{u'-' * len(title)}")
203 def generate_py_rst_files():
204 """Generate all rst files for all python modules."""
206 dirs_ignore_list = [u"__pycache__", ]
208 py_libs = get_files(PATH_PY_LIBS, PY_EXT)
209 file_names = create_rst_file_names_set(py_libs, RESOURCES_DIR)
211 for file_name in file_names:
212 path = join(WORKING_DIR, *file_name.split(u".")[:-1])
213 dirs, files = scan_dir(path)
215 for item in dirs_ignore_list:
222 full_path = join(WORKING_DIR, file_name)
223 with open(full_path, mode="a") as fh:
224 if getsize(full_path) == 0:
225 package = file_name.split(u".")[-2]
226 fh.write(f"{package}\n")
227 fh.write(u"=" * len(f"{package}"))
228 module_path = file_name.split(u".")[:-1]
230 write_toc(fh, module_path, dirs)
232 module_name = file.split(u".")[0]
233 write_module_title(fh, module_name)
234 fh.write(rst_py_module.format(
235 u".".join(module_path), module_name)
239 def generate_rf_rst_files(
240 file_names, incl_tests=True, incl_keywords=True, incl_suite_setup=False,
241 incl_variables=False):
242 """Generate rst files for the given robot modules.
244 :param file_names: List of file names to be included in the documentation
246 :param incl_tests: If True, tests will be included in the documentation.
247 :param incl_keywords: If True, keywords will be included in the
249 :param incl_suite_setup: If True, the suite setup will be included in the
251 :param incl_variables: If True, the variables will be included in the
253 :type file_names: set
254 :type incl_tests: bool
255 :type incl_keywords: bool
256 :type incl_suite_setup: bool
257 :type incl_variables: bool
260 for file_name in file_names:
261 path = join(WORKING_DIR, *file_name.split(u".")[:-1])
262 dirs, files = scan_dir(path)
264 full_path = join(WORKING_DIR, file_name)
265 with open(full_path, mode="a") as fh:
266 if getsize(full_path) == 0:
267 package = file_name.split(u".")[-2]
268 fh.write(f"{package}\n")
269 fh.write(u"=" * len(f"{package}") + u"\n")
270 module_path = file_name.split(u".")[:-1]
272 write_toc(fh, module_path, dirs)
274 module_name = file.split(u".")[0]
275 write_module_title(fh, module_name)
276 path = join(join(*module_path), module_name + RF_EXT)
278 fh.write(rst_rf_suite_setup.format(path))
280 fh.write(rst_rf_variables.format(path))
282 fh.write(rst_rf_keywords.format(path))
284 fh.write(rst_rf_tests.format(path))
287 def generate_kw_rst_files():
288 """Generate all rst files for all robot modules with keywords in libraries
289 directory (no tests)."""
291 rf_libs = get_files(PATH_RF_LIBS, RF_EXT)
292 file_names = create_rst_file_names_set(rf_libs, RESOURCES_DIR)
294 generate_rf_rst_files(file_names, incl_tests=False)
297 def generate_tests_rst_files():
298 """Generate all rst files for all robot modules with tests in tests
299 directory. Include also keywords defined in these modules."""
301 tests = get_files(PATH_TESTS, RF_EXT)
302 file_names = create_rst_file_names_set(tests, TESTS_DIR)
303 add_nested_folders_in_rst_set(file_names, PATH_TESTS)
305 generate_rf_rst_files(
306 file_names, incl_suite_setup=True, incl_variables=True
310 def fast_scandir(dirname):
311 subfolders = [f.path for f in scandir(dirname) if f.is_dir()]
312 for dirname in list(subfolders):
313 subfolders.extend(fast_scandir(dirname))
317 if __name__ == u"__main__":
319 # Generate all rst files:
320 generate_py_rst_files()
321 generate_kw_rst_files()
322 generate_tests_rst_files()