2 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # Looks for preprocessor macros with struct initializers and siphons them
17 # off into another file for later parsing; ostensibly to generate
18 # documentation from struct initializer data.
20 import os, sys, re, argparse, json
22 DEFAULT_OUTPUT = "build-root/docs/siphons"
23 DEFAULT_PREFIX = os.getcwd()
25 ap = argparse.ArgumentParser()
26 ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT,
27 help="Output directory for .siphon files [%s]" % DEFAULT_OUTPUT)
28 ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX,
29 help="Prefix to strip from input pathnames [%s]" % DEFAULT_PREFIX)
30 ap.add_argument("input", nargs='+', metavar="input_file",
31 help="Input C source files")
32 args = ap.parse_args()
34 """Patterns that match the start of code blocks we want to siphon"""
36 ( re.compile("(?P<m>VLIB_CLI_COMMAND)\s*[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"), "clicmd" ),
39 """Matches a siphon comment block start"""
40 siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$")
42 """Matches a siphon comment block stop"""
43 siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$")
45 """Siphon block directive delimiter"""
46 siphon_block_delimiter = "%%"
48 """Matches a siphon block directive such as '%clicmd:group_label Debug CLI%'"""
49 siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \
50 (siphon_block_delimiter, siphon_block_delimiter))
52 """Matches the start of an initializer block"""
53 siphon_initializer = re.compile("\s*=")
56 count open and close braces in str
57 return (0, index) when braces were found and count becomes 0.
58 index indicates the position at which the last closing brace was
60 return (-1, -1) if a closing brace is found before any opening one.
61 return (count, -1) if not all opening braces are closed, count is the
64 def count_braces(str, count=0, found=False):
65 for index in range(0, len(str)):
69 elif str[index] == '}':
71 # means we never found an open brace
75 if count == 0 and found:
80 # Collated output for each siphon
83 # Build a list of known siphons
85 for item in siphon_patterns:
87 if siphon not in known_siphons:
88 known_siphons.append(siphon)
90 # Setup information for siphons we know about
91 for siphon in known_siphons:
93 "file": "%s/%s.siphon" % (args.output, siphon),
98 # Pre-process file names in case they indicate a file with
101 for filename in args.input:
102 if filename.startswith('@'):
103 with open(filename[1:], 'r') as fp:
104 lines = fp.readlines()
106 files.append(line.strip())
109 files.append(filename)
111 # Iterate all the input files we've been given
112 for filename in files:
113 # Strip the current directory off the start of the
114 # filename for brevity
115 if filename[0:len(args.input_prefix)] == args.input_prefix:
116 filename = filename[len(args.input_prefix):]
117 if filename[0] == "/":
118 filename = filename[1:]
120 # Work out the abbreviated directory name
121 directory = os.path.dirname(filename)
122 if directory[0:2] == "./":
123 directory = directory[2:]
124 elif directory[0:len(args.input_prefix)] == args.input_prefix:
125 directory = directory[len(args.input_prefix):]
126 if directory[0] == "/":
127 directory = directory[1:]
129 # Open the file and explore its contents...
130 sys.stderr.write("Siphoning from %s...\n" % filename)
132 with open(filename) as fd:
142 str = line[:-1] # filter \n
144 """See if there is a block directive and if so extract it"""
145 def process_block_directive(str, directives):
146 m = siphon_block_directive.search(str)
149 v = m.group(3).strip()
151 # Return only the parts we did not match
152 return str[0:m.start(1)] + str[m.end(4):]
156 def process_block_prefix(str):
157 if str.startswith(" * "):
164 # See if the line contains the start of a siphon doc block
165 m = siphon_block_start.search(str)
170 # Now check if the block closes on the same line
171 m = siphon_block_stop.search(t)
176 # Check for directives
177 t = process_block_directive(t, directives)
179 # Filter for normal comment prefixes
180 t = process_block_prefix(t)
189 # Check to see if we have an end block marker
190 m = siphon_block_stop.search(str)
197 # Check for directives
198 t = process_block_directive(t, directives)
200 # Filter for normal comment prefixes
201 t = process_block_prefix(t)
204 siphon_block += t + "\n"
211 # Look for blocks we need to siphon
212 for p in siphon_patterns:
214 siphon = [ p[1], str + "\n", 0 ]
215 siphon_line = line_num
217 # see if we have an initializer
218 m = siphon_initializer.search(str)
220 # count the braces on this line
221 (count, index) = count_braces(str[m.start():])
223 # TODO - it's possible we have the initializer all on the first line
224 # we should check for it, but also account for the possibility that
225 # the open brace is on the next line
228 # close_siphon = siphon
231 # no initializer: close the siphon right now
232 close_siphon = siphon
235 # See if we should end the siphon here - do we have balanced
237 (count, index) = count_braces(str, count=siphon[2], found=True)
239 # braces balanced - add the substring and close the siphon
240 siphon[1] += str[:index+1] + ";\n"
241 close_siphon = siphon
244 # add the whole string, move on
246 siphon[1] += str + "\n"
248 if close_siphon is not None:
249 # Write the siphoned contents to the right place
250 siphon_name = close_siphon[0]
252 # Copy directives for the file
254 for key in directives:
256 (sn, label) = key.split(":")
257 if sn == siphon_name:
258 details[label] = directives[key]
260 details[key] = directives[key]
262 # Copy details for this block
263 details['file'] = filename
264 details['line_start'] = siphon_line
265 details['line_end'] = line_num
266 details['siphon_block'] = siphon_block.strip()
269 if "group" not in details:
270 if "group_label" in details:
271 # use the filename since group labels are mostly of file scope
272 details['group'] = details['file']
274 details['group'] = directory
276 if "group_label" not in details:
277 details['group_label'] = details['group']
279 details["block"] = close_siphon[1]
282 output[siphon_name]['items'].append(details)
289 for key in directives.keys():
293 if filename.endswith("/dir.dox"):
294 # very special! use the parent directory name
299 (sn, label) = key.split(":")
303 if 'global' not in output[sn]:
304 output[sn]['global'] = {}
305 if l not in output[sn]['global']:
306 output[sn]['global'][l] = {}
307 if 'file' not in output[sn]:
308 output[sn]['file'] = "%s/%s.siphon" % (args.output, sn)
309 if 'items' not in output[sn]:
310 output[sn]['items'] = []
312 output[sn]['global'][l][label] = directives[key]
316 for siphon in output.keys():
317 sys.stderr.write("Saving siphon %s...\n" % siphon)
319 with open(s['file'], "a") as fp:
320 json.dump(s, fp, separators=(',', ': '), indent=4, sort_keys=True)