2 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # Filter for .siphon files that are generated by other filters.
17 # The idea is to siphon off certain initializers so that we can better
18 # auto-document the contents of that initializer.
20 import os, sys, re, argparse, cgi, json
21 import pyparsing as pp
25 DEFAULT_SIPHON ="clicmd"
27 DEFAULT_PREFIX = os.getcwd()
30 'clicmd': "VLIB_CLI_COMMAND",
33 ap = argparse.ArgumentParser()
34 ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON,
35 choices=siphon_map.keys(),
36 help="Siphon type to process [%s]" % DEFAULT_SIPHON)
37 ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT,
38 help="Output directory for .md files [%s]" % DEFAULT_OUTPUT)
39 ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX,
40 help="Prefix to strip from input pathnames [%s]" % DEFAULT_PREFIX)
41 ap.add_argument("input", nargs='+', metavar="input_file",
42 help="Input .siphon files")
43 args = ap.parse_args()
45 if args.output is None:
46 sys.stderr.write("Error: Siphon processor requires --output to be set.")
50 def clicmd_index_sort(cfg, group, dec):
51 if group in dec and 'group_label' in dec[group]:
52 return dec[group]['group_label']
55 def clicmd_index_header(cfg):
56 s = "# CLI command index\n"
60 def clicmd_index_section(cfg, group, md):
61 return "\n@subpage %s\n\n" % md
63 def clicmd_index_entry(cfg, meta, item):
65 return "* [%s](@ref %s)\n" % (v["path"], meta["label"])
67 def clicmd_sort(cfg, meta, item):
68 return item['value']['path']
70 def clicmd_header(cfg, group, md, dec):
71 if group in dec and 'group_label' in dec[group]:
72 label = dec[group]['group_label']
75 return "\n@page %s %s\n" % (md, label)
77 def clicmd_format(cfg, meta, item):
79 s = "\n@section %s %s\n" % (meta['label'], v['path'])
81 # The text from '.short_help = '.
82 # Later we should split this into short_help and usage_help
83 # since the latter is how it is primarily used but the former
86 tmp = v["short_help"].strip()
88 # Bit hacky. Add a trailing period if it doesn't have one.
92 s += "### Summary/usage\n %s\n\n" % tmp
94 # This is seldom used and will likely be deprecated
98 s += "### Long help\n %s\n\n" % tmp
100 # Extracted from the code in /*? ... ?*/ blocks
101 if "siphon_block" in item["meta"]:
102 sb = item["meta"]["siphon_block"]
105 # hack. still needed?
106 sb = sb.replace("\n", "\\n")
108 sb = json.loads('"'+sb+'"')
109 s += "### Description\n%s\n\n" % sb
113 # Gives some developer-useful linking
114 if "item" in meta or "function" in v:
115 s += "### Declaration and implementation\n\n"
118 s += "Declaration: @ref %s (%s:%d)\n\n" % \
119 (meta['item'], meta["file"], int(item["meta"]["line_start"]))
122 s += "Implementation: @ref %s.\n\n" % v["function"]
128 "VLIB_CLI_COMMAND": {
129 "index_sort_key": clicmd_index_sort,
130 "index_header": clicmd_index_header,
131 "index_section": clicmd_index_section,
132 "index_entry": clicmd_index_entry,
133 'sort_key': clicmd_sort,
134 "header": clicmd_header,
135 "format": clicmd_format,
140 # PyParsing definition for our struct initializers which look like this:
141 # VLIB_CLI_COMMAND (show_sr_tunnel_command, static) = {
142 # .path = "show sr tunnel",
143 # .short_help = "show sr tunnel [name <sr-tunnel-name>]",
144 # .function = show_sr_tunnel_fn,
146 def getMacroInitializerBNF():
148 ident = pp.Word(pp.alphas + "_", pp.alphas + pp.nums + "_")
149 intNum = pp.Word(pp.nums)
150 hexNum = pp.Literal("0x") + pp.Word(pp.hexnums)
151 octalNum = pp.Literal("0") + pp.Word("01234567")
152 integer = (hexNum | octalNum | intNum) + \
153 pp.Optional(pp.Literal("ULL") | pp.Literal("LL") | pp.Literal("L"))
154 floatNum = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?') + pp.Optional(pp.Literal("f"))
155 char = pp.Literal("'") + pp.Word(pp.printables, exact=1) + pp.Literal("'")
156 arrayIndex = integer | ident
158 lbracket = pp.Literal("(").suppress()
159 rbracket = pp.Literal(")").suppress()
160 lbrace = pp.Literal("{").suppress()
161 rbrace = pp.Literal("}").suppress()
162 comma = pp.Literal(",").suppress()
163 equals = pp.Literal("=").suppress()
164 dot = pp.Literal(".").suppress()
165 semicolon = pp.Literal(";").suppress()
167 # initializer := { [member = ] (variable | expression | { initializer } ) }
171 typeSpec = pp.Optional("unsigned") + \
172 pp.oneOf("int long short float double char u8 i8 void") + \
173 pp.Optional(pp.Word("*"), default="")
174 typeCast = pp.Combine( "(" + ( typeSpec | typeName ) + ")" ).suppress()
176 string = pp.Combine(pp.OneOrMore(pp.QuotedString(quoteChar='"',
177 escChar='\\', multiline=True)), adjacent=False)
178 literal = pp.Optional(typeCast) + (integer | floatNum | char | string)
179 var = pp.Combine(pp.Optional(typeCast) + varName + pp.Optional("[" + arrayIndex + "]"))
181 expr = (literal | var) # TODO
184 member = pp.Combine(dot + varName + pp.Optional("[" + arrayIndex + "]"))
187 entry = pp.Group(pp.Optional(member + equals, default="") + value)
188 entries = (pp.ZeroOrMore(entry + comma) + entry + pp.Optional(comma)) | \
189 (pp.ZeroOrMore(entry + comma))
191 cs << (lbrace + entries + rbrace)
194 params = pp.Group(pp.ZeroOrMore(expr + comma) + expr)
195 macroParams = lbracket + params + rbracket
197 mi = macroName + pp.Optional(macroParams) + equals + pp.Group(cs) + semicolon
198 mi.ignore(pp.cppStyleComment)
202 mi = getMacroInitializerBNF()
204 # Parse the input file into a more usable dictionary structure
208 for filename in args.input:
209 sys.stderr.write("Parsing items in file \"%s\"...\n" % filename)
211 with open(filename, "r") as fd:
214 cmds['_global'] = data['global']
216 # iterate the items loaded and regroup it
217 for item in data["items"]:
219 o = mi.parseString(item['block']).asList()
221 sys.stderr.write("Exception parsing item: %s\n%s\n" \
222 % (json.dumps(item, separators=(',', ': '), indent=4),
226 group = item['group']
231 if group not in cmds:
234 if file not in cmds[group]:
235 cmds[group][file] = {}
237 if macro not in cmds[group][file]:
238 cmds[group][file][macro] = {}
249 c['meta'][key] = item[key]
251 for i in c['params']:
252 c['value'][i[0]] = cgi.escape(i[1])
254 cmds[group][file][macro][param] = c
257 # Write the header for this siphon type
258 cfg = siphons[siphon_map[args.type]]
259 sys.stdout.write(cfg["index_header"](cfg))
262 def group_sort_key(item):
263 if "index_sort_key" in cfg:
264 return cfg["index_sort_key"](cfg, item, cmds['_global'])
267 # Iterate the dictionary and process it
268 for group in sorted(cmds.keys(), key=group_sort_key):
269 if group.startswith('_'):
272 sys.stderr.write("Processing items in group \"%s\"...\n" % group)
274 cfg = siphons[siphon_map[args.type]]
275 md = group.replace("/", "_").replace(".", "_")
276 sys.stdout.write(cfg["index_section"](cfg, group, md))
279 dec = cmds['_global']
280 contents += cfg["header"](cfg, group, md, dec)
282 for file in sorted(cmds[group].keys()):
283 if group.startswith('_'):
286 sys.stderr.write("- Processing items in file \"%s\"...\n" % file)
288 for macro in sorted(cmds[group][file].keys()):
289 if macro != siphon_map[args.type]:
291 sys.stderr.write("-- Processing items in macro \"%s\"...\n" % macro)
301 def item_sort_key(item):
302 if "sort_key" in cfg:
303 return cfg["sort_key"](cfg, meta, cmds[group][file][macro][item])
306 for param in sorted(cmds[group][file][macro].keys(), key=item_sort_key):
307 sys.stderr.write("--- Processing item \"%s\"...\n" % param)
311 # mangle "md" and the item to make a reference label
312 meta["label"] = "%s___%s" % (meta["md"], param)
314 if "index_entry" in cfg:
315 s = cfg["index_entry"](cfg, meta, cmds[group][file][macro][param])
319 contents += cfg["format"](cfg, meta, cmds[group][file][macro][param])
321 sys.stdout.write(contents)