1 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at:
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Generation template class
26 # Classes register themselves in this dictionary
27 """Mapping of known processors to their classes"""
30 """Mapping of known output formats to their classes"""
35 """Generate rendered output for siphoned data."""
42 """Name of an identifier used by this siphon"""
46 """The pyparsing object to use to parse with"""
52 """Group key to (directory,file) mapping"""
58 """Directory to look for siphon rendering templates"""
59 template_directory = None
61 """Directory to output parts in"""
64 """Template environment, if we're using templates"""
67 def __init__(self, template_directory, format, outdir, repository_link):
68 super(Siphon, self).__init__()
69 self.log = logging.getLogger("siphon.process.%s" % self.name)
71 # Get our output format details
72 fmt_klass = formats[format]
76 # Sort out the template search path
78 return os.sep.join((template_directory, fmt.name, name))
80 self.template_directory = template_directory
86 loader = jinja2.FileSystemLoader(searchpath=searchpath)
87 self._tplenv = jinja2.Environment(
91 keep_trailing_newline=True,
94 # Convenience, get a reference to the internal escape and
95 # unescape methods in html.parser. These then become
96 # available to templates to use, if needed.
97 self._h = html.parser.HTMLParser()
98 self.escape = html.escape
99 self.unescape = html.unescape
101 # TODO: customize release
102 self.repository_link = repository_link
106 """Returns an object to be used as the sorting key in the item index."""
108 def index_sort_key(self, group):
111 """Returns a string to use as the header at the top of the item index."""
113 def index_header(self):
114 return self.template("index_header")
116 """Returns the string fragment to use for each section in the item
119 def index_section(self, group):
120 return self.template("index_section", group=group)
122 """Returns the string fragment to use for each entry in the item index."""
124 def index_entry(self, meta, item):
125 return self.template("index_entry", meta=meta, item=item)
127 """Returns an object, typically a string, to be used as the sorting key
128 for items within a section."""
130 def item_sort_key(self, item):
133 """Returns a key for grouping items together."""
135 def group_key(self, directory, file, macro, name):
136 _global = self._cmds["_global"]
138 if file in _global and "group_label" in _global[file]:
139 self._group[file] = (directory, file)
142 self._group[directory] = (directory, None)
145 """Returns a key for identifying items within a grouping."""
147 def item_key(self, directory, file, macro, name):
150 """Returns a string to use as the header when rendering the item."""
152 def item_header(self, group):
153 return self.template("item_header", group=group)
155 """Returns a string to use as the body when rendering the item."""
157 def item_format(self, meta, item):
158 return self.template("item_format", meta=meta, item=item)
160 """Returns a string to use as the label for the page reference."""
162 def page_label(self, group):
163 return "_".join((self.name, self.sanitize_label(group)))
165 """Returns a title to use for a page."""
167 def page_title(self, group):
168 _global = self._cmds["_global"]
169 (directory, file) = self._group[group]
171 if file and file in _global and "group_label" in _global[file]:
172 return _global[file]["group_label"]
174 if directory in _global and "group_label" in _global[directory]:
175 return _global[directory]["group_label"]
179 """Returns a string to use as the label for the section reference."""
181 def item_label(self, group, item):
182 return "__".join((self.name, item))
184 """Label sanitizer; for creating Doxygen references"""
186 def sanitize_label(self, value):
187 return value.replace(" ", "_").replace("/", "_").replace(".", "_")
189 """Template processor"""
191 def template(self, name, **kwargs):
192 tpl = self._tplenv.get_template(name + self._format.extension)
193 return tpl.render(this=self, **kwargs)
197 """Parse the input file into a more usable dictionary structure."""
199 def load_json(self, files):
205 for filename in files:
206 filename = os.path.relpath(filename)
207 self.log.info('Parsing items in file "%s".' % filename)
209 with open(filename, "r") as fd:
212 self._cmds["_global"] = data["global"]
214 # iterate the items loaded and regroup it
215 for item in data["items"]:
217 o = self._parser.parse(item["block"])
220 "Exception parsing item: %s\n%s"
222 json.dumps(item, separators=(",", ": "), indent=4),
228 # Augment the item with metadata
233 o["meta"][key] = item[key]
235 # Load some interesting fields
236 directory = item["directory"]
241 # Generate keys to group items by
242 group_key = self.group_key(directory, file, macro, name)
243 item_key = self.item_key(directory, file, macro, name)
245 if group_key not in self._cmds:
246 self._cmds[group_key] = {}
248 self._cmds[group_key][item_key] = o
250 """Iterate over the input data, calling render methods to generate the
253 def process(self, out=None):
257 # Accumulated body contents
260 # Write the header for this siphon type
261 out.write(self.index_header())
263 # Sort key helper for the index
264 def group_sort_key(group):
265 return self.index_sort_key(group)
267 # Iterate the dictionary and process it
268 for group in sorted(self._cmds.keys(), key=group_sort_key):
269 if group.startswith("_"):
273 'Processing items in group "%s" (%s).' % (group, group_sort_key(group))
276 # Generate the section index entry (write it now)
277 out.write(self.index_section(group))
279 # Generate the item header (save for later)
280 contents += self.item_header(group)
282 def item_sort_key(key):
283 return self.item_sort_key(self._cmds[group][key])
285 for key in sorted(self._cmds[group].keys(), key=item_sort_key):
287 '--- Processing key "%s" (%s).' % (key, item_sort_key(key))
290 o = self._cmds[group][key]
292 "directory": o["meta"]["directory"],
293 "file": o["meta"]["file"],
297 "label": self.item_label(group, key),
300 # Generate the index entry for the item (write it now)
301 out.write(self.index_entry(meta, o))
303 # Generate the item itself (save for later)
304 contents += self.item_format(meta, o)
306 page_name = self.separate_page_names(group)
308 path = os.path.join(self.outdir, page_name)
309 with open(path, "w+") as page:
313 # Deliver the accumulated body output
316 def do_cliexstart(self, matchobj):
317 title = matchobj.group(1)
318 title = " ".join(title.splitlines())
319 content = matchobj.group(2)
320 content = re.sub(r"\n", r"\n ", content)
321 return "\n\n.. code-block:: console\n\n %s\n %s\n\n" % (title, content)
323 def do_clistart(self, matchobj):
324 content = matchobj.group(1)
325 content = re.sub(r"\n", r"\n ", content)
326 return "\n\n.. code-block:: console\n\n %s\n\n" % content
328 def do_cliexcmd(self, matchobj):
329 content = matchobj.group(1)
330 content = " ".join(content.splitlines())
331 return "\n\n.. code-block:: console\n\n %s\n\n" % content
333 def process_list(self, matchobj):
334 content = matchobj.group(1)
335 content = self.reindent(content, 2)
336 return "@@@@%s\nBBBB" % content
338 def process_special(self, s):
339 # ----------- markers to remove
340 s = re.sub(r"@cliexpar\s*", r"", s)
341 s = re.sub(r"@parblock\s*", r"", s)
342 s = re.sub(r"@endparblock\s*", r"", s)
343 s = re.sub(r"<br>", "", s)
344 # ----------- emphasis
346 s = re.sub(r"<b><em>\s*", "``", s)
347 s = re.sub(r"\s*</b></em>", "``", s)
348 s = re.sub(r"\s*</em></b>", "``", s)
350 s = re.sub(r"<b>\s*", "**", s)
351 s = re.sub(r"\s*</b>", "**", s)
353 s = re.sub(r"<code>\s*", "``", s)
354 s = re.sub(r"\s*</code>", "``", s)
356 s = re.sub(r"'?<em>\s*", r"``", s)
357 s = re.sub(r"\s*</em>'?", r"``", s)
359 s = re.sub(r"@c\s(\S+)", r"``\1``", s)
361 s = re.sub(r"@todo[^\n]*", "", s)
362 s = re.sub(r"@TODO[^\n]*", "", s)
363 # ----------- code blocks
364 s = re.sub(r"@cliexcmd{(.+?)}", self.do_cliexcmd, s, flags=re.DOTALL)
366 r"@cliexstart{(.+?)}(.+?)@cliexend", self.do_cliexstart, s, flags=re.DOTALL
368 s = re.sub(r"@clistart(.+?)@cliend", self.do_clistart, s, flags=re.DOTALL)
370 s = re.sub(r"^\s*-", r"\n@@@@", s, flags=re.MULTILINE)
371 s = re.sub(r"@@@@(.*?)\n\n+", self.process_list, s, flags=re.DOTALL)
372 s = re.sub(r"BBBB@@@@", r"-", s)
373 s = re.sub(r"@@@@", r"-", s)
374 s = re.sub(r"BBBB", r"\n\n", s)
375 # ----------- Cleanup remains
376 s = re.sub(r"@cliexend\s*", r"", s)
379 def separate_page_names(self, group):
382 # This push the given textblock <indent> spaces right
383 def reindent(self, s, indent):
385 s = re.sub(r"\n", "\n" + ind, s)
388 # This aligns the given textblock left (no indent)
389 def noindent(self, s):
390 s = re.sub(r"\n[ \f\v\t]*", "\n", s)
394 class Format(object):
395 """Output format class"""
397 """Name of this output format"""
400 """Expected file extension of templates that build this format"""
404 class FormatMarkdown(Format):
405 """Markdown output format"""
411 # Register 'markdown'
412 formats["markdown"] = FormatMarkdown
415 class FormatItemlist(Format):
416 """Itemlist output format"""
419 extension = ".itemlist"
422 # Register 'itemlist'
423 formats["itemlist"] = FormatItemlist