1 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at:
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Generation template class
26 # Classes register themselves in this dictionary
27 """Mapping of known processors to their classes"""
30 """Mapping of known output formats to their classes"""
35 """Generate rendered output for siphoned data."""
42 """Name of an identifier used by this siphon"""
46 """The pyparsing object to use to parse with"""
52 """Group key to (directory,file) mapping"""
58 """Directory to look for siphon rendering templates"""
59 template_directory = None
61 """Directory to output parts in"""
64 """Template environment, if we're using templates"""
67 def __init__(self, template_directory, format, outdir, repository_link):
68 super(Siphon, self).__init__()
69 self.log = logging.getLogger("siphon.process.%s" % self.name)
71 # Get our output format details
72 fmt_klass = formats[format]
76 # Sort out the template search path
78 return os.sep.join((template_directory, fmt.name, name))
80 self.template_directory = template_directory
86 loader = jinja2.FileSystemLoader(searchpath=searchpath)
87 self._tplenv = jinja2.Environment(
91 keep_trailing_newline=True,
94 # Convenience, get a reference to the internal escape and
95 # unescape methods in html.parser. These then become
96 # available to templates to use, if needed.
97 self._h = html.parser.HTMLParser()
98 self.escape = html.escape
99 self.unescape = html.unescape
101 # TODO: customize release
102 self.repository_link = repository_link
106 """Returns an object to be used as the sorting key in the item index."""
108 def index_sort_key(self, group):
111 """Returns a string to use as the header at the top of the item index."""
113 def index_header(self):
114 return self.template("index_header")
116 """Returns the string fragment to use for each section in the item
119 def index_section(self, group):
120 return self.template("index_section", group=group)
122 """Returns the string fragment to use for each entry in the item index."""
124 def index_entry(self, meta, item):
125 return self.template("index_entry", meta=meta, item=item)
127 """Returns an object, typically a string, to be used as the sorting key
128 for items within a section."""
130 def item_sort_key(self, item):
133 """Returns a key for grouping items together."""
135 def group_key(self, directory, file, macro, name):
136 _global = self._cmds["_global"]
138 if file in _global and "group_label" in _global[file]:
139 self._group[file] = (directory, file)
142 self._group[directory] = (directory, None)
145 """Returns a key for identifying items within a grouping."""
147 def item_key(self, directory, file, macro, name):
150 """Returns a string to use as the header when rendering the item."""
152 def item_header(self, group):
153 return self.template("item_header", group=group)
155 """Returns a string to use as the body when rendering the item."""
157 def item_format(self, meta, item):
158 return self.template("item_format", meta=meta, item=item)
160 """Returns a string to use as the label for the page reference."""
162 def page_label(self, group):
163 return "_".join((self.name, self.sanitize_label(group)))
165 """Returns a title to use for a page."""
167 def page_title(self, group):
168 _global = self._cmds["_global"]
169 (directory, file) = self._group[group]
171 if file and file in _global and "group_label" in _global[file]:
172 return _global[file]["group_label"]
174 if directory in _global and "group_label" in _global[directory]:
175 return _global[directory]["group_label"]
179 """Returns a string to use as the label for the section reference."""
181 def item_label(self, group, item):
182 return "__".join((self.name, item))
184 """Label sanitizer; for creating Doxygen references"""
186 def sanitize_label(self, value):
187 return value.replace(" ", "_").replace("/", "_").replace(".", "_")
189 """Template processor"""
191 def template(self, name, **kwargs):
192 tpl = self._tplenv.get_template(name + self._format.extension)
193 return tpl.render(this=self, **kwargs)
197 """Parse the input file into a more usable dictionary structure."""
199 def load_json(self, files):
205 for filename in files:
206 filename = os.path.relpath(filename)
207 self.log.info('Parsing items in file "%s".' % filename)
209 with open(filename, "r") as fd:
212 self._cmds["_global"] = data["global"]
214 # iterate the items loaded and regroup it
215 for item in data["items"]:
217 o = self._parser.parse(item["block"])
220 "Exception parsing item: %s\n%s"
222 json.dumps(item, separators=(",", ": "), indent=4),
228 # Augment the item with metadata
233 o["meta"][key] = item[key]
235 # Load some interesting fields
236 directory = item["directory"]
241 # Generate keys to group items by
242 group_key = self.group_key(directory, file, macro, name)
243 item_key = self.item_key(directory, file, macro, name)
245 if group_key not in self._cmds:
246 self._cmds[group_key] = {}
248 self._cmds[group_key][item_key] = o
250 """Iterate over the input data, calling render methods to generate the
253 def process(self, out=None):
258 # Accumulated body contents
261 # Write the header for this siphon type
262 out.write(self.index_header())
264 # Sort key helper for the index
265 def group_sort_key(group):
266 return self.index_sort_key(group)
268 # Iterate the dictionary and process it
269 for group in sorted(self._cmds.keys(), key=group_sort_key):
270 if group.startswith("_"):
274 'Processing items in group "%s" (%s).' % (group, group_sort_key(group))
277 # Generate the section index entry (write it now)
278 out.write(self.index_section(group))
280 # Generate the item header (save for later)
281 contents += self.item_header(group)
283 def item_sort_key(key):
284 return self.item_sort_key(self._cmds[group][key])
286 for key in sorted(self._cmds[group].keys(), key=item_sort_key):
288 '--- Processing key "%s" (%s).' % (key, item_sort_key(key))
291 o = self._cmds[group][key]
293 "directory": o["meta"]["directory"],
294 "file": o["meta"]["file"],
298 "label": self.item_label(group, key),
301 # Generate the index entry for the item (write it now)
302 out.write(self.index_entry(meta, o))
304 # Generate the item itself (save for later)
305 contents += self.item_format(meta, o)
307 page_name = self.separate_page_names(group)
309 path = os.path.join(self.outdir, page_name)
310 with open(path, "w+") as page:
314 # Deliver the accumulated body output
317 def do_cliexstart(self, matchobj):
318 title = matchobj.group(1)
319 title = " ".join(title.splitlines())
320 content = matchobj.group(2)
321 content = re.sub(r"\n", r"\n ", content)
322 return "\n\n.. code-block:: console\n\n %s\n %s\n\n" % (title, content)
324 def do_clistart(self, matchobj):
325 content = matchobj.group(1)
326 content = re.sub(r"\n", r"\n ", content)
327 return "\n\n.. code-block:: console\n\n %s\n\n" % content
329 def do_cliexcmd(self, matchobj):
330 content = matchobj.group(1)
331 content = " ".join(content.splitlines())
332 return "\n\n.. code-block:: console\n\n %s\n\n" % content
334 def process_list(self, matchobj):
335 content = matchobj.group(1)
336 content = self.reindent(content, 2)
337 return "@@@@%s\nBBBB" % content
339 def process_special(self, s):
340 # ----------- markers to remove
341 s = re.sub(r"@cliexpar\s*", r"", s)
342 s = re.sub(r"@parblock\s*", r"", s)
343 s = re.sub(r"@endparblock\s*", r"", s)
344 s = re.sub(r"<br>", "", s)
345 # ----------- emphasis
347 s = re.sub(r"<b><em>\s*", "``", s)
348 s = re.sub(r"\s*</b></em>", "``", s)
349 s = re.sub(r"\s*</em></b>", "``", s)
351 s = re.sub(r"<b>\s*", "**", s)
352 s = re.sub(r"\s*</b>", "**", s)
354 s = re.sub(r"<code>\s*", "``", s)
355 s = re.sub(r"\s*</code>", "``", s)
357 s = re.sub(r"'?<em>\s*", r"``", s)
358 s = re.sub(r"\s*</em>'?", r"``", s)
360 s = re.sub(r"@c\s(\S+)", r"``\1``", s)
362 s = re.sub(r"@todo[^\n]*", "", s)
363 s = re.sub(r"@TODO[^\n]*", "", s)
364 # ----------- code blocks
365 s = re.sub(r"@cliexcmd{(.+?)}", self.do_cliexcmd, s, flags=re.DOTALL)
367 r"@cliexstart{(.+?)}(.+?)@cliexend", self.do_cliexstart, s, flags=re.DOTALL
369 s = re.sub(r"@clistart(.+?)@cliend", self.do_clistart, s, flags=re.DOTALL)
371 s = re.sub(r"^\s*-", r"\n@@@@", s, flags=re.MULTILINE)
372 s = re.sub(r"@@@@(.*?)\n\n+", self.process_list, s, flags=re.DOTALL)
373 s = re.sub(r"BBBB@@@@", r"-", s)
374 s = re.sub(r"@@@@", r"-", s)
375 s = re.sub(r"BBBB", r"\n\n", s)
376 # ----------- Cleanup remains
377 s = re.sub(r"@cliexend\s*", r"", s)
380 def separate_page_names(self, group):
383 # This push the given textblock <indent> spaces right
384 def reindent(self, s, indent):
386 s = re.sub(r"\n", "\n" + ind, s)
389 # This aligns the given textblock left (no indent)
390 def noindent(self, s):
391 s = re.sub(r"\n[ \f\v\t]*", "\n", s)
395 class Format(object):
396 """Output format class"""
398 """Name of this output format"""
401 """Expected file extension of templates that build this format"""
405 class FormatMarkdown(Format):
406 """Markdown output format"""
412 # Register 'markdown'
413 formats["markdown"] = FormatMarkdown
416 class FormatItemlist(Format):
417 """Itemlist output format"""
420 extension = ".itemlist"
423 # Register 'itemlist'
424 formats["itemlist"] = FormatItemlist