doxygen/siphon/generate.py

   1 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at:
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Generate .siphon source fragments for later processing
  16
  17 import json
  18 import logging
  19 import os
  20 import re
  21
  22 """List of (regexp, siphon_name) tuples for matching the start of C
  23    initializer blocks in source files. Each siphon class registers
  24    themselves on this list."""
  25 siphon_patterns = []
  26
  27 class Generate(object):
  28     """Matches a siphon comment block start"""
  29     siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$")
  30
  31     """Matches a siphon comment block stop"""
  32     siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$")
  33
  34     """Siphon block directive delimiter"""
  35     siphon_block_delimiter = "%%"
  36
  37     """Matches a siphon block directive such as
  38        '%clicmd:group_label Debug CLI%'"""
  39     siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \
  40             (siphon_block_delimiter, siphon_block_delimiter))
  41
  42     """Matches the start of an initializer block"""
  43     siphon_initializer = re.compile("\s*=")
  44
  45     """Collated output for each siphon"""
  46     output = None
  47
  48     """Directory prefix to strip from input filenames to keep things tidy."""
  49     input_prefix = None
  50
  51     """List of known siphons"""
  52     known_siphons = None
  53
  54     """Logging handler"""
  55     log = None
  56
  57
  58     def __init__(self, output_directory, input_prefix):
  59         super(Generate, self).__init__()
  60         self.log = logging.getLogger("siphon.generate")
  61
  62         # Build a list of known siphons
  63         self.known_siphons = []
  64         for item in siphon_patterns:
  65             siphon = item[1]
  66             if siphon not in self.known_siphons:
  67                 self.known_siphons.append(siphon)
  68
  69         # Setup information for siphons we know about
  70         self.output = {}
  71         for siphon in self.known_siphons:
  72             self.output[siphon] = {
  73                     "file": "%s/%s.siphon" % (output_directory, siphon),
  74                     "global": {},
  75                     "items": [],
  76                 }
  77
  78         self.input_prefix = input_prefix
  79
  80
  81     """
  82     count open and close braces in str
  83     return (0, index) when braces were found and count becomes 0.
  84     index indicates the position at which the last closing brace was
  85     found.
  86     return (-1, -1) if a closing brace is found before any opening one.
  87     return (count, -1) if not all opening braces are closed, count is the
  88     current depth
  89     """
  90     def count_braces(self, str, count=0, found=False):
  91         for index in range(0, len(str)):
  92             if str[index] == '{':
  93                 count += 1;
  94                 found = True
  95             elif str[index] == '}':
  96                 if count == 0:
  97                     # means we never found an open brace
  98                     return (-1, -1)
  99                 count -= 1;
 100
 101             if count == 0 and found:
 102                 return (count, index)
 103
 104         return (count, -1)
 105
 106     def parse(self, filename):
 107         # Strip the current directory off the start of the
 108         # filename for brevity
 109         if filename[0:len(self.input_prefix)] == self.input_prefix:
 110             filename = filename[len(self.input_prefix):]
 111             if filename[0] == "/":
 112                 filename = filename[1:]
 113
 114         # Work out the abbreviated directory name
 115         directory = os.path.dirname(filename)
 116         if directory[0:2] == "./":
 117             directory = directory[2:]
 118         elif directory[0:len(self.input_prefix)] == self.input_prefix:
 119             directory = directory[len(self.input_prefix):]
 120         if directory[0] == "/":
 121             directory = directory[1:]
 122
 123         # Open the file and explore its contents...
 124         self.log.info("Siphoning from %s." % filename)
 125         directives = {}
 126         with open(filename) as fd:
 127             siphon = None
 128             close_siphon = None
 129             siphon_block = ""
 130             in_block = False
 131             line_num = 0
 132             siphon_line = 0
 133
 134             for line in fd:
 135                 line_num += 1
 136                 str = line[:-1] # filter \n
 137
 138                 """See if there is a block directive and if so extract it"""
 139                 def process_block_directive(str, directives):
 140                     m = self.siphon_block_directive.search(str)
 141                     if m is not None:
 142                         k = m.group(2)
 143                         v = m.group(3).strip()
 144                         directives[k] = v
 145                         # Return only the parts we did not match
 146                         return str[0:m.start(1)] + str[m.end(4):]
 147
 148                     return str
 149
 150                 def process_block_prefix(str):
 151                     if str.startswith(" * "):
 152                         str = str[3:]
 153                     elif str == " *":
 154                         str = ""
 155                     return str
 156
 157                 if not in_block:
 158                     # See if the line contains the start of a siphon doc block
 159                     m = self.siphon_block_start.search(str)
 160                     if m is not None:
 161                         in_block = True
 162                         t = m.group(1)
 163
 164                         # Now check if the block closes on the same line
 165                         m = self.siphon_block_stop.search(t)
 166                         if m is not None:
 167                             t = m.group(1)
 168                             in_block = False
 169
 170                         # Check for directives
 171                         t = process_block_directive(t, directives)
 172
 173                         # Filter for normal comment prefixes
 174                         t = process_block_prefix(t)
 175
 176                         # Add what is left
 177                         siphon_block += t
 178
 179                         # Skip to next line
 180                         continue
 181
 182                 else:
 183                     # Check to see if we have an end block marker
 184                     m = self.siphon_block_stop.search(str)
 185                     if m is not None:
 186                         in_block = False
 187                         t = m.group(1)
 188                     else:
 189                         t = str
 190
 191                     # Check for directives
 192                     t = process_block_directive(t, directives)
 193
 194                     # Filter for normal comment prefixes
 195                     t = process_block_prefix(t)
 196
 197                     # Add what is left
 198                     siphon_block += t + "\n"
 199
 200                     # Skip to next line
 201                     continue
 202
 203
 204                 if siphon is None:
 205                     # Look for blocks we need to siphon
 206                     for p in siphon_patterns:
 207                         if p[0].match(str):
 208                             siphon = [ p[1], str + "\n", 0 ]
 209                             siphon_line = line_num
 210
 211                             # see if we have an initializer
 212                             m = self.siphon_initializer.search(str)
 213                             if m is not None:
 214                                 # count the braces on this line
 215                                 (count, index) = \
 216                                     self.count_braces(str[m.start():])
 217                                 siphon[2] = count
 218                                 # TODO - it's possible we have the
 219                                 # initializer all on the first line
 220                                 # we should check for it, but also
 221                                 # account for the possibility that
 222                                 # the open brace is on the next line
 223                                 #if count == 0:
 224                                 #    # braces balanced
 225                                 #    close_siphon = siphon
 226                                 #    siphon = None
 227                             else:
 228                                 # no initializer: close the siphon right now
 229                                 close_siphon = siphon
 230                                 siphon = None
 231                 else:
 232                     # See if we should end the siphon here - do we have
 233                     # balanced braces?
 234                     (count, index) = self.count_braces(str,
 235                             count=siphon[2], found=True)
 236                     if count == 0:
 237                         # braces balanced - add the substring and
 238                         # close the siphon
 239                         siphon[1] += str[:index+1] + ";\n"
 240                         close_siphon = siphon
 241                         siphon = None
 242                     else:
 243                         # add the whole string, move on
 244                         siphon[2] = count
 245                         siphon[1] += str + "\n"
 246
 247                 if close_siphon is not None:
 248                     # Write the siphoned contents to the right place
 249                     siphon_name = close_siphon[0]
 250
 251                     # Copy directives for the file
 252                     details = {}
 253                     for key in directives:
 254                         if ":" in key:
 255                             (sn, label) = key.split(":")
 256                             if sn == siphon_name:
 257                                 details[label] = directives[key]
 258                         else:
 259                             details[key] = directives[key]
 260
 261                     # Copy details for this block
 262                     details['file'] = filename
 263                     details['directory'] = directory
 264                     details['line_start'] = siphon_line
 265                     details['line_end'] = line_num
 266                     details['siphon_block'] = siphon_block.strip()
 267                     details["block"] = close_siphon[1]
 268
 269                     # Store the item
 270                     self.output[siphon_name]['items'].append(details)
 271
 272                     # All done
 273                     close_siphon = None
 274                     siphon_block = ""
 275
 276             # Update globals
 277             for key in directives.keys():
 278                 if ':' not in key:
 279                     continue
 280
 281                 if filename.endswith("/dir.dox"):
 282                     # very special! use the parent directory name
 283                     l = directory
 284                 else:
 285                     l = filename
 286
 287                 (sn, label) = key.split(":")
 288
 289                 if sn not in self.output:
 290                     self.output[sn] = {}
 291                 if 'global' not in self.output[sn]:
 292                     self.output[sn]['global'] = {}
 293                 if l not in self.output[sn]['global']:
 294                     self.output[sn]['global'][l] = {}
 295
 296                 self.output[sn]['global'][l][label] = directives[key]
 297
 298     def deliver(self):
 299         # Write out the data
 300         for siphon in self.output.keys():
 301             self.log.info("Saving siphon data %s." % siphon)
 302             s = self.output[siphon]
 303             with open(s['file'], "a") as fp:
 304                 json.dump(s, fp,
 305                     separators=(',', ': '), indent=4, sort_keys=True)
 306