doxygen/siphon/generate.py

   1 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at:
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Generate .siphon source fragments for later processing
  16
  17 import logging
  18 import os, sys, re, json
  19
  20 """List of (regexp, siphon_name) tuples for matching the start of C
  21    initializer blocks in source files. Each siphon class registers
  22    themselves on tihs list."""
  23 siphon_patterns = []
  24
  25 class Generate(object):
  26     """Matches a siphon comment block start"""
  27     siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$")
  28
  29     """Matches a siphon comment block stop"""
  30     siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$")
  31
  32     """Siphon block directive delimiter"""
  33     siphon_block_delimiter = "%%"
  34
  35     """Matches a siphon block directive such as
  36        '%clicmd:group_label Debug CLI%'"""
  37     siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \
  38             (siphon_block_delimiter, siphon_block_delimiter))
  39
  40     """Matches the start of an initializer block"""
  41     siphon_initializer = re.compile("\s*=")
  42
  43     """Collated output for each siphon"""
  44     output = None
  45
  46     """Directory prefix to strip from input filenames to keep things tidy."""
  47     input_prefix = None
  48
  49     """List of known siphons"""
  50     known_siphons = None
  51
  52     """Logging handler"""
  53     log = None
  54
  55
  56     def __init__(self, output_directory, input_prefix):
  57         super(Generate, self).__init__()
  58         self.log = logging.getLogger("siphon.generate")
  59
  60         # Build a list of known siphons
  61         self.known_siphons = []
  62         for item in siphon_patterns:
  63             siphon = item[1]
  64             if siphon not in self.known_siphons:
  65                 self.known_siphons.append(siphon)
  66
  67         # Setup information for siphons we know about
  68         self.output = {}
  69         for siphon in self.known_siphons:
  70             self.output[siphon] = {
  71                     "file": "%s/%s.siphon" % (output_directory, siphon),
  72                     "global": {},
  73                     "items": [],
  74                 }
  75
  76         self.input_prefix = input_prefix
  77
  78
  79     """
  80     count open and close braces in str
  81     return (0, index) when braces were found and count becomes 0.
  82     index indicates the position at which the last closing brace was
  83     found.
  84     return (-1, -1) if a closing brace is found before any opening one.
  85     return (count, -1) if not all opening braces are closed, count is the
  86     current depth
  87     """
  88     def count_braces(self, str, count=0, found=False):
  89         for index in range(0, len(str)):
  90             if str[index] == '{':
  91                 count += 1;
  92                 found = True
  93             elif str[index] == '}':
  94                 if count == 0:
  95                     # means we never found an open brace
  96                     return (-1, -1)
  97                 count -= 1;
  98
  99             if count == 0 and found:
 100                 return (count, index)
 101
 102         return (count, -1)
 103
 104     def parse(self, filename):
 105         # Strip the current directory off the start of the
 106         # filename for brevity
 107         if filename[0:len(self.input_prefix)] == self.input_prefix:
 108             filename = filename[len(self.input_prefix):]
 109             if filename[0] == "/":
 110                 filename = filename[1:]
 111
 112         # Work out the abbreviated directory name
 113         directory = os.path.dirname(filename)
 114         if directory[0:2] == "./":
 115             directory = directory[2:]
 116         elif directory[0:len(self.input_prefix)] == self.input_prefix:
 117             directory = directory[len(self.input_prefix):]
 118         if directory[0] == "/":
 119             directory = directory[1:]
 120
 121         # Open the file and explore its contents...
 122         self.log.info("Siphoning from %s." % filename)
 123         directives = {}
 124         with open(filename) as fd:
 125             siphon = None
 126             close_siphon = None
 127             siphon_block = ""
 128             in_block = False
 129             line_num = 0
 130             siphon_line = 0
 131
 132             for line in fd:
 133                 line_num += 1
 134                 str = line[:-1] # filter \n
 135
 136                 """See if there is a block directive and if so extract it"""
 137                 def process_block_directive(str, directives):
 138                     m = self.siphon_block_directive.search(str)
 139                     if m is not None:
 140                         k = m.group(2)
 141                         v = m.group(3).strip()
 142                         directives[k] = v
 143                         # Return only the parts we did not match
 144                         return str[0:m.start(1)] + str[m.end(4):]
 145
 146                     return str
 147
 148                 def process_block_prefix(str):
 149                     if str.startswith(" * "):
 150                         str = str[3:]
 151                     elif str == " *":
 152                         str = ""
 153                     return str
 154
 155                 if not in_block:
 156                     # See if the line contains the start of a siphon doc block
 157                     m = self.siphon_block_start.search(str)
 158                     if m is not None:
 159                         in_block = True
 160                         t = m.group(1)
 161
 162                         # Now check if the block closes on the same line
 163                         m = self.siphon_block_stop.search(t)
 164                         if m is not None:
 165                             t = m.group(1)
 166                             in_block = False
 167
 168                         # Check for directives
 169                         t = process_block_directive(t, directives)
 170
 171                         # Filter for normal comment prefixes
 172                         t = process_block_prefix(t)
 173
 174                         # Add what is left
 175                         siphon_block += t
 176
 177                         # Skip to next line
 178                         continue
 179
 180                 else:
 181                     # Check to see if we have an end block marker
 182                     m = self.siphon_block_stop.search(str)
 183                     if m is not None:
 184                         in_block = False
 185                         t = m.group(1)
 186                     else:
 187                         t = str
 188
 189                     # Check for directives
 190                     t = process_block_directive(t, directives)
 191
 192                     # Filter for normal comment prefixes
 193                     t = process_block_prefix(t)
 194
 195                     # Add what is left
 196                     siphon_block += t + "\n"
 197
 198                     # Skip to next line
 199                     continue
 200
 201
 202                 if siphon is None:
 203                     # Look for blocks we need to siphon
 204                     for p in siphon_patterns:
 205                         if p[0].match(str):
 206                             siphon = [ p[1], str + "\n", 0 ]
 207                             siphon_line = line_num
 208
 209                             # see if we have an initializer
 210                             m = self.siphon_initializer.search(str)
 211                             if m is not None:
 212                                 # count the braces on this line
 213                                 (count, index) = \
 214                                     self.count_braces(str[m.start():])
 215                                 siphon[2] = count
 216                                 # TODO - it's possible we have the
 217                                 # initializer all on the first line
 218                                 # we should check for it, but also
 219                                 # account for the possibility that
 220                                 # the open brace is on the next line
 221                                 #if count == 0:
 222                                 #    # braces balanced
 223                                 #    close_siphon = siphon
 224                                 #    siphon = None
 225                             else:
 226                                 # no initializer: close the siphon right now
 227                                 close_siphon = siphon
 228                                 siphon = None
 229                 else:
 230                     # See if we should end the siphon here - do we have
 231                     # balanced braces?
 232                     (count, index) = self.count_braces(str,
 233                             count=siphon[2], found=True)
 234                     if count == 0:
 235                         # braces balanced - add the substring and
 236                         # close the siphon
 237                         siphon[1] += str[:index+1] + ";\n"
 238                         close_siphon = siphon
 239                         siphon = None
 240                     else:
 241                         # add the whole string, move on
 242                         siphon[2] = count
 243                         siphon[1] += str + "\n"
 244
 245                 if close_siphon is not None:
 246                     # Write the siphoned contents to the right place
 247                     siphon_name = close_siphon[0]
 248
 249                     # Copy directives for the file
 250                     details = {}
 251                     for key in directives:
 252                         if ":" in key:
 253                             (sn, label) = key.split(":")
 254                             if sn == siphon_name:
 255                                 details[label] = directives[key]
 256                         else:
 257                             details[key] = directives[key]
 258
 259                     # Copy details for this block
 260                     details['file'] = filename
 261                     details['directory'] = directory
 262                     details['line_start'] = siphon_line
 263                     details['line_end'] = line_num
 264                     details['siphon_block'] = siphon_block.strip()
 265                     details["block"] = close_siphon[1]
 266
 267                     # Store the item
 268                     self.output[siphon_name]['items'].append(details)
 269
 270                     # All done
 271                     close_siphon = None
 272                     siphon_block = ""
 273
 274             # Update globals
 275             for key in directives.keys():
 276                 if ':' not in key:
 277                     continue
 278
 279                 if filename.endswith("/dir.dox"):
 280                     # very special! use the parent directory name
 281                     l = directory
 282                 else:
 283                     l = filename
 284
 285                 (sn, label) = key.split(":")
 286
 287                 if sn not in self.output:
 288                     self.output[sn] = {}
 289                 if 'global' not in self.output[sn]:
 290                     self.output[sn]['global'] = {}
 291                 if l not in self.output[sn]['global']:
 292                     self.output[sn]['global'][l] = {}
 293
 294                 self.output[sn]['global'][l][label] = directives[key]
 295
 296     def deliver(self):
 297         # Write out the data
 298         for siphon in self.output.keys():
 299             self.log.info("Saving siphon data %s." % siphon)
 300             s = self.output[siphon]
 301             with open(s['file'], "a") as fp:
 302                 json.dump(s, fp,
 303                     separators=(',', ': '), indent=4, sort_keys=True)
 304