docs/_scripts/siphon/generate.py

   1 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at:
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Generate .siphon source fragments for later processing
  16
  17 import json
  18 import logging
  19 import os
  20 import re
  21
  22 """List of (regexp, siphon_name) tuples for matching the start of C
  23    initializer blocks in source files. Each siphon class registers
  24    themselves on this list."""
  25 siphon_patterns = []
  26
  27
  28 class Generate(object):
  29     """Matches a siphon comment block start"""
  30
  31     siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$")
  32
  33     """Matches a siphon comment block stop"""
  34     siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$")
  35
  36     """Siphon block directive delimiter"""
  37     siphon_block_delimiter = "%%"
  38
  39     """Matches a siphon block directive such as
  40        '%clicmd:group_label Debug CLI%'"""
  41     siphon_block_directive = re.compile(
  42         "(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)"
  43         % (siphon_block_delimiter, siphon_block_delimiter)
  44     )
  45
  46     """Matches the start of an initializer block"""
  47     siphon_initializer = re.compile("\s*=")
  48
  49     """Collated output for each siphon"""
  50     output = None
  51
  52     """Directory prefix to strip from input filenames to keep things tidy."""
  53     input_prefix = None
  54
  55     """List of known siphons"""
  56     known_siphons = None
  57
  58     """Logging handler"""
  59     log = None
  60
  61     def __init__(self, output_directory, input_prefix):
  62         super(Generate, self).__init__()
  63         self.log = logging.getLogger("siphon.generate")
  64
  65         # Build a list of known siphons
  66         self.known_siphons = []
  67         for item in siphon_patterns:
  68             siphon = item[1]
  69             if siphon not in self.known_siphons:
  70                 self.known_siphons.append(siphon)
  71
  72         # Setup information for siphons we know about
  73         self.output = {}
  74         for siphon in self.known_siphons:
  75             self.output[siphon] = {
  76                 "file": "%s/%s.siphon" % (output_directory, siphon),
  77                 "global": {},
  78                 "items": [],
  79             }
  80
  81         self.input_prefix = input_prefix
  82
  83     """
  84     count open and close braces in str
  85     return (0, index) when braces were found and count becomes 0.
  86     index indicates the position at which the last closing brace was
  87     found.
  88     return (-1, -1) if a closing brace is found before any opening one.
  89     return (count, -1) if not all opening braces are closed, count is the
  90     current depth
  91     """
  92
  93     def count_braces(self, str, count=0, found=False):
  94         for index in range(0, len(str)):
  95             if str[index] == "{":
  96                 count += 1
  97                 found = True
  98             elif str[index] == "}":
  99                 if count == 0:
 100                     # means we never found an open brace
 101                     return (-1, -1)
 102                 count -= 1
 103
 104             if count == 0 and found:
 105                 return (count, index)
 106
 107         return (count, -1)
 108
 109     def parse(self, filename):
 110         # Strip the current directory off the start of the
 111         # filename for brevity
 112         if filename[0 : len(self.input_prefix)] == self.input_prefix:
 113             filename = filename[len(self.input_prefix) :]
 114             if filename[0] == "/":
 115                 filename = filename[1:]
 116
 117         # Work out the abbreviated directory name
 118         directory = os.path.dirname(filename)
 119         if directory[0:2] == "./":
 120             directory = directory[2:]
 121         elif directory[0 : len(self.input_prefix)] == self.input_prefix:
 122             directory = directory[len(self.input_prefix) :]
 123         if directory[0] == "/":
 124             directory = directory[1:]
 125
 126         # Open the file and explore its contents...
 127         self.log.info("Siphoning from %s." % filename)
 128         directives = {}
 129         with open(filename) as fd:
 130             siphon = None
 131             close_siphon = None
 132             siphon_block = ""
 133             in_block = False
 134             line_num = 0
 135             siphon_line = 0
 136
 137             for line in fd:
 138                 line_num += 1
 139                 str = line[:-1]  # filter \n
 140
 141                 """See if there is a block directive and if so extract it"""
 142
 143                 def process_block_directive(str, directives):
 144                     m = self.siphon_block_directive.search(str)
 145                     if m is not None:
 146                         k = m.group(2)
 147                         v = m.group(3).strip()
 148                         directives[k] = v
 149                         # Return only the parts we did not match
 150                         return str[0 : m.start(1)] + str[m.end(4) :]
 151
 152                     return str
 153
 154                 def process_block_prefix(str):
 155                     if str.startswith(" * "):
 156                         str = str[3:]
 157                     elif str == " *":
 158                         str = ""
 159                     return str
 160
 161                 if not in_block:
 162                     # See if the line contains the start of a siphon doc block
 163                     m = self.siphon_block_start.search(str)
 164                     if m is not None:
 165                         in_block = True
 166                         t = m.group(1)
 167
 168                         # Now check if the block closes on the same line
 169                         m = self.siphon_block_stop.search(t)
 170                         if m is not None:
 171                             t = m.group(1)
 172                             in_block = False
 173
 174                         # Check for directives
 175                         t = process_block_directive(t, directives)
 176
 177                         # Filter for normal comment prefixes
 178                         t = process_block_prefix(t)
 179
 180                         # Add what is left
 181                         siphon_block += t
 182
 183                         # Skip to next line
 184                         continue
 185
 186                 else:
 187                     # Check to see if we have an end block marker
 188                     m = self.siphon_block_stop.search(str)
 189                     if m is not None:
 190                         in_block = False
 191                         t = m.group(1)
 192                     else:
 193                         t = str
 194
 195                     # Check for directives
 196                     t = process_block_directive(t, directives)
 197
 198                     # Filter for normal comment prefixes
 199                     t = process_block_prefix(t)
 200
 201                     # Add what is left
 202                     siphon_block += t + "\n"
 203
 204                     # Skip to next line
 205                     continue
 206
 207                 if siphon is None:
 208                     # Look for blocks we need to siphon
 209                     for p in siphon_patterns:
 210                         if p[0].match(str):
 211                             siphon = [p[1], str + "\n", 0]
 212                             siphon_line = line_num
 213
 214                             # see if we have an initializer
 215                             m = self.siphon_initializer.search(str)
 216                             if m is not None:
 217                                 # count the braces on this line
 218                                 (count, index) = self.count_braces(str[m.start() :])
 219                                 siphon[2] = count
 220                                 # TODO - it's possible we have the
 221                                 # initializer all on the first line
 222                                 # we should check for it, but also
 223                                 # account for the possibility that
 224                                 # the open brace is on the next line
 225                                 # if count == 0:
 226                                 #    # braces balanced
 227                                 #    close_siphon = siphon
 228                                 #    siphon = None
 229                             else:
 230                                 # no initializer: close the siphon right now
 231                                 close_siphon = siphon
 232                                 siphon = None
 233                 else:
 234                     # See if we should end the siphon here - do we have
 235                     # balanced braces?
 236                     (count, index) = self.count_braces(str, count=siphon[2], found=True)
 237                     if count == 0:
 238                         # braces balanced - add the substring and
 239                         # close the siphon
 240                         siphon[1] += str[: index + 1] + ";\n"
 241                         close_siphon = siphon
 242                         siphon = None
 243                     else:
 244                         # add the whole string, move on
 245                         siphon[2] = count
 246                         siphon[1] += str + "\n"
 247
 248                 if close_siphon is not None:
 249                     # Write the siphoned contents to the right place
 250                     siphon_name = close_siphon[0]
 251
 252                     # Copy directives for the file
 253                     details = {}
 254                     for key in directives:
 255                         if ":" in key:
 256                             (sn, label) = key.split(":")
 257                             if sn == siphon_name:
 258                                 details[label] = directives[key]
 259                         else:
 260                             details[key] = directives[key]
 261
 262                     # Copy details for this block
 263                     details["file"] = filename
 264                     details["directory"] = directory
 265                     details["line_start"] = siphon_line
 266                     details["line_end"] = line_num
 267                     details["siphon_block"] = siphon_block.strip()
 268                     details["block"] = close_siphon[1]
 269
 270                     # Store the item
 271                     self.output[siphon_name]["items"].append(details)
 272
 273                     # All done
 274                     close_siphon = None
 275                     siphon_block = ""
 276
 277             # Update globals
 278             for key in directives.keys():
 279                 if ":" not in key:
 280                     continue
 281
 282                 if filename.endswith("/dir.dox"):
 283                     # very special! use the parent directory name
 284                     l = directory
 285                 else:
 286                     l = filename
 287
 288                 (sn, label) = key.split(":")
 289
 290                 if sn not in self.output:
 291                     self.output[sn] = {}
 292                 if "global" not in self.output[sn]:
 293                     self.output[sn]["global"] = {}
 294                 if l not in self.output[sn]["global"]:
 295                     self.output[sn]["global"][l] = {}
 296
 297                 self.output[sn]["global"][l][label] = directives[key]
 298
 299     def deliver(self):
 300         # Write out the data
 301         for siphon in self.output.keys():
 302             self.log.info("Saving siphon data %s." % siphon)
 303             s = self.output[siphon]
 304             with open(s["file"], "a") as fp:
 305                 json.dump(s, fp, separators=(",", ": "), indent=4, sort_keys=True)