ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor pre-Doxy siphon scripts; VPP-396

Browse Source 
- Modularize the code to make the Siphon process easier to
  maintain.
- Move much of the output rendering into Jinja2 templates.
- Add syscfg siphon type for startup config documentation.
- Add sample syscfg documentation.
- Add clicfg and syscfg preamble docs, adapted from their wiki pages.
- Fix sorting of CLI items across multiple directories.

Change-Id: Ib8288fe005adfea68ceed75a38ff8eba25d3cc79
Signed-off-by: Chris Luke <chrisy@flirble.org>
This commit is contained in:
Chris Luke
2016-09-12 08:55:13 -04:00
parent ce64b8e5b2
commit 90f52bf990
28 changed files with 1580 additions and 660 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
doxygen/siphon/__init__.py Normal file
View File

@@ -0,0 +1,24 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Siphon classes
import generate
import generate_clicmd
import generate_syscfg
import parsers
import process
import process_clicmd
import process_syscfg

304
doxygen/siphon/generate.py Normal file
View File

File diff suppressed because it is too large Load Diff

22
doxygen/siphon/generate_clicmd.py Normal file
View File

@@ -0,0 +1,22 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import generate, re
# Register our regexp
generate.siphon_patterns.append((
re.compile("(?P<m>VLIB_CLI_COMMAND)\s*"
"[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"),
"clicmd"
))

22
doxygen/siphon/generate_syscfg.py Normal file
View File

@@ -0,0 +1,22 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import generate, re
# Register our regexp
generate.siphon_patterns.append((
re.compile("(?P<m>VLIB_CONFIG_FUNCTION)\s*"
'[(](?P<fn>[a-zA-Z0-9_]+)\s*,\s*"(?P<name>[^"]*)"[)]'),
"syscfg"
))

149
doxygen/siphon/parsers.py Normal file
View File

@@ -0,0 +1,149 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import cgi, pyparsing as pp
# Some useful primitives
ident = pp.Word(pp.alphas + "_", pp.alphas + pp.nums + "_")
intNum = pp.Word(pp.nums)
hexNum = pp.Literal("0x") + pp.Word(pp.hexnums)
octalNum = pp.Literal("0") + pp.Word("01234567")
integer = (hexNum | octalNum | intNum) + \
pp.Optional(pp.Literal("ULL") | pp.Literal("LL") | pp.Literal("L"))
floatNum = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?') + pp.Optional(pp.Literal("f"))
char = pp.Literal("'") + pp.Word(pp.printables, exact=1) + pp.Literal("'")
arrayIndex = integer | ident
lbracket = pp.Literal("(").suppress()
rbracket = pp.Literal(")").suppress()
lbrace = pp.Literal("{").suppress()
rbrace = pp.Literal("}").suppress()
comma = pp.Literal(",").suppress()
equals = pp.Literal("=").suppress()
dot = pp.Literal(".").suppress()
semicolon = pp.Literal(";").suppress()
# initializer := { [member = ] (variable | expression | { initializer } ) }
typeName = ident
varName = ident
typeSpec = pp.Optional("unsigned") + \
pp.oneOf("int long short float double char u8 i8 void") + \
pp.Optional(pp.Word("*"), default="")
typeCast = pp.Combine( "(" + ( typeSpec | typeName ) + ")" ).suppress()
string = pp.Combine(pp.OneOrMore(pp.QuotedString(quoteChar='"',
escChar='\\', multiline=True)), adjacent=False)
literal = pp.Optional(typeCast) + (integer | floatNum | char | string)
var = pp.Combine(pp.Optional(typeCast) + varName +
pp.Optional("[" + arrayIndex + "]"))
# This could be more complete, but suffices for our uses
expr = (literal | var)
"""Parse and render a block of text into a Python dictionary."""
class Parser(object):
"""Compiled PyParsing BNF"""
_parser = None
def __init__(self):
super(Parser, self).__init__()
self._parser = self.BNF()
def BNF(self):
raise NotImplementedError
def item(self, item):
raise NotImplementedError
def parse(self, input):
item = self._parser.parseString(input).asList()
return self.item(item)
"""Parser for function-like macros - without the closing semi-colon."""
class ParserFunctionMacro(Parser):
def BNF(self):
# VLIB_CONFIG_FUNCTION (unix_config, "unix")
macroName = ident
params = pp.Group(pp.ZeroOrMore(expr + comma) + expr)
macroParams = lbracket + params + rbracket
return macroName + macroParams
def item(self, item):
r = {
"macro": item[0],
"name": item[1][1],
"function": item[1][0],
}
return r
"""Parser for function-like macros with a closing semi-colon."""
class ParseFunctionMacroStmt(ParserFunctionMacro):
def BNF(self):
# VLIB_CONFIG_FUNCTION (unix_config, "unix");
function_macro = super(ParseFunctionMacroStmt, self).BNF()
mi = function_macro + semicolon
mi.ignore(pp.cppStyleComment)
return mi
"""
Parser for our struct initializers which are composed from a
function-like macro, equals sign, and then a normal C struct initalizer
block.
"""
class MacroInitializer(ParserFunctionMacro):
def BNF(self):
# VLIB_CLI_COMMAND (show_sr_tunnel_command, static) = {
# .path = "show sr tunnel",
# .short_help = "show sr tunnel [name <sr-tunnel-name>]",
# .function = show_sr_tunnel_fn,
# };
cs = pp.Forward()
member = pp.Combine(dot + varName + pp.Optional("[" + arrayIndex + "]"),
adjacent=False)
value = (expr | cs)
entry = pp.Group(pp.Optional(member + equals, default="") + value)
entries = (pp.ZeroOrMore(entry + comma) + entry + pp.Optional(comma)) | \
(pp.ZeroOrMore(entry + comma))
cs << (lbrace + entries + rbrace)
macroName = ident
params = pp.Group(pp.ZeroOrMore(expr + comma) + expr)
macroParams = lbracket + params + rbracket
function_macro = super(MacroInitializer, self).BNF()
mi = function_macro + equals + pp.Group(cs) + semicolon
mi.ignore(pp.cppStyleComment)
return mi
def item(self, item):
r = {
"macro": item[0],
"name": item[1][0],
"params": item[2],
"value": {},
}
for param in item[2]:
r["value"][param[0]] = cgi.escape(param[1])
return r

271
doxygen/siphon/process.py Normal file
View File

@@ -0,0 +1,271 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Generation template class
import logging, os,sys, cgi, json, jinja2, HTMLParser
# Classes register themselves in this dictionary
"""Mapping of known processors to their classes"""
siphons = {}
"""Generate rendered output for siphoned data."""
class Siphon(object):
# Set by subclasses
"""Our siphon name"""
name = None
# Set by subclasses
"""Name of an identifier used by this siphon"""
identifier = None
# Set by subclasses
"""The pyparsing object to use to parse with"""
_parser = None
"""The input data"""
_cmds = None
"""Group key to (directory,file) mapping"""
_group = None
"""Logging handler"""
log = None
"""Directory to look for siphon rendering templates"""
template_directory = None
"""Template environment, if we're using templates"""
_tplenv = None
def __init__(self, template_directory=None):
super(Siphon, self).__init__()
self.log = logging.getLogger("siphon.process.%s" % self.name)
if template_directory is not None:
self.template_directory = template_directory
searchpath = [
template_directory + "/" + self.name,
template_directory + "/" + "default",
]
loader = jinja2.FileSystemLoader(searchpath=searchpath)
self._tplenv = jinja2.Environment(
loader=loader,
trim_blocks=True,
keep_trailing_newline=True)
# Convenience, get a reference to the internal escape and
# unescape methods in cgi and HTMLParser. These then become
# available to templates to use, if needed.
self._h = HTMLParser.HTMLParser()
self.escape = cgi.escape
self.unescape = self._h.unescape
# Output renderers
"""Returns an object to be used as the sorting key in the item index."""
def index_sort_key(self, group):
return group
"""Returns a string to use as the header at the top of the item index."""
def index_header(self):
return self.template("index_header")
"""Returns the string fragment to use for each section in the item
index."""
def index_section(self, group):
return self.template("index_section", group=group)
"""Returns the string fragment to use for each entry in the item index."""
def index_entry(self, meta, item):
return self.template("index_entry", meta=meta, item=item)
"""Returns an object, typically a string, to be used as the sorting key
for items within a section."""
def item_sort_key(self, item):
return item['name']
"""Returns a key for grouping items together."""
def group_key(self, directory, file, macro, name):
_global = self._cmds['_global']
if file in _global and 'group_label' in _global[file]:
self._group[file] = (directory, file)
return file
self._group[directory] = (directory, None)
return directory
"""Returns a key for identifying items within a grouping."""
def item_key(self, directory, file, macro, name):
return name
"""Returns a string to use as the header when rendering the item."""
def item_header(self, group):
return self.template("item_header", group=group)
"""Returns a string to use as the body when rendering the item."""
def item_format(self, meta, item):
return self.template("item_format", meta=meta, item=item)
"""Returns a string to use as the label for the page reference."""
def page_label(self, group):
return "_".join((
self.name,
self.sanitize_label(group)
))
"""Returns a title to use for a page."""
def page_title(self, group):
_global = self._cmds['_global']
(directory, file) = self._group[group]
if file and file in _global and 'group_label' in _global[file]:
return _global[file]['group_label']
if directory in _global and 'group_label' in _global[directory]:
return _global[directory]['group_label']
return directory
"""Returns a string to use as the label for the section reference."""
def item_label(self, group, item):
return "__".join((
self.name,
item
))
"""Label sanitizer; for creating Doxygen references"""
def sanitize_label(self, value):
return value.replace(" ", "_") \
.replace("/", "_") \
.replace(".", "_")
"""Template processor"""
def template(self, name, **kwargs):
tpl = self._tplenv.get_template(name + ".md")
return tpl.render(
this=self,
**kwargs)
# Processing methods
"""Parse the input file into a more usable dictionary structure."""
def load_json(self, files):
self._cmds = {}
self._group = {}
line_num = 0
line_start = 0
for filename in files:
filename = os.path.relpath(filename)
self.log.info("Parsing items in file \"%s\"." % filename)
data = None
with open(filename, "r") as fd:
data = json.load(fd)
self._cmds['_global'] = data['global']
# iterate the items loaded and regroup it
for item in data["items"]:
try:
o = self._parser.parse(item['block'])
except:
self.log.error("Exception parsing item: %s\n%s" \
% (json.dumps(item, separators=(',', ': '),
indent=4),
item['block']))
raise
# Augment the item with metadata
o["meta"] = {}
for key in item:
if key == 'block':
continue
o['meta'][key] = item[key]
# Load some interesting fields
directory = item['directory']
file = item['file']
macro = o["macro"]
name = o["name"]
# Generate keys to group items by
group_key = self.group_key(directory, file, macro, name)
item_key = self.item_key(directory, file, macro, name)
if group_key not in self._cmds:
self._cmds[group_key] = {}
self._cmds[group_key][item_key] = o
"""Iterate over the input data, calling render methods to generate the
output."""
def process(self, out=None):
if out is None:
out = sys.stdout
# Accumulated body contents
contents = ""
# Write the header for this siphon type
out.write(self.index_header())
# Sort key helper for the index
def group_sort_key(group):
return self.index_sort_key(group)
# Iterate the dictionary and process it
for group in sorted(self._cmds.keys(), key=group_sort_key):
if group.startswith('_'):
continue
self.log.info("Processing items in group \"%s\" (%s)." % \
(group, group_sort_key(group)))
# Generate the section index entry (write it now)
out.write(self.index_section(group))
# Generate the item header (save for later)
contents += self.item_header(group)
def item_sort_key(key):
return self.item_sort_key(self._cmds[group][key])
for key in sorted(self._cmds[group].keys(), key=item_sort_key):
self.log.debug("--- Processing key \"%s\" (%s)." % \
(key, item_sort_key(key)))
o = self._cmds[group][key]
meta = {
"directory": o['meta']['directory'],
"file": o['meta']['file'],
"macro": o['macro'],
"key": key,
"label": self.item_label(group, key),
}
# Generate the index entry for the item (write it now)
out.write(self.index_entry(meta, o))
# Generate the item itself (save for later)
contents += self.item_format(meta, o)
# Deliver the accumulated body output
out.write(contents)

56
doxygen/siphon/process_clicmd.py Normal file
View File

@@ -0,0 +1,56 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Generate clicmd formatted output
import process, parsers
class SiphonCLICMD(process.Siphon):
name = "clicmd"
identifier = "VLIB_CLI_COMMAND"
def __init__(self, *args, **kwargs):
super(SiphonCLICMD, self).__init__(*args, **kwargs)
self._parser = parsers.MacroInitializer()
# Output renderers
def index_sort_key(self, group):
_global = self._cmds['_global']
if group not in self._group:
return group
(directory, file) = self._group[group]
if file in _global and 'group_label' in _global[file]:
return _global[file]['group_label']
if directory in _global and 'group_label' in _global[directory]:
return _global[directory]['group_label']
return group
def item_sort_key(self, item):
return item['value']['path']
def item_label(self, group, item):
return "_".join((
self.name,
self.sanitize_label(self._cmds[group][item]['value']['path'])
))
# Register our processor
process.siphons["clicmd"] = SiphonCLICMD

30
doxygen/siphon/process_syscfg.py Normal file
View File

@@ -0,0 +1,30 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Generate syscfg formatted output
import process, parsers
class SiphonSYSCFG(process.Siphon):
name = "syscfg"
identifier = "VLIB_CONFIG_FUNCTION"
def __init__(self, *args, **kwargs):
super(SiphonSYSCFG, self).__init__(*args, **kwargs)
self._parser = parsers.ParseFunctionMacroStmt()
# Register our processor
process.siphons["syscfg"] = SiphonSYSCFG