ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor pre-Doxy siphon scripts; VPP-396

Browse Source 
- Modularize the code to make the Siphon process easier to
  maintain.
- Move much of the output rendering into Jinja2 templates.
- Add syscfg siphon type for startup config documentation.
- Add sample syscfg documentation.
- Add clicfg and syscfg preamble docs, adapted from their wiki pages.
- Fix sorting of CLI items across multiple directories.

Change-Id: Ib8288fe005adfea68ceed75a38ff8eba25d3cc79
Signed-off-by: Chris Luke <chrisy@flirble.org>
This commit is contained in:
Chris Luke
2016-09-12 08:55:13 -04:00
parent ce64b8e5b2
commit 90f52bf990
28 changed files with 1580 additions and 660 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doxygen/Makefile
+36 -12
View File
@@ -16,14 +16,18 @@
# Build the documentation
#
# Default target
.PHONY: all
all: doxygen
# These should be passed in by the root Makefile
WS_ROOT ?= $(CURDIR)/..
BR ?= $(WS_ROOT)/build-root
OS_ID ?= $(shell grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
# Package dependencies
DOC_DEB_DEPENDS = doxygen graphviz python-pyparsing
DOC_RPM_DEPENDS = doxygen graphviz pyparsing
DOC_DEB_DEPENDS = doxygen graphviz python-pyparsing python-jinja2
DOC_RPM_DEPENDS = doxygen graphviz pyparsing python-jinja2
# Doxygen configuration and our utility scripts
DOXY_DIR ?= $(WS_ROOT)/doxygen
@@ -104,19 +108,22 @@ SIPHON_OUTPUT ?= $(DOXY_OUTPUT)/siphon_docs
EXTRA_DOXY_INPUT += $(SIPHON_OUTPUT)
# All the siphon types we know about
SIPHONS ?= clicmd
SIPHONS ?= clicmd syscfg
SIPHON_FILES = $(addprefix $(SIPHON_INPUT)/,$(addsuffix .siphon,$(SIPHONS)))
SIPHON_DOCS = $(addprefix $(SIPHON_OUTPUT)/,$(addsuffix .md,$(SIPHONS)))
$(BR)/.doxygen-bootstrap.ok:
$(BR)/.doxygen-bootstrap.ok: Makefile
@echo "Checking whether dependencies for Doxygen are installed..."
ifeq ($(OS_ID),ubuntu)
@set -e; inst=; \
for i in $(DOC_DEB_DEPENDS); do \
dpkg-query --show $$i >/dev/null 2>&1 || inst="$$inst $$i"; \
done; \
if [ "$$inst" ]; then sudo apt-get $(CONFIRM) $(FORCE) install $$inst; fi
if [ "$$inst" ]; then \
sudo apt-get update; \
sudo apt-get $(CONFIRM) $(FORCE) install $$inst; \
fi
@if [ ! -s /usr/lib/graphviz/config6a ]; then \
echo "Rebuidlding system Graphviz configuration."; \
sudo dot -c; \
@@ -145,8 +152,12 @@ $(BR)/.doxygen-siphon.dep: Makefile
# Include the source -> siphon dependencies
-include $(BR)/.doxygen-siphon.dep
# Generate .siphon files that contain fragments of source file that
# relate to the siphons we support.
.NOTPARALLEL: $(SIPHON_FILES)
$(SIPHON_FILES): $(DOXY_DIR)/siphon_generate.py $(BR)/.doxygen-bootstrap.ok
$(SIPHON_FILES): $(BR)/.doxygen-bootstrap.ok \
$(DOXY_DIR)/siphon-generate \
$(wildcard $(DOXY_DIR)/siphon/*.py)
@rm -rf "$(SIPHON_INPUT)" "$(SIPHON_OUTPUT)"
@mkdir -p "$(SIPHON_INPUT)" "$(SIPHON_OUTPUT)"
@touch $(SIPHON_INPUT)/files
@@ -159,23 +170,33 @@ $(SIPHON_FILES): $(DOXY_DIR)/siphon_generate.py $(BR)/.doxygen-bootstrap.ok
>> $(SIPHON_INPUT)/files; \
done
@echo "Generating siphons..."
@set -e; cd "$(WS_ROOT)"; $(DOXY_DIR)/siphon_generate.py \
@set -e; \
cd "$(WS_ROOT)"; \
$(DOXY_DIR)/siphon-generate \
--output="$(SIPHON_INPUT)" \
"@$(SIPHON_INPUT)/files"
# Process the .siphon source fragments and render them into doxygen flavored
# markdown documentation
.DELETE_ON_ERROR: $(SIPHON_DOCS)
$(SIPHON_OUTPUT)/%.md: $(SIPHON_INPUT)/%.siphon $(DOXY_DIR)/siphon_process.py
$(SIPHON_OUTPUT)/%.md: $(SIPHON_INPUT)/%.siphon \
$(DOXY_DIR)/siphon-process \
$(wildcard $(DOXY_DIR)/siphon/*.py) \
$(wildcard $(DOXY_DIR)/siphon_templates/*/*.md)
@echo "Processing siphon from $(notdir $<)..."
@set -e; cd "$(WS_ROOT)"; \
$(DOXY_DIR)/siphon_process.py --type=$(basename $(notdir $<)) \
--output="$(SIPHON_OUTPUT)" $< > $@
@set -e; \
cd "$(WS_ROOT)"; \
$(DOXY_DIR)/siphon-process \
--type=$(basename $(notdir $<)) \
--output="$@" \
"$<"
# This target can be used just to generate the siphoned docs
.PHONY: doxygen-siphon
doxygen-siphon: $(SIPHON_DOCS)
# Generate the doxygen docs
.PHONY: doxygen
doxygen: $(SIPHON_DOCS)
@mkdir -p "$(DOXY_OUTPUT)"
@echo "Running Doxygen..."
@@ -189,6 +210,9 @@ doxygen: $(SIPHON_DOCS)
VERSION="`git describe --tags --dirty`" \
doxygen $(DOXY_DIR)/doxygen.cfg
.PHONY: wipe-doxygen
wipe-doxygen:
rm -rf "$(BR)/docs" "$(BR)/.doxygen-siphon.d"
.PHONY: clean
clean: wipe-doxygen
doxygen/dir.dox.sample
+1
View File
@@ -27,3 +27,4 @@ This looks like a C file but it is not part of the build; it is purely
for documentation.
*/
/*? %%clicmd:group_label CLI section description%% ?*/
/*? %%syscfg:group_label Startup config section description%% ?*/
doxygen/doxygen.cfg
+3
View File
@@ -244,6 +244,9 @@ ALIASES += "cliexcmd{1}=@clistart<b>vpp# <em>\1</em></b>@cliend"
ALIASES += "cliexstart{1}=@cliexcmd{\1}@clistart"
ALIASES += "cliexend=@cliend"
## Formatting for config directives
ALIASES += "cfgcmd{2}=@par <code><pre>\1 \2</pre></code>"
ALIASES += "cfgcmd{1}=@par <code><pre>\1</pre></code>"
# This tag can be used to specify a number of word-keyword mappings (TCL only).
# A mapping has the form "name=value". For example adding "class=itcl::class"
doxygen/siphon-generate
Executable
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env python
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Looks for preprocessor macros with struct initializers and siphons them
# off into another file for later parsing; ostensibly to generate
# documentation from struct initializer data.
import os, sys, argparse, logging
import siphon
DEFAULT_LOGFILE = None
DEFAULT_LOGLEVEL = "info"
DEFAULT_OUTPUT = "build-root/docs/siphons"
DEFAULT_PREFIX = os.getcwd()
ap = argparse.ArgumentParser()
ap.add_argument("--log-file", default=DEFAULT_LOGFILE,
help="Log file [%s]" % DEFAULT_LOGFILE)
ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL,
choices=["debug", "info", "warning", "error", "critical"],
help="Logging level [%s]" % DEFAULT_LOGLEVEL)
ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT,
help="Output directory for .siphon files [%s]" % DEFAULT_OUTPUT)
ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX,
help="Prefix to strip from input pathnames [%s]" % DEFAULT_PREFIX)
ap.add_argument("input", nargs='+', metavar="input_file",
help="Input C source files")
args = ap.parse_args()
logging.basicConfig(filename=args.log_file,
level=getattr(logging, args.log_level.upper(), None))
log = logging.getLogger("siphon_generate")
generate = siphon.generate.Generate(output_directory=args.output,
input_prefix=args.input_prefix)
# Pre-process file names in case they indicate a file with
# a list of files
files = []
for filename in args.input:
if filename.startswith('@'):
with open(filename[1:], 'r') as fp:
lines = fp.readlines()
for line in lines:
file = line.strip()
if file not in files:
files.append(file)
lines = None
else:
if filename not in files:
files.append(filename)
# Iterate all the input files we've been given
for filename in files:
generate.parse(filename)
# Write the extracted data
generate.deliver()
# All done
doxygen/siphon-process
Executable
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env python
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Filter for .siphon files that are generated by other filters.
# The idea is to siphon off certain initializers so that we can better
# auto-document the contents of that initializer.
import os, sys, argparse, logging
import siphon
DEFAULT_LOGFILE = None
DEFAULT_LOGLEVEL = "info"
DEFAULT_SIPHON ="clicmd"
DEFAULT_OUTPUT = None
DEFAULT_TEMPLATES = os.path.dirname(__file__) + "/siphon_templates"
ap = argparse.ArgumentParser()
ap.add_argument("--log-file", default=DEFAULT_LOGFILE,
help="Log file [%s]" % DEFAULT_LOGFILE)
ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL,
choices=["debug", "info", "warning", "error", "critical"],
help="Logging level [%s]" % DEFAULT_LOGLEVEL)
ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON,
choices=siphon.process.siphons.keys(),
help="Siphon type to process [%s]" % DEFAULT_SIPHON)
ap.add_argument("--output", '-o', metavar="file", default=DEFAULT_OUTPUT,
help="Output file (uses stdout if not defined) [%s]" % DEFAULT_OUTPUT)
ap.add_argument("--templates", metavar="directory", default=DEFAULT_TEMPLATES,
help="Path to render templates directory [%s]" % DEFAULT_TEMPLATES)
ap.add_argument("input", nargs='+', metavar="input_file",
help="Input .siphon files")
args = ap.parse_args()
logging.basicConfig(filename=args.log_file,
level=getattr(logging, args.log_level.upper(), None))
log = logging.getLogger("siphon_process")
# Determine where to send the generated output
if args.output is None:
out = sys.stdout
else:
out = open(args.output, "w+")
# Get our processor
klass = siphon.process.siphons[args.type]
processor = klass(template_directory=args.templates)
# Load the input files
processor.load_json(args.input)
# Process the data
processor.process(out=out)
# All done
doxygen/siphon/__init__.py
+24
View File
@@ -0,0 +1,24 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Siphon classes
import generate
import generate_clicmd
import generate_syscfg
import parsers
import process
import process_clicmd
import process_syscfg
doxygen/siphon/generate.py
+304
View File
File diff suppressed because it is too large Load Diff
doxygen/siphon/generate_clicmd.py
+22
View File
@@ -0,0 +1,22 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import generate, re
# Register our regexp
generate.siphon_patterns.append((
re.compile("(?P<m>VLIB_CLI_COMMAND)\s*"
"[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"),
"clicmd"
))
doxygen/siphon/generate_syscfg.py
+22
View File
@@ -0,0 +1,22 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import generate, re
# Register our regexp
generate.siphon_patterns.append((
re.compile("(?P<m>VLIB_CONFIG_FUNCTION)\s*"
'[(](?P<fn>[a-zA-Z0-9_]+)\s*,\s*"(?P<name>[^"]*)"[)]'),
"syscfg"
))
doxygen/siphon/parsers.py
+149
View File
@@ -0,0 +1,149 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import cgi, pyparsing as pp
# Some useful primitives
ident = pp.Word(pp.alphas + "_", pp.alphas + pp.nums + "_")
intNum = pp.Word(pp.nums)
hexNum = pp.Literal("0x") + pp.Word(pp.hexnums)
octalNum = pp.Literal("0") + pp.Word("01234567")
integer = (hexNum | octalNum | intNum) + \
pp.Optional(pp.Literal("ULL") | pp.Literal("LL") | pp.Literal("L"))
floatNum = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?') + pp.Optional(pp.Literal("f"))
char = pp.Literal("'") + pp.Word(pp.printables, exact=1) + pp.Literal("'")
arrayIndex = integer | ident
lbracket = pp.Literal("(").suppress()
rbracket = pp.Literal(")").suppress()
lbrace = pp.Literal("{").suppress()
rbrace = pp.Literal("}").suppress()
comma = pp.Literal(",").suppress()
equals = pp.Literal("=").suppress()
dot = pp.Literal(".").suppress()
semicolon = pp.Literal(";").suppress()
# initializer := { [member = ] (variable | expression | { initializer } ) }
typeName = ident
varName = ident
typeSpec = pp.Optional("unsigned") + \
pp.oneOf("int long short float double char u8 i8 void") + \
pp.Optional(pp.Word("*"), default="")
typeCast = pp.Combine( "(" + ( typeSpec | typeName ) + ")" ).suppress()
string = pp.Combine(pp.OneOrMore(pp.QuotedString(quoteChar='"',
escChar='\\', multiline=True)), adjacent=False)
literal = pp.Optional(typeCast) + (integer | floatNum | char | string)
var = pp.Combine(pp.Optional(typeCast) + varName +
pp.Optional("[" + arrayIndex + "]"))
# This could be more complete, but suffices for our uses
expr = (literal | var)
"""Parse and render a block of text into a Python dictionary."""
class Parser(object):
"""Compiled PyParsing BNF"""
_parser = None
def __init__(self):
super(Parser, self).__init__()
self._parser = self.BNF()
def BNF(self):
raise NotImplementedError
def item(self, item):
raise NotImplementedError
def parse(self, input):
item = self._parser.parseString(input).asList()
return self.item(item)
"""Parser for function-like macros - without the closing semi-colon."""
class ParserFunctionMacro(Parser):
def BNF(self):
# VLIB_CONFIG_FUNCTION (unix_config, "unix")
macroName = ident
params = pp.Group(pp.ZeroOrMore(expr + comma) + expr)
macroParams = lbracket + params + rbracket
return macroName + macroParams
def item(self, item):
r = {
"macro": item[0],
"name": item[1][1],
"function": item[1][0],
}
return r
"""Parser for function-like macros with a closing semi-colon."""
class ParseFunctionMacroStmt(ParserFunctionMacro):
def BNF(self):
# VLIB_CONFIG_FUNCTION (unix_config, "unix");
function_macro = super(ParseFunctionMacroStmt, self).BNF()
mi = function_macro + semicolon
mi.ignore(pp.cppStyleComment)
return mi
"""
Parser for our struct initializers which are composed from a
function-like macro, equals sign, and then a normal C struct initalizer
block.
"""
class MacroInitializer(ParserFunctionMacro):
def BNF(self):
# VLIB_CLI_COMMAND (show_sr_tunnel_command, static) = {
# .path = "show sr tunnel",
# .short_help = "show sr tunnel [name <sr-tunnel-name>]",
# .function = show_sr_tunnel_fn,
# };
cs = pp.Forward()
member = pp.Combine(dot + varName + pp.Optional("[" + arrayIndex + "]"),
adjacent=False)
value = (expr | cs)
entry = pp.Group(pp.Optional(member + equals, default="") + value)
entries = (pp.ZeroOrMore(entry + comma) + entry + pp.Optional(comma)) | \
(pp.ZeroOrMore(entry + comma))
cs << (lbrace + entries + rbrace)
macroName = ident
params = pp.Group(pp.ZeroOrMore(expr + comma) + expr)
macroParams = lbracket + params + rbracket
function_macro = super(MacroInitializer, self).BNF()
mi = function_macro + equals + pp.Group(cs) + semicolon
mi.ignore(pp.cppStyleComment)
return mi
def item(self, item):
r = {
"macro": item[0],
"name": item[1][0],
"params": item[2],
"value": {},
}
for param in item[2]:
r["value"][param[0]] = cgi.escape(param[1])
return r
doxygen/siphon/process.py
+271
View File
@@ -0,0 +1,271 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Generation template class
import logging, os,sys, cgi, json, jinja2, HTMLParser
# Classes register themselves in this dictionary
"""Mapping of known processors to their classes"""
siphons = {}
"""Generate rendered output for siphoned data."""
class Siphon(object):
# Set by subclasses
"""Our siphon name"""
name = None
# Set by subclasses
"""Name of an identifier used by this siphon"""
identifier = None
# Set by subclasses
"""The pyparsing object to use to parse with"""
_parser = None
"""The input data"""
_cmds = None
"""Group key to (directory,file) mapping"""
_group = None
"""Logging handler"""
log = None
"""Directory to look for siphon rendering templates"""
template_directory = None
"""Template environment, if we're using templates"""
_tplenv = None
def __init__(self, template_directory=None):
super(Siphon, self).__init__()
self.log = logging.getLogger("siphon.process.%s" % self.name)
if template_directory is not None:
self.template_directory = template_directory
searchpath = [
template_directory + "/" + self.name,
template_directory + "/" + "default",
]
loader = jinja2.FileSystemLoader(searchpath=searchpath)
self._tplenv = jinja2.Environment(
loader=loader,
trim_blocks=True,
keep_trailing_newline=True)
# Convenience, get a reference to the internal escape and
# unescape methods in cgi and HTMLParser. These then become
# available to templates to use, if needed.
self._h = HTMLParser.HTMLParser()
self.escape = cgi.escape
self.unescape = self._h.unescape
# Output renderers
"""Returns an object to be used as the sorting key in the item index."""
def index_sort_key(self, group):
return group
"""Returns a string to use as the header at the top of the item index."""
def index_header(self):
return self.template("index_header")
"""Returns the string fragment to use for each section in the item
index."""
def index_section(self, group):
return self.template("index_section", group=group)
"""Returns the string fragment to use for each entry in the item index."""
def index_entry(self, meta, item):
return self.template("index_entry", meta=meta, item=item)
"""Returns an object, typically a string, to be used as the sorting key
for items within a section."""
def item_sort_key(self, item):
return item['name']
"""Returns a key for grouping items together."""
def group_key(self, directory, file, macro, name):
_global = self._cmds['_global']
if file in _global and 'group_label' in _global[file]:
self._group[file] = (directory, file)
return file
self._group[directory] = (directory, None)
return directory
"""Returns a key for identifying items within a grouping."""
def item_key(self, directory, file, macro, name):
return name
"""Returns a string to use as the header when rendering the item."""
def item_header(self, group):
return self.template("item_header", group=group)
"""Returns a string to use as the body when rendering the item."""
def item_format(self, meta, item):
return self.template("item_format", meta=meta, item=item)
"""Returns a string to use as the label for the page reference."""
def page_label(self, group):
return "_".join((
self.name,
self.sanitize_label(group)
))
"""Returns a title to use for a page."""
def page_title(self, group):
_global = self._cmds['_global']
(directory, file) = self._group[group]
if file and file in _global and 'group_label' in _global[file]:
return _global[file]['group_label']
if directory in _global and 'group_label' in _global[directory]:
return _global[directory]['group_label']
return directory
"""Returns a string to use as the label for the section reference."""
def item_label(self, group, item):
return "__".join((
self.name,
item
))
"""Label sanitizer; for creating Doxygen references"""
def sanitize_label(self, value):
return value.replace(" ", "_") \
.replace("/", "_") \
.replace(".", "_")
"""Template processor"""
def template(self, name, **kwargs):
tpl = self._tplenv.get_template(name + ".md")
return tpl.render(
this=self,
**kwargs)
# Processing methods
"""Parse the input file into a more usable dictionary structure."""
def load_json(self, files):
self._cmds = {}
self._group = {}
line_num = 0
line_start = 0
for filename in files:
filename = os.path.relpath(filename)
self.log.info("Parsing items in file \"%s\"." % filename)
data = None
with open(filename, "r") as fd:
data = json.load(fd)
self._cmds['_global'] = data['global']
# iterate the items loaded and regroup it
for item in data["items"]:
try:
o = self._parser.parse(item['block'])
except:
self.log.error("Exception parsing item: %s\n%s" \
% (json.dumps(item, separators=(',', ': '),
indent=4),
item['block']))
raise
# Augment the item with metadata
o["meta"] = {}
for key in item:
if key == 'block':
continue
o['meta'][key] = item[key]
# Load some interesting fields
directory = item['directory']
file = item['file']
macro = o["macro"]
name = o["name"]
# Generate keys to group items by
group_key = self.group_key(directory, file, macro, name)
item_key = self.item_key(directory, file, macro, name)
if group_key not in self._cmds:
self._cmds[group_key] = {}
self._cmds[group_key][item_key] = o
"""Iterate over the input data, calling render methods to generate the
output."""
def process(self, out=None):
if out is None:
out = sys.stdout
# Accumulated body contents
contents = ""
# Write the header for this siphon type
out.write(self.index_header())
# Sort key helper for the index
def group_sort_key(group):
return self.index_sort_key(group)
# Iterate the dictionary and process it
for group in sorted(self._cmds.keys(), key=group_sort_key):
if group.startswith('_'):
continue
self.log.info("Processing items in group \"%s\" (%s)." % \
(group, group_sort_key(group)))
# Generate the section index entry (write it now)
out.write(self.index_section(group))
# Generate the item header (save for later)
contents += self.item_header(group)
def item_sort_key(key):
return self.item_sort_key(self._cmds[group][key])
for key in sorted(self._cmds[group].keys(), key=item_sort_key):
self.log.debug("--- Processing key \"%s\" (%s)." % \
(key, item_sort_key(key)))
o = self._cmds[group][key]
meta = {
"directory": o['meta']['directory'],
"file": o['meta']['file'],
"macro": o['macro'],
"key": key,
"label": self.item_label(group, key),
}
# Generate the index entry for the item (write it now)
out.write(self.index_entry(meta, o))
# Generate the item itself (save for later)
contents += self.item_format(meta, o)
# Deliver the accumulated body output
out.write(contents)
doxygen/siphon/process_clicmd.py
+56
View File
@@ -0,0 +1,56 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Generate clicmd formatted output
import process, parsers
class SiphonCLICMD(process.Siphon):
name = "clicmd"
identifier = "VLIB_CLI_COMMAND"
def __init__(self, *args, **kwargs):
super(SiphonCLICMD, self).__init__(*args, **kwargs)
self._parser = parsers.MacroInitializer()
# Output renderers
def index_sort_key(self, group):
_global = self._cmds['_global']
if group not in self._group:
return group
(directory, file) = self._group[group]
if file in _global and 'group_label' in _global[file]:
return _global[file]['group_label']
if directory in _global and 'group_label' in _global[directory]:
return _global[directory]['group_label']
return group
def item_sort_key(self, item):
return item['value']['path']
def item_label(self, group, item):
return "_".join((
self.name,
self.sanitize_label(self._cmds[group][item]['value']['path'])
))
# Register our processor
process.siphons["clicmd"] = SiphonCLICMD
doxygen/siphon/process_syscfg.py
+30
View File
@@ -0,0 +1,30 @@
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Generate syscfg formatted output
import process, parsers
class SiphonSYSCFG(process.Siphon):
name = "syscfg"
identifier = "VLIB_CONFIG_FUNCTION"
def __init__(self, *args, **kwargs):
super(SiphonSYSCFG, self).__init__(*args, **kwargs)
self._parser = parsers.ParseFunctionMacroStmt()
# Register our processor
process.siphons["syscfg"] = SiphonSYSCFG
doxygen/siphon_generate.py
-322
View File
File diff suppressed because it is too large Load Diff
doxygen/siphon_process.py
-323
View File
File diff suppressed because it is too large Load Diff
doxygen/siphon_templates/clicmd/index_entry.md
+17
View File
@@ -0,0 +1,17 @@
{#
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#}
{% set v = item['value'] %}
{{ "* [%s](@ref %s)" % (v['path'], meta["label"]) }}
doxygen/siphon_templates/clicmd/index_header.md
+130
View File
@@ -0,0 +1,130 @@
{#
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#}
# Debug CLI {{'{#'}}clicmd}
The VPP network stack comes equipped with a set of commands that are useful
for debugging.
The easiest way to access the CLI (with proper permissions) is to use the
vppctl command:
```
sudo vppctl <cli-command>
```
The CLI parser matches static keyword strings, eventually invoking an action
function. Unambiguous partial keyword matching always occurs. The action
functions consume input until satisfied or until they fail. This model makes
for easy coding, but does not guarantee useful "help" output. It's up to the
CLI command writer to add useful help strings.
You can find the source code of CLI commands by searching for instances of the
@c VLIB_CLI_COMMAND macro in the code source files.
Please help maintain and improve this document to make and keep these commands
clear and useful!
@todo Document where to modify this CLI intro text.
## Debug and Telnet CLI
The debug CLI is enabled with the unix interactive parameter or startup
configuration option. This causes VPP to start without daemonizing and
presents a command line interface on the terminal where it is run.
The Telnet CLI is enabled with the `cli-listen localhost:5002` option which
will cause VPP to listen for TCP connections on the localhost address port
@c 5002. A Telnet client can then connect to this port (for example, `telnet
localhost 5002`) and will receive a command line prompt.
This configuration will enable both mechanisms:
```
unix {
interactive
cli-listen localhost:5002
}
```
The debug CLI can operate in line mode, which may be useful when running
inside an IDE like Emacs. This is enabled with the option
`unix cli-line-mode`. Several other options exist that alter how this
CLI works, see the @ref syscfg section for details.
The CLI starts with a banner graphic (which can be disabled) and a prompt. The
prompt will typically read `vpp` for a release version of VPP and `DBGvpp#`
for a development version with debugging enabled, for example:
_______ _ _ _____ ___
__/ __/ _ \ (_)__ | | / / _ \/ _ \
_/ _// // / / / _ \ | |/ / ___/ ___/
/_/ /____(_)_/\___/ |___/_/ /_/
vpp#
versus:
_______ _ _ _____ ___
__/ __/ _ \ (_)__ | | / / _ \/ _ \
_/ _// // / / / _ \ | |/ / ___/ ___/
/_/ /____(_)_/\___/ |___/_/ /_/
DBGvpp#
This prompt can be configured with the `unix cli-prompt` setting and the
banner is disabled with `unix cli-no-banner`.
## CLI features
The CLI has several editing features that make it easy to use.
- Cursor keys left/right will move the cursor within a command line;
typing will insert at the cursor; erase will erase at the cursor.
- Ctrl-left/right will search for the start of the next word to
the left or right.
- Home/end will jump the cursor to the start and end of the line.
- Cursor keys up/down and ^P/^N iterate through the command history
buffer. Lines from the history buffer may be edited. New commands
are added to the end of the buffer when executed; though
duplicates of the previous command are not added.
- ^U erases the line contents from the left of the cursor to the
start.
- ^K erases the contents from the cursor to the end.
- ^S/^R will search the command history forwards or in reverse for
a command; start typing for matches to auto complete.
- ^L will clear the screen (if supported by the terminal) and repaint
the prompt and any current line. The cursor position is also
retained.
- The CLI can be closed with the quit command. Alternatively, ^D on
an empty input line will also close the session. Closing the debug
session will also shutdown VPP.
Output that exceeds the length of a terminal page will be buffered, up to a
limit.
- Space or page-down displays the next page.
- Enter or down-arrow displays the next line.
- Page-up goes back a page.
- Up-arrow goes up a line.
- Home/end jump to the start/end of the buffered output.
- The key q quits the pager. Space and enter will also quit the
pager if the end of the buffer has been reached.
## Index of CLI commands
[TOC]
doxygen/siphon_templates/clicmd/item_format.md
+59
View File
@@ -0,0 +1,59 @@
{#
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#}
{% set v = item['value'] %}
{{ "@section %s %s" % (meta['label'], v['path']) }}
{% if 'short_help' in v %}
### Summary/usage
{% set str = v['short_help'] %}
{% set period = "." if str[-1] != "." else "" %}
{% set prefix = " " if "[" in str or "&lt;" in str or "|" in str else "" %}
{% set str = this.unescape(str) %}
{{ "%s%s%s" % (prefix, str, period) }}
{% endif %}
{% if 'long_help' in v %}
{# This is seldom used and will likely be deprecated #}
### Long help
{{ v['long_help'] }}
{%- endif %}
{% if 'siphon_block' in item['meta'] %}
{% set sb = item["meta"]["siphon_block"] %}
{% if sb %}
{# Extracted from the code in /*? ... ?*/ blocks #}
### Description
{{ sb }}
{% endif %}
{% endif %}
{% if "item" in meta or "function" in v %}
{# Gives some developer-useful linking #}
### Declaration and implementation
{% if "item" in meta %}
{{ "Declaration: @ref %s (@ref %s line %d)" %
(meta['item'], meta["file"], item["meta"]["line_start"]) }}
{% endif %}
{% if "function" in v %}
{{ "Implementation: @ref %s." % v["function"] }}
{% endif %}
{% endif %}
doxygen/siphon_templates/default/index_entry.md
+16
View File
@@ -0,0 +1,16 @@
{#
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#}
{{ "* [%s](@ref %s)" % (item["name"], meta["label"]) }}
doxygen/siphon_templates/default/index_section.md
+18
View File
@@ -0,0 +1,18 @@
{#
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#}
@subpage {{ this.page_label(group) }}
doxygen/siphon_templates/default/item_format.md
+16
View File
@@ -0,0 +1,16 @@
{#
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#}
{{ raise NotImplementedError }}
doxygen/siphon_templates/default/item_header.md
+18
View File
@@ -0,0 +1,18 @@
{#
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#}
{{ "@page %s %s" % (this.page_label(group), this.page_title(group)) }}
doxygen/siphon_templates/syscfg/index_header.md
+111
View File
@@ -0,0 +1,111 @@
{#
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#}
# Startup Configuration {{'{#'}}syscfg}
The VPP network stack comes with several configuration options that can be
provided either on the command line or in a configuration file.
Specific applications built on the stack have been known to require a dozen
arguments, depending on requirements. This section describes commonly-used
options and parameters.
You can find command-line argument parsers in the source code by searching for
instances of the `VLIB_CONFIG_FUNCTION` macro. The invocation
`VLIB_CONFIG_FUNCTION(foo_config, "foo")` will cause the function
`foo_config` to receive all the options and values supplied in a parameter
block named "`foo`", for example: `foo { arg1 arg2 arg3 ... }`.
@todo Tell the nice people where this document lives so that the might
help improve it!
## Command-line arguments
Parameters are grouped by a section name. When providing more than one
parameter to a section all parameters for that section must be wrapped in
curly braces.
```
/usr/bin/vpp unix { interactive cli-listen 127.0.0.1:5002 }
```
Which will produce output similar to this:
<startup diagnostic messages>
_______ _ _ _____ ___
__/ __/ _ \ (_)__ | | / / _ \/ _ \
_/ _// // / / / _ \ | |/ / ___/ ___/
/_/ /____(_)_/\___/ |___/_/ /_/
vpp# <start-typing>
When providing only one such parameter the braces are optional. For example,
the following command argument, `unix interactive` does not have braces:
```
/usr/bin/vpp unix interactive
```
The command line can be presented as a single string or as several; anything
given on the command line is concatenated with spaces into a single string
before parsing.
VPP applications must be able to locate their own executable images. The
simplest way to ensure this will work is to invoke a VPP application by giving
its absolute path; for example: `/usr/bin/vpp <options>`. At startup, VPP
applications parse through their own ELF-sections (primarily) to make lists
of init, configuration, and exit handlers.
When developing with VPP, in _gdb_ it's often sufficient to start an application
like this at the `(gdb)` prompt:
```
run unix interactive
```
## Configuration file
It is also possible to supply parameters in a startup configuration file the
path of which is provided to the VPP application on its command line.
The format of the configuration file is a simple text file with the same
content as the command line but with the benefit of being able to use newlines
to make the content easier to read. For example:
```
unix {
nodaemon
log /tmp/vpp.log
full-coredump
cli-listen localhost:5002
}
api-trace {
on
}
dpdk {
dev 0000:03:00.0
}
```
VPP is then instructed to load this file with the `-c` option:
```
/usr/bin/vpp -c /etc/vpp/startup.conf
```
## Index of startup command sections
[TOC]
doxygen/siphon_templates/syscfg/item_format.md
+42
View File
@@ -0,0 +1,42 @@
{#
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#}
{% set v = item['value'] %}
{{ "@section %s %s" % (meta['label'], item['name']) }}
{% if 'siphon_block' in item['meta'] %}
{% set sb = item["meta"]["siphon_block"] %}
{% if sb %}
{# Extracted from the code in /*? ... ?*/ blocks #}
### Description
{{ sb }}
{% endif %}
{% endif %}
{% if "item" in meta or "function" in item %}
{# Gives some developer-useful linking #}
### Declaration and implementation
{% if "item" in meta %}
{{ "Declaration: @ref %s (@ref %s line %d)" %
(meta['item'], meta["file"], item["meta"]["line_start"]) }}
{% endif %}
{% if "function" in item %}
{{ "Implementation: @ref %s." % item["function"] }}
{% endif %}
{% endif %}
vlib/vlib/unix/cj.c
+34 -1
View File
@@ -17,6 +17,16 @@
*------------------------------------------------------------------
*/
/**
* @file
* Circular joournal diagnostic mechanism.
*
* The @c cj thread-safe circular log buffer scheme is occasionally useful
* when chasing bugs. Calls to it should not be checked in.
*/
/*? %%clicmd:group_label Circular Journal %% ?*/
/*? %%syscfg:group_label Circular Journal %% ?*/
#include <stdio.h>
#include <vlib/vlib.h>
@@ -94,6 +104,18 @@ cj_config (vlib_main_t * vm, unformat_input_t * input)
return 0;
}
/*?
* Configure the circular journal diagnostic mechanism. This is only useful
* if you, the deveoper, have written code to make use of the circular
* journal.
*
* @cfgcmd{records, &lt;number&gt;}
* Configure the number of records to allocate for the circular journal.
*
* @cfgcmd{on}
* Enable the collection of records in the circular journal at the
* earliest opportunity.
?*/
VLIB_CONFIG_FUNCTION (cj_config, "cj");
void
@@ -220,10 +242,21 @@ cj_command_fn (vlib_main_t * vm,
return 0;
}
/*?
* Enable, disable the collection of diagnostic data into a
* circular journal or dump the circular journal diagnostic data.
* This is only useful if you, the deveoper, have written code to make
* use of the circular journal.
*
* When dumping the data it is formatted and sent to @c stderr of the
* VPP process; when running VPP in <code>unix interactive</code> mode
* this is typically the same place as the Debug CLI.
?*/
/* *INDENT-OFF* */
VLIB_CLI_COMMAND (cj_command,static) = {
.path = "cj",
.short_help = "cj",
.short_help = "cj <enable | disable | dump>",
.function = cj_command_fn,
};
/* *INDENT-ON* */
vlib/vlib/unix/cli.c
+5 -1
View File
@@ -42,7 +42,8 @@
* Provides a command line interface so humans can interact with VPP.
* This is predominantly a debugging and testing mechanism.
*/
/*? %%clicmd:group_label Debug CLI %% ?*/
/*? %%clicmd:group_label Command line session %% ?*/
/*? %%syscfg:group_label Command line session %% ?*/
#include <vlib/vlib.h>
#include <vlib/unix/unix.h>
@@ -2504,6 +2505,9 @@ unix_cli_config (vlib_main_t * vm, unformat_input_t * input)
return 0;
}
/*?
* This module has no configurable parameters.
?*/
VLIB_CONFIG_FUNCTION (unix_cli_config, "unix-cli");
/** Called when VPP is shutting down, this restores the system
vlib/vlib/unix/dir.dox
+2 -1
View File
@@ -23,5 +23,6 @@
VLIB application library Unix interface layer.
*/
/*? %%clicmd:group_label VLIB Unix stuff%% ?*/
/*? %%clicmd:group_label Unix Interface %% ?*/
/*? %%syscfg:group_label Unix Interface %% ?*/
vlib/vlib/unix/main.c
+53
View File
@@ -407,6 +407,59 @@ unix_config (vlib_main_t * vm, unformat_input_t * input)
}
/* unix { ... } configuration. */
/*?
*
* @cfgcmd{interactive}
* Attach CLI to stdin/out and provide a debugging command line interface.
* Implies @c nodaemon.
*
* @cfgcmd{nodaemon}
* Do not fork or background the VPP process. Typically used when invoking
* VPP applications from a process monitor.
*
* @cfgcmd{exec, &lt;filename&gt;}
* @par <code>startup-config &lt;filename&gt;</code>
* Read startup operational configuration from @c filename.
* The contents of the file will be performed as though entered at the CLI.
* The two keywords are aliases for the same function; if both are specified,
* only the last will have an effect.
*
* @cfgcmd{log, &lt;filename&gt;}
* Logs the startup configuration and all subsequent CLI commands in
* @c filename.
* Very useful in situations where folks don't remember or can't be bothered
* to include CLI commands in bug reports.
*
* @cfgcmd{full-coredump}
* Ask the Linux kernel to dump all memory-mapped address regions, instead
* of just text+data+bss.
*
* @cfgcmd{cli-listen, &lt;address:port&gt;}
* Bind the CLI to listen at the address and port given. @clocalhost
* on TCP port @c 5002, given as <tt>cli-listen localhost:5002</tt>,
* is typical.
*
* @cfgcmd{cli-line-mode}
* Disable character-by-character I/O on stdin. Useful when combined with,
* for example, <tt>emacs M-x gud-gdb</tt>.
*
* @cfgcmd{cli-prompt, &lt;string&gt;}
* Configure the CLI prompt to be @c string.
*
* @cfgcmd{cli-history-limit, &lt;nn&gt;}
* Limit commmand history to @c nn lines. A value of @c 0
* disables command history. Default value: @c 50
*
* @cfgcmd{cli-no-banner}
* Disable the login banner on stdin and Telnet connections.
*
* @cfgcmd{cli-no-pager}
* Disable the output pager.
*
* @cfgcmd{cli-pager-buffer-limit, &lt;nn&gt;}
* Limit pager buffer to @c nn lines of output.
* A value of @c 0 disables the pager. Default value: @c 100000
?*/
VLIB_CONFIG_FUNCTION (unix_config, "unix");
static clib_error_t *