d9b0c6fbf7
Drop pycodestyle for code style checking in favor of black. Black is much faster, stable PEP8 compliant code style checker offering also automatic formatting. It aims to be very stable and produce smallest diffs. It's used by many small and big projects. Running checkstyle with black takes a few seconds with a terse output. Thus, test-checkstyle-diff is no longer necessary. Expand scope of checkstyle to all python files in the repo, replacing test-checkstyle with checkstyle-python. Also, fixstyle-python is now available for automatic style formatting. Note: python virtualenv has been consolidated in test/Makefile, test/requirements*.txt which will eventually be moved to a central location. This is required to simply the automated generation of docker executor images in the CI. Type: improvement Change-Id: I022a326603485f58585e879ac0f697fceefbc9c8 Signed-off-by: Klement Sekera <klement.sekera@gmail.com> Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
425 lines
13 KiB
Python
425 lines
13 KiB
Python
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at:
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Generation template class
|
|
|
|
import html.parser
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import re
|
|
|
|
import jinja2
|
|
|
|
# Classes register themselves in this dictionary
|
|
"""Mapping of known processors to their classes"""
|
|
siphons = {}
|
|
|
|
"""Mapping of known output formats to their classes"""
|
|
formats = {}
|
|
|
|
|
|
class Siphon(object):
|
|
"""Generate rendered output for siphoned data."""
|
|
|
|
# Set by subclasses
|
|
"""Our siphon name"""
|
|
name = None
|
|
|
|
# Set by subclasses
|
|
"""Name of an identifier used by this siphon"""
|
|
identifier = None
|
|
|
|
# Set by subclasses
|
|
"""The pyparsing object to use to parse with"""
|
|
_parser = None
|
|
|
|
"""The input data"""
|
|
_cmds = None
|
|
|
|
"""Group key to (directory,file) mapping"""
|
|
_group = None
|
|
|
|
"""Logging handler"""
|
|
log = None
|
|
|
|
"""Directory to look for siphon rendering templates"""
|
|
template_directory = None
|
|
|
|
"""Directory to output parts in"""
|
|
outdir = None
|
|
|
|
"""Template environment, if we're using templates"""
|
|
_tplenv = None
|
|
|
|
def __init__(self, template_directory, format, outdir, repository_link):
|
|
super(Siphon, self).__init__()
|
|
self.log = logging.getLogger("siphon.process.%s" % self.name)
|
|
|
|
# Get our output format details
|
|
fmt_klass = formats[format]
|
|
fmt = fmt_klass()
|
|
self._format = fmt
|
|
|
|
# Sort out the template search path
|
|
def _tpldir(name):
|
|
return os.sep.join((template_directory, fmt.name, name))
|
|
|
|
self.template_directory = template_directory
|
|
searchpath = [
|
|
_tpldir(self.name),
|
|
_tpldir("default"),
|
|
]
|
|
self.outdir = outdir
|
|
loader = jinja2.FileSystemLoader(searchpath=searchpath)
|
|
self._tplenv = jinja2.Environment(
|
|
loader=loader,
|
|
trim_blocks=True,
|
|
autoescape=False,
|
|
keep_trailing_newline=True,
|
|
)
|
|
|
|
# Convenience, get a reference to the internal escape and
|
|
# unescape methods in html.parser. These then become
|
|
# available to templates to use, if needed.
|
|
self._h = html.parser.HTMLParser()
|
|
self.escape = html.escape
|
|
self.unescape = html.unescape
|
|
|
|
# TODO: customize release
|
|
self.repository_link = repository_link
|
|
|
|
# Output renderers
|
|
|
|
"""Returns an object to be used as the sorting key in the item index."""
|
|
|
|
def index_sort_key(self, group):
|
|
return group
|
|
|
|
"""Returns a string to use as the header at the top of the item index."""
|
|
|
|
def index_header(self):
|
|
return self.template("index_header")
|
|
|
|
"""Returns the string fragment to use for each section in the item
|
|
index."""
|
|
|
|
def index_section(self, group):
|
|
return self.template("index_section", group=group)
|
|
|
|
"""Returns the string fragment to use for each entry in the item index."""
|
|
|
|
def index_entry(self, meta, item):
|
|
return self.template("index_entry", meta=meta, item=item)
|
|
|
|
"""Returns an object, typically a string, to be used as the sorting key
|
|
for items within a section."""
|
|
|
|
def item_sort_key(self, item):
|
|
return item["name"]
|
|
|
|
"""Returns a key for grouping items together."""
|
|
|
|
def group_key(self, directory, file, macro, name):
|
|
_global = self._cmds["_global"]
|
|
|
|
if file in _global and "group_label" in _global[file]:
|
|
self._group[file] = (directory, file)
|
|
return file
|
|
|
|
self._group[directory] = (directory, None)
|
|
return directory
|
|
|
|
"""Returns a key for identifying items within a grouping."""
|
|
|
|
def item_key(self, directory, file, macro, name):
|
|
return name
|
|
|
|
"""Returns a string to use as the header when rendering the item."""
|
|
|
|
def item_header(self, group):
|
|
return self.template("item_header", group=group)
|
|
|
|
"""Returns a string to use as the body when rendering the item."""
|
|
|
|
def item_format(self, meta, item):
|
|
return self.template("item_format", meta=meta, item=item)
|
|
|
|
"""Returns a string to use as the label for the page reference."""
|
|
|
|
def page_label(self, group):
|
|
return "_".join((self.name, self.sanitize_label(group)))
|
|
|
|
"""Returns a title to use for a page."""
|
|
|
|
def page_title(self, group):
|
|
_global = self._cmds["_global"]
|
|
(directory, file) = self._group[group]
|
|
|
|
if file and file in _global and "group_label" in _global[file]:
|
|
return _global[file]["group_label"]
|
|
|
|
if directory in _global and "group_label" in _global[directory]:
|
|
return _global[directory]["group_label"]
|
|
|
|
return directory
|
|
|
|
"""Returns a string to use as the label for the section reference."""
|
|
|
|
def item_label(self, group, item):
|
|
return "__".join((self.name, item))
|
|
|
|
"""Label sanitizer; for creating Doxygen references"""
|
|
|
|
def sanitize_label(self, value):
|
|
return value.replace(" ", "_").replace("/", "_").replace(".", "_")
|
|
|
|
"""Template processor"""
|
|
|
|
def template(self, name, **kwargs):
|
|
tpl = self._tplenv.get_template(name + self._format.extension)
|
|
return tpl.render(this=self, **kwargs)
|
|
|
|
# Processing methods
|
|
|
|
"""Parse the input file into a more usable dictionary structure."""
|
|
|
|
def load_json(self, files):
|
|
self._cmds = {}
|
|
self._group = {}
|
|
|
|
line_num = 0
|
|
line_start = 0
|
|
for filename in files:
|
|
filename = os.path.relpath(filename)
|
|
self.log.info('Parsing items in file "%s".' % filename)
|
|
data = None
|
|
with open(filename, "r") as fd:
|
|
data = json.load(fd)
|
|
|
|
self._cmds["_global"] = data["global"]
|
|
|
|
# iterate the items loaded and regroup it
|
|
for item in data["items"]:
|
|
try:
|
|
o = self._parser.parse(item["block"])
|
|
except Exception:
|
|
self.log.error(
|
|
"Exception parsing item: %s\n%s"
|
|
% (
|
|
json.dumps(item, separators=(",", ": "), indent=4),
|
|
item["block"],
|
|
)
|
|
)
|
|
raise
|
|
|
|
# Augment the item with metadata
|
|
o["meta"] = {}
|
|
for key in item:
|
|
if key == "block":
|
|
continue
|
|
o["meta"][key] = item[key]
|
|
|
|
# Load some interesting fields
|
|
directory = item["directory"]
|
|
file = item["file"]
|
|
macro = o["macro"]
|
|
name = o["name"]
|
|
|
|
# Generate keys to group items by
|
|
group_key = self.group_key(directory, file, macro, name)
|
|
item_key = self.item_key(directory, file, macro, name)
|
|
|
|
if group_key not in self._cmds:
|
|
self._cmds[group_key] = {}
|
|
|
|
self._cmds[group_key][item_key] = o
|
|
|
|
"""Iterate over the input data, calling render methods to generate the
|
|
output."""
|
|
|
|
def process(self, out=None):
|
|
|
|
if out is None:
|
|
out = sys.stdout
|
|
|
|
# Accumulated body contents
|
|
contents = ""
|
|
|
|
# Write the header for this siphon type
|
|
out.write(self.index_header())
|
|
|
|
# Sort key helper for the index
|
|
def group_sort_key(group):
|
|
return self.index_sort_key(group)
|
|
|
|
# Iterate the dictionary and process it
|
|
for group in sorted(self._cmds.keys(), key=group_sort_key):
|
|
if group.startswith("_"):
|
|
continue
|
|
|
|
self.log.info(
|
|
'Processing items in group "%s" (%s).' % (group, group_sort_key(group))
|
|
)
|
|
|
|
# Generate the section index entry (write it now)
|
|
out.write(self.index_section(group))
|
|
|
|
# Generate the item header (save for later)
|
|
contents += self.item_header(group)
|
|
|
|
def item_sort_key(key):
|
|
return self.item_sort_key(self._cmds[group][key])
|
|
|
|
for key in sorted(self._cmds[group].keys(), key=item_sort_key):
|
|
self.log.debug(
|
|
'--- Processing key "%s" (%s).' % (key, item_sort_key(key))
|
|
)
|
|
|
|
o = self._cmds[group][key]
|
|
meta = {
|
|
"directory": o["meta"]["directory"],
|
|
"file": o["meta"]["file"],
|
|
"macro": o["macro"],
|
|
"name": o["name"],
|
|
"key": key,
|
|
"label": self.item_label(group, key),
|
|
}
|
|
|
|
# Generate the index entry for the item (write it now)
|
|
out.write(self.index_entry(meta, o))
|
|
|
|
# Generate the item itself (save for later)
|
|
contents += self.item_format(meta, o)
|
|
|
|
page_name = self.separate_page_names(group)
|
|
if page_name != "":
|
|
path = os.path.join(self.outdir, page_name)
|
|
with open(path, "w+") as page:
|
|
page.write(contents)
|
|
contents = ""
|
|
|
|
# Deliver the accumulated body output
|
|
out.write(contents)
|
|
|
|
def do_cliexstart(self, matchobj):
|
|
title = matchobj.group(1)
|
|
title = " ".join(title.splitlines())
|
|
content = matchobj.group(2)
|
|
content = re.sub(r"\n", r"\n ", content)
|
|
return "\n\n.. code-block:: console\n\n %s\n %s\n\n" % (title, content)
|
|
|
|
def do_clistart(self, matchobj):
|
|
content = matchobj.group(1)
|
|
content = re.sub(r"\n", r"\n ", content)
|
|
return "\n\n.. code-block:: console\n\n %s\n\n" % content
|
|
|
|
def do_cliexcmd(self, matchobj):
|
|
content = matchobj.group(1)
|
|
content = " ".join(content.splitlines())
|
|
return "\n\n.. code-block:: console\n\n %s\n\n" % content
|
|
|
|
def process_list(self, matchobj):
|
|
content = matchobj.group(1)
|
|
content = self.reindent(content, 2)
|
|
return "@@@@%s\nBBBB" % content
|
|
|
|
def process_special(self, s):
|
|
# ----------- markers to remove
|
|
s = re.sub(r"@cliexpar\s*", r"", s)
|
|
s = re.sub(r"@parblock\s*", r"", s)
|
|
s = re.sub(r"@endparblock\s*", r"", s)
|
|
s = re.sub(r"<br>", "", s)
|
|
# ----------- emphasis
|
|
# <b><em>
|
|
s = re.sub(r"<b><em>\s*", "``", s)
|
|
s = re.sub(r"\s*</b></em>", "``", s)
|
|
s = re.sub(r"\s*</em></b>", "``", s)
|
|
# <b>
|
|
s = re.sub(r"<b>\s*", "**", s)
|
|
s = re.sub(r"\s*</b>", "**", s)
|
|
# <code>
|
|
s = re.sub(r"<code>\s*", "``", s)
|
|
s = re.sub(r"\s*</code>", "``", s)
|
|
# <em>
|
|
s = re.sub(r"'?<em>\s*", r"``", s)
|
|
s = re.sub(r"\s*</em>'?", r"``", s)
|
|
# @c <something>
|
|
s = re.sub(r"@c\s(\S+)", r"``\1``", s)
|
|
# ----------- todos
|
|
s = re.sub(r"@todo[^\n]*", "", s)
|
|
s = re.sub(r"@TODO[^\n]*", "", s)
|
|
# ----------- code blocks
|
|
s = re.sub(r"@cliexcmd{(.+?)}", self.do_cliexcmd, s, flags=re.DOTALL)
|
|
s = re.sub(
|
|
r"@cliexstart{(.+?)}(.+?)@cliexend", self.do_cliexstart, s, flags=re.DOTALL
|
|
)
|
|
s = re.sub(r"@clistart(.+?)@cliend", self.do_clistart, s, flags=re.DOTALL)
|
|
# ----------- lists
|
|
s = re.sub(r"^\s*-", r"\n@@@@", s, flags=re.MULTILINE)
|
|
s = re.sub(r"@@@@(.*?)\n\n+", self.process_list, s, flags=re.DOTALL)
|
|
s = re.sub(r"BBBB@@@@", r"-", s)
|
|
s = re.sub(r"@@@@", r"-", s)
|
|
s = re.sub(r"BBBB", r"\n\n", s)
|
|
# ----------- Cleanup remains
|
|
s = re.sub(r"@cliexend\s*", r"", s)
|
|
return s
|
|
|
|
def separate_page_names(self, group):
|
|
return ""
|
|
|
|
# This push the given textblock <indent> spaces right
|
|
def reindent(self, s, indent):
|
|
ind = " " * indent
|
|
s = re.sub(r"\n", "\n" + ind, s)
|
|
return s
|
|
|
|
# This aligns the given textblock left (no indent)
|
|
def noindent(self, s):
|
|
s = re.sub(r"\n[ \f\v\t]*", "\n", s)
|
|
return s
|
|
|
|
|
|
class Format(object):
|
|
"""Output format class"""
|
|
|
|
"""Name of this output format"""
|
|
name = None
|
|
|
|
"""Expected file extension of templates that build this format"""
|
|
extension = None
|
|
|
|
|
|
class FormatMarkdown(Format):
|
|
"""Markdown output format"""
|
|
|
|
name = "markdown"
|
|
extension = ".md"
|
|
|
|
|
|
# Register 'markdown'
|
|
formats["markdown"] = FormatMarkdown
|
|
|
|
|
|
class FormatItemlist(Format):
|
|
"""Itemlist output format"""
|
|
|
|
name = "itemlist"
|
|
extension = ".itemlist"
|
|
|
|
|
|
# Register 'itemlist'
|
|
formats["itemlist"] = FormatItemlist
|