Add support for using documentation siphons in multiple ways

Experiental support for generating  multiple output formats from the
same siphoned data.

Adds a contrived example to generate a plain list of all CLI commands
(the "itemlist" format).

Eventually we can consider moving the tempate procesisng into the
Output class as well as a way to override how the data is traversed
(ordered).

Change-Id: I77629a74a8fa0c7e583993469dc50491f72f13e7
Signed-off-by: Chris Luke <chrisy@flirble.org>

doxygen/Makefile

@@ -121,6 +121,7 @@ SIPHONS ?= clicmd syscfg
 
 SIPHON_FILES = $(addprefix $(SIPHON_INPUT)/,$(addsuffix .siphon,$(SIPHONS)))
 SIPHON_DOCS = $(addprefix $(SIPHON_OUTPUT)/,$(addsuffix .md,$(SIPHONS)))
+SIPHON_ITEMLIST = $(addprefix $(SIPHON_OUTPUT)/,$(addsuffix .itemlist,$(filter clicmd,$(SIPHONS))))
 
 $(BR)/.doxygen-bootstrap.ok: Makefile
 	@echo "Checking whether dependencies for Doxygen are installed..."
@@ -200,24 +201,37 @@ $(SIPHON_FILES): $(BR)/.doxygen-bootstrap.ok \
 		--output="$(SIPHON_INPUT)" \
 		"@$(SIPHON_INPUT)/files"
 
-# Process the .siphon source fragments and render them into doxygen flavored
-# markdown documentation
-.DELETE_ON_ERROR: $(SIPHON_DOCS)
-$(SIPHON_OUTPUT)/%.md: $(SIPHON_INPUT)/%.siphon \
+# Evaluate this to build a siphon doc output target for each desired
+# output type:
+# $(eval $(call siphon-process,file_extension,output_type_name))
+define siphon-process
+$(SIPHON_OUTPUT)/%.$(1): $(SIPHON_INPUT)/%.siphon \
 		$(DOXY_DIR)/siphon-process \
 		$(wildcard $(DOXY_DIR)/siphon/*.py) \
-		$(wildcard $(DOXY_DIR)/siphon_templates/*/*.md)
-	@echo "Processing siphon from $(notdir $<)..."
+		$(wildcard $(DOXY_DIR)/siphon_templates/$(2)/*/*.$(1))
+	@echo "Processing siphon for $(2) from $$(notdir $$<)..."
 	@set -e; \
 	cd "$(WS_ROOT)"; \
 	$(DOXY_DIR)/siphon-process \
-		--type=$(basename $(notdir $<)) \
-		--output="$@" \
-		"$<"
+		--type=$$(basename $$(notdir $$<)) \
+		--format=$(2) \
+		--output="$$@" \
+		"$$<"
+endef
 
-# This target can be used just to generate the siphoned docs
+# Process the .siphon source fragments and render them into doxygen flavored
+# markdown documentation
+.DELETE_ON_ERROR: $(SIPHON_DOCS)
+$(eval $(call siphon-process,md,markdown))
+
+# Process the .siphon source fragments and render them into a list of cli
+# commands.
+.DELETE_ON_ERROR: $(SIPHON_ITEMLIST)
+$(eval $(call siphon-process,itemlist,itemlist))
+
+# This target can be used just to generate the siphoned things
 .PHONY: doxygen-siphon
-doxygen-siphon: $(SIPHON_DOCS)
+doxygen-siphon: $(SIPHON_DOCS) $(SIPHON_ITEMLIST)
 
 # Generate the doxygen docs
 .PHONY: doxygen

doxygen/siphon-process

@@ -23,6 +23,7 @@ import siphon
 DEFAULT_LOGFILE = None
 DEFAULT_LOGLEVEL = "info"
 DEFAULT_SIPHON ="clicmd"
+DEFAULT_FORMAT = "markdown"
 DEFAULT_OUTPUT = None
 DEFAULT_TEMPLATES = os.path.dirname(__file__) + "/siphon_templates"
 
@@ -36,6 +37,9 @@ ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL,
 ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON,
         choices=siphon.process.siphons.keys(),
         help="Siphon type to process [%s]" % DEFAULT_SIPHON)
+ap.add_argument("--format", '-f', default=DEFAULT_FORMAT,
+        choices=siphon.process.formats.keys(),
+        help="Output format to generate [%s]" % DEFAULT_FORMAT)
 ap.add_argument("--output", '-o', metavar="file", default=DEFAULT_OUTPUT,
         help="Output file (uses stdout if not defined) [%s]" % DEFAULT_OUTPUT)
 ap.add_argument("--templates", metavar="directory", default=DEFAULT_TEMPLATES,
@@ -56,7 +60,7 @@ else:
 
 # Get our processor
 klass = siphon.process.siphons[args.type]
-processor = klass(template_directory=args.templates)
+processor = klass(template_directory=args.templates, format=args.format)
 
 # Load the input files
 processor.load_json(args.input)

doxygen/siphon/process.py

@@ -20,6 +20,9 @@ import logging, os,sys, cgi, json, jinja2, HTMLParser
 """Mapping of known processors to their classes"""
 siphons = {}
 
+"""Mapping of known output formats to their classes"""
+formats = {}
+
 
 """Generate rendered output for siphoned data."""
 class Siphon(object):
@@ -51,28 +54,36 @@ class Siphon(object):
     """Template environment, if we're using templates"""
     _tplenv = None
 
-    def __init__(self, template_directory=None):
+    def __init__(self, template_directory, format):
         super(Siphon, self).__init__()
         self.log = logging.getLogger("siphon.process.%s" % self.name)
 
-        if template_directory is not None:
-          self.template_directory = template_directory
-          searchpath = [
-              template_directory + "/" + self.name,
-              template_directory + "/" + "default",
-          ]
-          loader = jinja2.FileSystemLoader(searchpath=searchpath)
-          self._tplenv = jinja2.Environment(
-              loader=loader,
-              trim_blocks=True,
-              keep_trailing_newline=True)
+        # Get our output format details
+        fmt_klass = formats[format]
+        fmt = fmt_klass()
+        self._format = fmt
 
-          # Convenience, get a reference to the internal escape and
-          # unescape methods in cgi and HTMLParser. These then become
-          # available to templates to use, if needed.
-          self._h = HTMLParser.HTMLParser()
-          self.escape = cgi.escape
-          self.unescape = self._h.unescape
+        # Sort out the template search path
+        def _tpldir(name):
+            return os.sep.join((template_directory, fmt.name, name))
+
+        self.template_directory = template_directory
+        searchpath = [
+            _tpldir(self.name),
+            _tpldir("default"),
+        ]
+        loader = jinja2.FileSystemLoader(searchpath=searchpath)
+        self._tplenv = jinja2.Environment(
+            loader=loader,
+            trim_blocks=True,
+            keep_trailing_newline=True)
+
+        # Convenience, get a reference to the internal escape and
+        # unescape methods in cgi and HTMLParser. These then become
+        # available to templates to use, if needed.
+        self._h = HTMLParser.HTMLParser()
+        self.escape = cgi.escape
+        self.unescape = self._h.unescape
 
 
     # Output renderers
@@ -157,7 +168,7 @@ class Siphon(object):
 
     """Template processor"""
     def template(self, name, **kwargs):
-      tpl = self._tplenv.get_template(name + ".md")
+      tpl = self._tplenv.get_template(name + self._format.extension)
       return tpl.render(
             this=self,
             **kwargs)
@@ -270,3 +281,31 @@ class Siphon(object):
 
         # Deliver the accumulated body output
         out.write(contents)
+
+
+"""Output format class"""
+class Format(object):
+
+    """Name of this output format"""
+    name = None
+
+    """Expected file extension of templates that build this format"""
+    extension = None
+
+
+"""Markdown output format"""
+class FormatMarkdown(Format):
+    name = "markdown"
+    extension = ".md"
+
+# Register 'markdown'
+formats["markdown"] = FormatMarkdown
+
+
+"""Itemlist output format"""
+class FormatItemlist(Format):
+    name = "itemlist"
+    extension = ".itemlist"
+
+# Register 'itemlist'
+formats["itemlist"] = FormatItemlist

doxygen/siphon_templates/itemlist/clicmd/item_format.itemlist

@@ -0,0 +1,17 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{# Just output the command path #}
+{{ item['value']['path'] }}

doxygen/siphon_templates/itemlist/default/index_entry.itemlist

@@ -0,0 +1,15 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}

doxygen/siphon_templates/itemlist/default/index_header.itemlist

@@ -0,0 +1,15 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}

doxygen/siphon_templates/itemlist/default/index_section.itemlist

@@ -0,0 +1,15 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}

doxygen/siphon_templates/itemlist/default/item_header.itemlist

@@ -0,0 +1,15 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}

doxygen/siphon_templates/itemlist/syscfg/item_format.itemlist

@@ -0,0 +1,17 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{# Just output the item name #}
+{{ item['name'] }}

doxygen/siphon_templates/markdown/default/item_format.md

@@ -0,0 +1,16 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{{ raise NotImplementedError }}