docs: better docs, mv doxygen to sphinx

This patch refactors the VPP sphinx docs
in order to make it easier to consume
for external readers as well as VPP developers.

It also makes sphinx the single source
of documentation, which simplifies maintenance
and operation.

Most important updates are:

- reformat the existing documentation as rst
- split RELEASE.md and move it into separate rst files
- remove section 'events'
- remove section 'archive'
- remove section 'related projects'
- remove section 'feature by release'
- remove section 'Various links'
- make (Configuration reference, CLI docs,
  developer docs) top level items in the list
- move 'Use Cases' as part of 'About VPP'
- move 'Troubleshooting' as part of 'Getting Started'
- move test framework docs into 'Developer Documentation'
- add a 'Contributing' section for gerrit,
  docs and other contributer related infos
- deprecate doxygen and test-docs targets
- redirect the "make doxygen" target to "make docs"

Type: refactor

Change-Id: I552a5645d5b7964d547f99b1336e2ac24e7c209f
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>

.gitignore

@@ -82,10 +82,7 @@ GTAGS
 /build-root/docs
 /build-root/.doxygen-bootstrap.ok
 /build-root/.doxygen-siphon.dep
-/docs/_build
-/docs/dynamic_includes
-/sphinx_venv
-!/docs/Makefile
+/docs/venv
 
 # language servers
 compile_commands.json

MAINTAINERS

@@ -33,11 +33,6 @@ I:	bonding
 M:	Steven Luong <sluong@cisco.com>
 F:	src/vnet/bonding/
 
-Doxygen
-I:	doxygen
-M:	Chris Luke <chrisy@flirble.org>
-F:	doxygen/
-
 Sphinx Documents
 I:	docs
 M:	John DeNisco <jdenisco@cisco.com>

Makefile

@@ -209,9 +209,9 @@ help:
 	@echo " checkstyle-test-diff - check test framework coding style (only changed files)"
 	@echo " checkstyle-api       - check api for incompatible changes"
 	@echo " fixstyle             - fix coding style"
-	@echo " doxygen              - (re)generate documentation"
-	@echo " bootstrap-doxygen    - setup Doxygen dependencies"
-	@echo " wipe-doxygen         - wipe all generated documentation"
+	@echo " doxygen              - DEPRECATED - use 'make docs'"
+	@echo " bootstrap-doxygen    - DEPRECATED"
+	@echo " wipe-doxygen         - DEPRECATED"
 	@echo " checkfeaturelist     - check FEATURE.yaml according to schema"
 	@echo " featurelist          - dump feature list in markdown"
 	@echo " json-api-files       - (re)-generate json api files"
@@ -451,11 +451,13 @@ test-dep:
 
 .PHONY: test-doc
 test-doc:
-	@make -C test doc
+	@echo "make test-doc is DEPRECATED: use 'make doc'"
+	sleep 300
 
 .PHONY: test-wipe-doc
 test-wipe-doc:
-	@make -C test wipe-doc
+	@echo "make test-wipe-doc is DEPRECATED"
+	sleep 300
 
 .PHONY: test-cov
 test-cov:
@@ -667,42 +669,28 @@ checkfeaturelist: centos-pyyaml
 # Build the documentation
 #
 
-# Doxygen configuration and our utility scripts
-export DOXY_DIR ?= $(WS_ROOT)/doxygen
-
-define make-doxy
-	@OS_ID="$(OS_ID)" make -C $(DOXY_DIR) $@
-endef
-
 .PHONY: bootstrap-doxygen
 bootstrap-doxygen:
-	$(call make-doxy)
+	@echo "make bootstrap-doxygen is DEPRECATED"
+	sleep 300
 
 .PHONY: doxygen
-doxygen: bootstrap-doxygen
-	$(call make-doxy)
+doxygen: docs
+	@echo "make doxygen is DEPRECATED: use 'make docs'"
+	sleep 300
 
 .PHONY: wipe-doxygen
 wipe-doxygen:
-	$(call make-doxy)
+	@echo "make wipe-doxygen is DEPRECATED"
+	sleep 300
 
-# Sphinx Documents
-export DOCS_DIR = $(WS_ROOT)/docs
-export VENV_DIR = $(WS_ROOT)/sphinx_venv
-export SPHINX_SCRIPTS_DIR = $(WS_ROOT)/docs/scripts
-
-.PHONY: docs-venv
-docs-venv:
-	@($(SPHINX_SCRIPTS_DIR)/sphinx-make.sh venv)
+.PHONY: docs-%
+docs-%:
+	@make -C $(WS_ROOT)/docs $*
 
 .PHONY: docs
-docs: $(DOCS_DIR)
-	@($(SPHINX_SCRIPTS_DIR)/sphinx-make.sh html)
-
-.PHONY: docs-clean
-docs-clean:
-	@rm -rf $(DOCS_DIR)/_build
-	@rm -rf $(VENV_DIR)
+docs:
+	@make -C $(WS_ROOT)/docs docs
 
 .PHONY: pkg-verify
 pkg-verify: install-dep $(BR)/.deps.ok install-ext-deps

README.md

@@ -19,32 +19,32 @@ For more information on VPP and its features please visit the
 ## Changes
 
 Details of the changes leading up to this version of VPP can be found under
-@ref release_notes.
+doc/releasenotes.
 
 
 ## Directory layout
 
 | Directory name         | Description                                 |
 | ---------------------- | ------------------------------------------- |
-|      build-data        | Build metadata                              |
-|      build-root        | Build output directory                      |
-|      doxygen           | Documentation generator configuration       |
-|      dpdk              | DPDK patches and build infrastructure       |
-| @ref extras/libmemif   | Client library for memif                    |
-| @ref src/examples      | VPP example code                            |
-| @ref src/plugins       | VPP bundled plugins directory               |
-| @ref src/svm           | Shared virtual memory allocation library    |
-|      src/tests         | Standalone tests (not part of test harness) |
-|      src/vat           | VPP API test program                        |
-| @ref src/vlib          | VPP application library                     |
-| @ref src/vlibapi       | VPP API library                             |
-| @ref src/vlibmemory    | VPP Memory management                       |
-| @ref src/vnet          | VPP networking                              |
-| @ref src/vpp           | VPP application                             |
-| @ref src/vpp-api       | VPP application API bindings                |
-| @ref src/vppinfra      | VPP core library                            |
-| @ref src/vpp/api       | Not-yet-relocated API bindings              |
-|      test              | Unit tests and Python test harness          |
+| build-data             | Build metadata                              |
+| build-root             | Build output directory                      |
+| docs                   | Sphinx Documentation                        |
+| dpdk                   | DPDK patches and build infrastructure       |
+| extras/libmemif        | Client library for memif                    |
+| src/examples           | VPP example code                            |
+| src/plugins            | VPP bundled plugins directory               |
+| src/svm                | Shared virtual memory allocation library    |
+| src/tests              | Standalone tests (not part of test harness) |
+| src/vat                | VPP API test program                        |
+| src/vlib               | VPP application library                     |
+| src/vlibapi            | VPP API library                             |
+| src/vlibmemory         | VPP Memory management                       |
+| src/vnet               | VPP networking                              |
+| src/vpp                | VPP application                             |
+| src/vpp-api            | VPP application API bindings                |
+| src/vppinfra           | VPP core library                            |
+| src/vpp/api            | Not-yet-relocated API bindings              |
+| test                   | Unit tests and Python test harness          |
 
 ## Getting started
 
@@ -99,8 +99,3 @@ end-user-oriented information. Also see @subpage dev_doc for developer notes.
 Visit the [VPP wiki](https://wiki.fd.io/view/VPP) for details on more
 advanced building strategies and other development notes.
 
-
-## Test Framework
-
-There is PyDoc generated documentation available for the VPP test framework.
-See @ref test_framework_doc for details.

docs/Makefile

@@ -5,33 +5,46 @@ ifeq ($(shell uname),Darwin)
 OS_ID = darwin
 endif
 
+# These should be passed in by the root Makefile
+WS_ROOT ?= $(CURDIR)/..
+BR ?= $(WS_ROOT)/build-root
+DOCS_DIR ?= $(WS_ROOT)/docs
+
+VENV_DIR ?= $(DOCS_DIR)/venv
+SPHINX_SCRIPTS_DIR ?= $(WS_ROOT)/docs/scripts
+
 # Work out the OS if we haven't already
 OS_ID ?= $(shell grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+OS_VERSION ?= $(shell grep '^VERSION_ID=' /etc/os-release  | cut -f2- -d= | sed -e 's/\"//g')
+PIP_VERSION ?= $(shell grep 'PIP_VERSION=' ${WS_ROOT}/test/Makefile | cut -d'=' -f2)
+PIP_TOOLS_VERSION ?= $(shell grep 'PIP_TOOLS_VERSION=' ${WS_ROOT}/test/Makefile | cut -d'=' -f2)
+
+PYTHON ?= "python3"
 
 DOC_DEB_DEPENDS = enchant
 DOC_RPM_DEPENDS = enchant
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-SPHINXPROJ    = fdio-vpp
-SOURCEDIR     = .
-BUILDDIR      = _build
+SPHINXOPTS        = --keep-going -n -W
+SPHINXBUILD       = sphinx-build
+SPHINXPROJ        = fdio-vpp
+SOURCEDIR         = .
+BUILDDIR          = ${BR}/docs
+BUILDDIR_SRC      = ${BUILDDIR}/src
+BUILDDIR_OUT      = ${BUILDDIR}/html
+SCRIPTS_DIR       = _scripts
+
 
 # Put it first so that "make" without argument is like "make help".
+.PHONY: help
 help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@( \
+	  . ${VENV_DIR}/bin/activate; \
+	  $(SPHINXBUILD) --help ;\
+	)
 
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-# Generate dynamic content
-	@python3 ./includes_renderer.py
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-spell:
+.PHONY: checkdeps
+checkdeps:
 	@echo "Checking whether dependencies for Docs are installed..."
 ifeq ($(OS_ID),ubuntu)
 	@set -e; inst=; \
@@ -45,4 +58,45 @@ ifeq ($(OS_ID),ubuntu)
 else ifneq ("$(wildcard /etc/redhat-release)","")
 	@sudo yum install $(CONFIRM) $(DOC_RPM_DEPENDS)
 endif
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" -W -b spelling  $(O)
+
+.PHONY: spell
+spell: clean checkdeps venv ${BUILDDIR_SRC}
+	@( \
+	  . ${VENV_DIR}/bin/activate; \
+	  make -C ${SCRIPTS_DIR} generate && \
+	  $(SPHINXBUILD) -b spelling $(SPHINXOPTS) $(BUILDDIR_SRC) $(BUILDDIR_OUT); \
+	)
+
+.PHONY: venv
+venv:
+	@( \
+	    if [ ! -d ${VENV_DIR} ]; then \
+	      ${PYTHON} -m venv ${VENV_DIR}; \
+	      . ${VENV_DIR}/bin/activate; \
+	      ${PYTHON} -m pip install pip==${PIP_VERSION}; \
+	      ${PYTHON} -m pip install pip-tools==${PIP_TOOLS_VERSION}; \
+	      ${PYTHON} -m pip install -r ${WS_ROOT}/test/requirements-3.txt; \
+	    fi; \
+	)
+
+${BUILDDIR_SRC}:
+	@mkdir -p ${BUILDDIR_SRC}
+	@cp -r $(SOURCEDIR) ${BUILDDIR_SRC}
+	@cd ${BUILDDIR_SRC} && find . -type l -exec cp --remove-destination -L ${DOCS_DIR}/{} {} \;
+
+.PHONY: docs
+docs: clean venv ${BUILDDIR_SRC}
+	@( \
+	  . ${VENV_DIR}/bin/activate; \
+	  make -C ${SCRIPTS_DIR} generate && \
+	  $(SPHINXBUILD) $(SPHINXOPTS) -b html $(BUILDDIR_SRC) $(BUILDDIR_OUT); \
+	)
+
+.PHONY: clean
+clean:
+	@rm -rf $(BUILDDIR)
+	@make -C ${SCRIPTS_DIR} clean
+
+.PHONY: build
+build: docs
+

docs/README

@@ -23,7 +23,7 @@ $ cd vpp
 Install the virtual environment
 ----------------------------------------------
 
-$ python -m pip install --user virtualenv 
+$ python -m pip install --user virtualenv
 $ python -m virtualenv env
 $ source env/bin/activate
 $ pip install -r docs/etc/requirements.txt

docs/_images/libmemif_architecture.png

@@ -0,0 +1 @@
+../../extras/libmemif/docs/architecture.png

docs/_scripts/Makefile

@@ -0,0 +1,212 @@
+# Copyright (c) 2021 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Build the documentation
+#
+
+# Default target
+.PHONY: all
+all: siphon
+
+# These should be passed in by the root Makefile
+WS_ROOT ?= $(CURDIR)/../..
+BR ?= $(WS_ROOT)/build-root
+
+# Tag used in github repository path.
+# Change this when genearting for a release
+VPP_TAG ?= master
+
+REPOSITORY_URL ?= https://github.com/FDio/vpp/blob/$(VPP_TAG)/
+
+# Doxygen configuration and our utility scripts
+SCRIPTS_DIR ?= $(WS_ROOT)/docs/_scripts
+
+# docs root directory
+DOCS_DIR ?= ${BR}/docs/src
+
+FEATURE_LIST_FILE = ${DOCS_DIR}/aboutvpp/featurelist.md
+
+# Siphoned fragements are processed into here
+DOCS_GENERATED_DIR ?= $(DOCS_DIR)/_generated
+
+# Siphoned fragments end up in here
+SIPHON_INPUT_DIR ?= $(DOCS_GENERATED_DIR)/fragments
+
+DYNAMIC_RENDER_DIR ?= ${DOCS_GENERATED_DIR}/includes
+
+# Primary source directories
+SIPHON_SRC ?= $(WS_ROOT)/src
+SIPHON_SRC_DIRECTORIES = \
+	$(shell find $(SIPHON_SRC) -name '*.md' -print | xargs dirname \
+	| sort | uniq) \
+	$(SIPHON_SRC)/vppinfra \
+	$(SIPHON_SRC)/svm \
+	$(SIPHON_SRC)/vlib \
+	$(SIPHON_SRC)/vlibapi \
+	$(SIPHON_SRC)/vlibmemory \
+	$(SIPHON_SRC)/vnet \
+	$(SIPHON_SRC)/vpp \
+	$(SIPHON_SRC)/vpp-api \
+	$(SIPHON_SRC)/examples
+
+# Input directories and files
+SIPHON_INPUT ?= \
+	$(wildcard $(WS_ROOT)/*.md) \
+	$(wildcard $(SCRIPTS_DIR)/*.md) \
+	$(SIPHON_SRC_DIRECTORIES) \
+	$(SIPHON_SRC)/plugins \
+	extras
+
+# Strip leading workspace path from input names
+SIPHON_INPUT := $(subst $(WS_ROOT)/,,$(SIPHON_INPUT))
+
+# Files to exclude, from pre-Doxygen steps, eg because they're
+# selectively compiled.
+# Examples would be to exclude non-DPDK related sources when
+# there's a DPDK equivalent that conflicts.
+# These must be left-anchored paths for the regexp below to work.
+SIPHON_EXCLUDE ?= \
+	$(SIPHON_SRC)/vpp-api/lua
+
+# Generate a regexp for filenames to exclude
+SIPHON_EXCLUDE_REGEXP = ($(subst .,\.,$(shell echo '$(strip $(SIPHON_EXCLUDE))' | sed -e 's/ /|/g')))
+
+# Include all the normal source directories in the include file path
+SIPHON_INCLUDE_PATH = $(SIPHON_SRC_DIRECTORIES)
+
+# Find API header directories and include them in the header path.
+# This is only useful if VPP and plugins are already built; nothing
+# here depends on those targets. We don't build documentation for these
+# header files, they're just added to the INCLUDE search path for Doxygen.
+_vpp_br = $(shell find "$(BR)" -maxdepth 1 -type d \
+	'(' -name build-vpp_debug-native -o -name build-vpp-native ')' -print \
+	| sed -e 's@^$(WS_ROOT)/*@@' -e 1q)
+ifneq ($(strip $(_vpp_br)),)
+SIPHON_INCLUDE_PATH += \
+	$(_vpp_br)/vlib-api \
+	$(_vpp_br)/vpp
+# Also include any plugin directories that exist
+SIPHON_INCLUDE_PATH += \
+	$(shell find $(WS_ROOT)/$(_vpp_br)/plugins -maxdepth 1 -type d | sed -e 's@^$(WS_ROOT)/*@@')
+endif
+
+# Discover if we have CPP available
+_cpp = $(shell which cpp)
+ifneq ($(strip $(_cpp)),)
+# Add whatever directories CPP normally includes to the header path
+SIPHON_INCLUDE_PATH += $(shell set -e; $(_cpp) -v </dev/null 2>&1 | awk 'f&&/^ /{print $$1} /^\#include/{f=1}')
+endif
+
+# All the siphon types we know about
+SIPHONS ?= clicmd syscfg
+
+SIPHON_FILES = $(addprefix $(SIPHON_INPUT_DIR)/,$(addsuffix .siphon,$(SIPHONS)))
+SIPHON_DOCS = $(addprefix $(DOCS_GENERATED_DIR)/,$(addsuffix .rst,$(SIPHONS)))
+
+BUILT_ON = $(shell date '+%d %B %Y')
+VPP_VERSION = $(shell ${WS_ROOT}/src/scripts/version)
+
+.PHONY: featurelist
+featurelist:
+	@( \
+	  cd $(WS_ROOT) && \
+	  find . -name FEATURE.yaml | \
+	    ./src/scripts/fts.py \
+	      --markdown \
+	      --repolink $(REPOSITORY_URL) > \
+	    $(FEATURE_LIST_FILE) ; \
+	)
+
+
+.PHONY: includes-render
+includes-render:
+	@mkdir -p "$(DYNAMIC_RENDER_DIR)"
+	@python3 $(SCRIPTS_DIR)/includes_renderer.py ${WS_ROOT} ${DYNAMIC_RENDER_DIR}
+
+.PHONY: template-index
+template-index:
+	@sed -ie "s/__VPP_VERSION__/${VPP_VERSION}/g" ${DOCS_DIR}/index.rst
+	@sed -ie "s/__BUILT_ON__/${BUILT_ON}/g" ${DOCS_DIR}/index.rst
+
+.NOTPARALLEL: $(SIPHON_FILES)
+$(SIPHON_FILES): $(SCRIPTS_DIR)/siphon-generate \
+		$(addprefix,$(WSROOT),$(SIPHON_INPUT)) \
+		$(wildcard $(SCRIPTS_DIR)/siphon/*.py)
+	@echo "Validating source tree..."
+	@set -e; for input in $(SIPHON_INPUT); do \
+		if [ ! -e "$(WS_ROOT)/$$input" ]; then \
+			echo "ERROR: Input path '$$input' does not exist." >&2; \
+			exit 1; \
+		fi; \
+	done
+	@rm -rf "$(SIPHON_INPUT_DIR)" "$(DOCS_GENERATED_DIR)"
+	@mkdir -p "$(SIPHON_INPUT_DIR)" "$(DOCS_GENERATED_DIR)"
+	@touch $(SIPHON_INPUT_DIR)/files
+	@echo "Collating source file list for siphoning..."
+	@for input in $(SIPHON_INPUT); do \
+		cd "$(WS_ROOT)"; \
+		find "$$input" -type f \
+			\( -name '*.[ch]' -or -name '*.dox' \) -print \
+			| grep -v -E '^src/examples/' \
+			| grep -v -E '^$(SIPHON_EXCLUDE_REGEXP)' \
+			>> $(SIPHON_INPUT_DIR)/files; \
+	done
+	@echo "Generating siphons..."
+	@set -e; \
+	cd "$(WS_ROOT)"; \
+	$(SCRIPTS_DIR)/siphon-generate \
+		--output="$(SIPHON_INPUT_DIR)" \
+		"@$(SIPHON_INPUT_DIR)/files"
+
+# Evaluate this to build a siphon doc output target for each desired
+# output type:
+# $(eval $(call siphon-process,file_extension,output_type_name))
+define siphon-process
+$(DOCS_GENERATED_DIR)/%.$(1): $(SIPHON_INPUT_DIR)/%.siphon \
+		$(SCRIPTS_DIR)/siphon-process \
+		$(wildcard $(SCRIPTS_DIR)/siphon/*.py) \
+		$(wildcard $(SCRIPTS_DIR)/siphon_templates/$(2)/*/*.$(1))
+	@echo "Processing siphon for $(2) from $$(notdir $$<)..."
+	@set -e; \
+	cd "$(WS_ROOT)"; \
+	mkdir -p $(DOCS_GENERATED_DIR)/$$(basename $$(notdir $$<)).$(1).dir; \
+	$(SCRIPTS_DIR)/siphon-process \
+		--type=$$(basename $$(notdir $$<)) \
+		--format=$(2) \
+		--repolink=$(REPOSITORY_URL)/ \
+		--outdir=$(DOCS_GENERATED_DIR)/$$(basename $$(notdir $$<)).$(1).dir \
+		--output="$$@" \
+		"$$<"
+endef
+
+# Process the .siphon source fragments and render them into siphon flavored
+# markdown documentation
+.DELETE_ON_ERROR: $(SIPHON_DOCS)
+$(eval $(call siphon-process,rst,markdown))
+
+# This target can be used just to generate the siphoned things
+.PHONY: siphon
+siphon: $(SIPHON_DOCS)
+	@cp $(DOCS_GENERATED_DIR)/clicmd.rst $(DOCS_DIR)/cli-reference/index.rst
+	@cp -r $(DOCS_GENERATED_DIR)/clicmd.rst.dir $(DOCS_DIR)/cli-reference/clis
+
+.PHONY: generate
+generate: siphon includes-render template-index featurelist
+
+.PHONY: clean
+clean:
+	@rm -rf $(BR)/.siphon.dep
+	@rm -rf $(SCRIPTS_DIR)/siphon/__pycache__
+

docs/_scripts/includes_renderer.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+#  Copyright (c) 2020. Vinci Consulting Corp. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import glob
+import inspect
+import os
+import re
+import sys
+
+
+class ContentRenderer:
+    def __init__(self, ws_root, output_dir):
+        self.ws_root = ws_root
+        self.output_dir = output_dir
+
+    def plugin_dir(self):
+        return os.path.join(self.ws_root, "src/plugins")
+
+    def render(self):
+        raise NotImplementedError
+
+
+class PluginRenderer(ContentRenderer):
+
+    def _render_entry(self, output_file, entry):
+        description = "<no-description-found>"
+        # we use glob because a plugin can (ioam for now)
+        # define the plugin definition in
+        # a further subdirectory.
+        path = os.path.join(self.plugin_dir(), entry.name, '**')
+        for f in glob.iglob(path, recursive=True):
+            if not f.endswith('.c'):
+                continue
+            with open(f, "r", encoding="utf-8") as src:
+                for match in self.regex.finditer(src.read()):
+                    description = "%s" % (match.group(1))
+
+        output_file.write(f"* {entry.name} - {description}\n")
+
+    def render(self):
+        pattern = r'VLIB_PLUGIN_REGISTER\s?\(\)\s*=\s*{.*\.description\s?=\s?"([^"]*)".*};'  # noqa: 501
+        self.regex = re.compile(pattern, re.MULTILINE | re.DOTALL)
+        fname = os.path.join(self.output_dir, "plugin_list.inc")
+        with open(fname, "w") as output_file:
+            with os.scandir(self.plugin_dir()) as pdir:
+                for entry in sorted(pdir, key=lambda entry: entry.name):
+                    if not entry.name.startswith('.') and entry.is_dir():
+                        self._render_entry(output_file, entry)
+
+
+renderers = [PluginRenderer]
+
+
+def main():
+    if len(sys.argv) != 3:
+        print("You need to pass WS_ROOT and OUTPUT_DIR")
+        exit(1)
+
+    print("rendering dynamic includes...")
+    for renderer in renderers:
+        renderer(*sys.argv[1:]).render()
+    print("done.")
+
+
+if __name__ == "__main__":
+    main()

docs/_scripts/publish-docs.sh

@@ -72,7 +72,7 @@ cp -r $SRC_DIR $TARGET_DIR
 
 # Create the feature list
 pushd ..
-source ./sphinx_venv/bin/activate
+source ./docs/venv/bin/activate
 find . -name FEATURE.yaml | ./src/scripts/fts.py --markdown > site/content/vppProject/vppfeatures/features.md
 deactivate
 popd

docs/_scripts/siphon-process

@@ -30,6 +30,8 @@ DEFAULT_SIPHON = "clicmd"
 DEFAULT_FORMAT = "markdown"
 DEFAULT_OUTPUT = None
 DEFAULT_TEMPLATES = os.path.dirname(__file__) + "/siphon_templates"
+DEFAULT_OUTPUT_DIR = os.path.dirname(__file__) + "/siphon_docs"
+DEFAULT_REPO_LINK = "https://github.com/FDio/vpp/blob/master/"
 
 ap = argparse.ArgumentParser()
 ap.add_argument("--log-file", default=DEFAULT_LOGFILE,
@@ -50,6 +52,12 @@ ap.add_argument("--output", '-o', metavar="file", default=DEFAULT_OUTPUT,
 ap.add_argument("--templates", metavar="directory", default=DEFAULT_TEMPLATES,
                 help="Path to render templates directory [%s]" %
                      DEFAULT_TEMPLATES)
+ap.add_argument("--outdir", metavar="directory", default=DEFAULT_OUTPUT_DIR,
+                help="Path to output rendered parts [%s]" %
+                     DEFAULT_OUTPUT_DIR)
+ap.add_argument("--repolink", metavar="repolink", default=DEFAULT_REPO_LINK,
+                help="Link to public repository [%s]" %
+                     DEFAULT_REPO_LINK)
 ap.add_argument("input", nargs='+', metavar="input_file",
                 help="Input .siphon files")
 args = ap.parse_args()
@@ -66,7 +74,12 @@ else:
 
 # Get our processor
 klass = siphon.process.siphons[args.type]
-processor = klass(template_directory=args.templates, format=args.format)
+processor = klass(
+    template_directory=args.templates,
+    format=args.format,
+    outdir=args.outdir,
+    repository_link=args.repolink
+)
 
 # Load the input files
 processor.load_json(args.input)

docs/_scripts/siphon/process.py

@@ -19,6 +19,7 @@ import json
 import logging
 import os
 import sys
+import re
 
 import jinja2
 
@@ -57,10 +58,13 @@ class Siphon(object):
     """Directory to look for siphon rendering templates"""
     template_directory = None
 
+    """Directory to output parts in"""
+    outdir = None
+
     """Template environment, if we're using templates"""
     _tplenv = None
 
-    def __init__(self, template_directory, format):
+    def __init__(self, template_directory, format, outdir, repository_link):
         super(Siphon, self).__init__()
         self.log = logging.getLogger("siphon.process.%s" % self.name)
 
@@ -78,11 +82,12 @@ class Siphon(object):
             _tpldir(self.name),
             _tpldir("default"),
         ]
+        self.outdir = outdir
         loader = jinja2.FileSystemLoader(searchpath=searchpath)
         self._tplenv = jinja2.Environment(
             loader=loader,
             trim_blocks=True,
-            autoescape=True,
+            autoescape=False,
             keep_trailing_newline=True)
 
         # Convenience, get a reference to the internal escape and
@@ -92,6 +97,9 @@ class Siphon(object):
         self.escape = html.escape
         self.unescape = html.unescape
 
+        # TODO: customize release
+        self.repository_link = repository_link
+
     # Output renderers
 
     """Returns an object to be used as the sorting key in the item index."""
@@ -284,9 +292,90 @@ class Siphon(object):
                 # Generate the item itself (save for later)
                 contents += self.item_format(meta, o)
 
+            page_name = self.separate_page_names(group)
+            if page_name != "":
+                path = os.path.join(self.outdir, page_name)
+                with open(path, "w+") as page:
+                    page.write(contents)
+                contents = ""
+
         # Deliver the accumulated body output
         out.write(contents)
 
+    def do_cliexstart(self, matchobj):
+        title = matchobj.group(1)
+        title = ' '.join(title.splitlines())
+        content = matchobj.group(2)
+        content = re.sub(r"\n", r"\n    ", content)
+        return "\n\n.. code-block:: console\n\n    %s\n    %s\n\n" % (title, content)
+
+    def do_clistart(self, matchobj):
+        content = matchobj.group(1)
+        content = re.sub(r"\n", r"\n    ", content)
+        return "\n\n.. code-block:: console\n\n    %s\n\n" % content
+
+    def do_cliexcmd(self, matchobj):
+        content = matchobj.group(1)
+        content = ' '.join(content.splitlines())
+        return "\n\n.. code-block:: console\n\n    %s\n\n" % content
+
+    def process_list(self, matchobj):
+        content = matchobj.group(1)
+        content = self.reindent(content, 2)
+        return "@@@@%s\nBBBB" % content
+
+    def process_special(self, s):
+        # ----------- markers to remove
+        s = re.sub(r"@cliexpar\s*", r"", s)
+        s = re.sub(r"@parblock\s*", r"", s)
+        s = re.sub(r"@endparblock\s*", r"", s)
+        s = re.sub(r"<br>", "", s)
+        # ----------- emphasis
+        # <b><em>
+        s = re.sub(r"<b><em>\s*", "``", s)
+        s = re.sub(r"\s*</b></em>", "``", s)
+        s = re.sub(r"\s*</em></b>", "``", s)
+        # <b>
+        s = re.sub(r"<b>\s*", "**", s)
+        s = re.sub(r"\s*</b>", "**", s)
+        # <code>
+        s = re.sub(r"<code>\s*", "``", s)
+        s = re.sub(r"\s*</code>", "``", s)
+        # <em>
+        s = re.sub(r"'?<em>\s*", r"``", s)
+        s = re.sub(r"\s*</em>'?", r"``", s)
+        # @c <something>
+        s = re.sub(r"@c\s(\S+)", r"``\1``", s)
+        # ----------- todos
+        s = re.sub(r"@todo[^\n]*", "", s)
+        s = re.sub(r"@TODO[^\n]*", "", s)
+        # ----------- code blocks
+        s = re.sub(r"@cliexcmd{(.+?)}", self.do_cliexcmd, s, flags=re.DOTALL)
+        s = re.sub(r"@cliexstart{(.+?)}(.+?)@cliexend", self.do_cliexstart, s, flags=re.DOTALL)
+        s = re.sub(r"@clistart(.+?)@cliend", self.do_clistart, s, flags=re.DOTALL)
+        # ----------- lists
+        s = re.sub(r"^\s*-", r"\n@@@@", s, flags=re.MULTILINE)
+        s = re.sub(r"@@@@(.*?)\n\n+", self.process_list, s, flags=re.DOTALL)
+        s = re.sub(r"BBBB@@@@", r"-", s)
+        s = re.sub(r"@@@@", r"-", s)
+        s = re.sub(r"BBBB", r"\n\n", s)
+        # ----------- Cleanup remains
+        s = re.sub(r"@cliexend\s*", r"", s)
+        return s
+
+    def separate_page_names(self, group):
+        return ""
+
+    # This push the given textblock <indent> spaces right
+    def reindent(self, s, indent):
+        ind = " " * indent
+        s = re.sub(r"\n", "\n" + ind, s)
+        return s
+
+    # This aligns the given textblock left (no indent)
+    def noindent(self, s):
+        s = re.sub(r"\n[ \f\v\t]*", "\n", s)
+        return s
 
 class Format(object):
     """Output format class"""

docs/_scripts/siphon/process_clicmd.py

@@ -15,7 +15,7 @@
 # Generate clicmd formatted output
 
 from . import process, parsers
-
+import os
 
 class SiphonCLICMD(process.Siphon):
 
@@ -28,6 +28,9 @@ class SiphonCLICMD(process.Siphon):
 
     # Output renderers
 
+    def separate_page_names(self, group):
+        return self.page_label(group) + ".rst"
+
     def index_sort_key(self, group):
         _global = self._cmds['_global']
         if group not in self._group:
@@ -51,6 +54,20 @@ class SiphonCLICMD(process.Siphon):
             self.sanitize_label(self._cmds[group][item]['value']['path'])
         ))
 
+    def page_title(self, group):
+        _global = self._cmds['_global']
+        (directory, file) = self._group[group]
+
+        if file and file in _global and 'group_label' in _global[file]:
+            return _global[file]['group_label']
+
+        if directory in _global and 'group_label' in _global[directory]:
+            return _global[directory]['group_label']
+
+        file_ext = os.path.basename(directory)
+        fname, ext = os.path.splitext(file_ext)
+        return "%s cli reference" % fname.capitalize()
+
 
 # Register our processor
 process.siphons["clicmd"] = SiphonCLICMD

docs/_scripts/siphon_templates/markdown/clicmd/index_entry.md

@@ -12,6 +12,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#}
-{% set v = item['value'] %}
-{{ "* [%s](@ref %s)" % (v['path'], meta["label"]) }}
+#}

docs/_scripts/siphon_templates/markdown/clicmd/index_header.md

@@ -13,5 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #}
-{# Just output the command path #}
-{{ item['value']['path'] }}
+.. _cmdreference:
+
+Reference
+=========
+
+.. toctree::
+    :maxdepth: 2
+

docs/_scripts/siphon_templates/markdown/clicmd/index_section.md

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #}
-{{ raise NotImplementedError }}
+    clis/{{ this.page_label(group) }}

docs/_scripts/siphon_templates/markdown/clicmd/item_format.md

@@ -14,30 +14,36 @@
 # limitations under the License.
 #}
 {% set v = item['value'] %}
-{{ "@section %s %s" % (meta['label'], v['path']) }}
 {% if 'short_help' in v %}
-
-### Summary/usage
-
 {% set str = v['short_help'] %}
-{% set period = "." if str[-1] != "." else "" %}
-{% set prefix = "    " if "[" in str or "<" in str or "|" in str else "" %}
 {% set str = this.unescape(str) %}
-{{ "%s%s%s" % (prefix, str, period) }}
+
+{# Summary/usage #}
+{{ item['value']['path'] }}
+-------------------------------------------------------------------------
+
+.. code-block:: console
+
+    {{ this.reindent(str, 4) }}
+
 {% endif %}
 {% if 'long_help' in v %}
+{% set long_help = v['long_help'] %}
+{% set long_help = this.unescape(long_help) %}
 {# This is seldom used and will likely be deprecated #}
+{# Long help #}
+.. code-block:: console
 
-### Long help
+    {{ this.reindent(long_help, 4) }}
 
-{{ v['long_help'] }}
 {% endif %}
 {% if 'siphon_block' in item['meta'] %}
 {% set sb = item["meta"]["siphon_block"] %}
+{% set sb = this.process_special(sb) %}
 {% if sb %}
 {# Extracted from the code in /*? ... ?*/ blocks #}
 
-### Description
+{# Description #}
 
 {{ sb }}
 {% endif %}
@@ -45,15 +51,12 @@
 {% if 'name' in meta or 'function' in v %}
 {# Gives some developer-useful linking #}
 
-### Declaration and implementation
 {% if "name" in meta %}
-
-{{ "Declaration: @ref %s (@ref %s line %d)" %
-   (meta['name'], meta["file"], item["meta"]["line_start"]) }}
+Declaration: ``{{ meta['name'] }}`` `{{ meta["file"] }} line {{ item["meta"]["line_start"] }} <{{ this.repository_link }}{{ meta["file"] }}#L{{ item["meta"]["line_start"] }}>`_
 {% endif %}
 {% if "function" in v %}
 
-{{ "Implementation: @ref %s." % v["function"] }}
+Implementation: ``{{ v["function"] }}``
 {% endif %}
 {% endif %}