tools: FEATURE.yaml meta-data infrastructure

Add tooling for feature metadata configuration files.
The main tool is in src/scripts/fts.py

make checkfeaturelist to validate against schema.
make featurelist to dump all feature lists to stdout.

Example feature definition:

name: IP in IP tunnelling
maintainer: Ole Troan <ot@cisco.com>
features:
  - IPv4/IPv6 over IPv4/IPv6 encapsulation:
    - Fragmentation and Reassembly
    - Configurable MTU
    - Inner to outer Traffic Class / TOS copy
    - Configurable Traffic Class / TOS
  - ICMPv4 / ICMPv6 proxying
  - 6RD (RFC5969):
    - Border Relay

description: "Implements IP{v4,v6} over IP{v4,v6} tunnelling as
              described in RFC2473. This module also implement the border relay of
	      6RD (RFC5969)."

state: production
properties: [API, CLI, STATS, MULTITHREAD]
missing:
  - Tunnel PMTUD
  - Tracking of FIB state for tunnel state
  - IPv6 extension headers (Tunnel encapsulation limit option)

JSON schema is embedded in fts.py

Example markdown: https://github.com/otroan/scratch/blob/master/features.md

Change-Id: I903b4ee6b316a9378c259e86dc937092e5d4b7da
Type: make
Signed-off-by: Ole Troan <ot@cisco.com>

Makefile

@@ -67,7 +67,7 @@ DEB_DEPENDS += lcov chrpath autoconf indent clang-format libnuma-dev
 DEB_DEPENDS += python-all python3-all python3-setuptools python-dev
 DEB_DEPENDS += python-virtualenv python-pip libffi6 check
 DEB_DEPENDS += libboost-all-dev libffi-dev python3-ply libmbedtls-dev
-DEB_DEPENDS += cmake ninja-build uuid-dev
+DEB_DEPENDS += cmake ninja-build uuid-dev python3-jsonschema
 ifeq ($(OS_VERSION_ID),14.04)
 	DEB_DEPENDS += libssl-dev
 else ifeq ($(OS_ID)-$(OS_VERSION_ID),debian-8)
@@ -94,7 +94,7 @@ ifeq ($(OS_ID),fedora)
 	RPM_DEPENDS += subunit subunit-devel
 	RPM_DEPENDS += compat-openssl10-devel
 	RPM_DEPENDS += python3-devel python3-ply
-	RPM_DEPENDS += python3-virtualenv
+	RPM_DEPENDS += python3-virtualenv python3-jsonschema
 	RPM_DEPENDS += cmake
 	RPM_DEPENDS_GROUPS = 'C Development Tools and Libraries'
 else
@@ -102,7 +102,7 @@ else
 	RPM_DEPENDS += openssl-devel
 	RPM_DEPENDS += python-devel python36-ply
 	RPM_DEPENDS += python36-devel python36-pip
-	RPM_DEPENDS += python-virtualenv
+	RPM_DEPENDS += python-virtualenv python36-jsonschema
 	RPM_DEPENDS += devtoolset-7
 	RPM_DEPENDS += cmake3
 	RPM_DEPENDS_GROUPS = 'Development Tools'
@@ -216,6 +216,8 @@ help:
 	@echo " doxygen             - (re)generate documentation"
 	@echo " bootstrap-doxygen   - setup Doxygen dependencies"
 	@echo " wipe-doxygen        - wipe all generated documentation"
+	@echo " checkfeaturelist    - check FEATURE.yaml according to schema"
+	@echo " featurelist         - dump feature list in markdown"
 	@echo " docs                 - Build the Sphinx documentation"
 	@echo " docs-venv         - Build the virtual environment for the Sphinx docs"
 	@echo " docs-clean        - Remove the generated files from the Sphinx docs"
@@ -544,6 +546,12 @@ checkstyle:
 fixstyle:
 	@build-root/scripts/checkstyle.sh --fix
 
+featurelist:
+	@build-root/scripts/fts.py --all --markdown
+
+checkfeaturelist:
+	@build-root/scripts/fts.py --validate --git-status
+
 #
 # Build the documentation
 #

build-root/scripts/fts.py

@@ -0,0 +1 @@
+../../src/scripts/fts.py

src/plugins/flowprobe/FEATURE.yaml

@@ -0,0 +1,13 @@
+name: IPFIX probe
+maintainer: Ole Troan <ot@cisco.com>
+features:
+  - L2 input feature
+  - IPv4 / IPv6 input feature
+  - Recording of L2, L3 and L4 information
+description: "IPFIX flow probe. Works in the L2, or IP input feature path."
+missing:
+ - Output path
+ - Export over IPv6
+ - Export over TCP/SCTP
+state: production
+properties: [API, CLI, STATS, MULTITHREAD]

src/plugins/map/FEATURE.yaml

@@ -0,0 +1,13 @@
+name: Mapping of Address and Port (MAP)
+maintainer: Ole Troan <ot@cisco.com>
+features:
+  - LW46 BR (RFC7596):
+    - Fragmentation and Reassembly
+  - MAP-E BR (RFC7597)
+  - MAP-T BR (RFC7599)
+description: "IPv4 as a service mechanisms. Tunnel or translate
+              an IPv4 address, an IPv4 subnet or a shared IPv4 address.
+              In shared IPv4 address mode, only UDP, TCP and restricted
+              ICMP is supported."
+state: production
+properties: [API, CLI, STATS, MULTITHREAD]

src/scripts/fts.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import ipaddress
+import yaml
+from pprint import pprint
+import re
+from jsonschema import validate
+import argparse
+from subprocess import run, PIPE
+
+# VPP feature JSON schema
+schema = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "properties": {
+        "name": {"type": "string"},
+        "description": { "type": "string" },
+        "maintainer": { "type": "string" },
+        "state": {"type": "string",
+                  "enum": ["production", "experimental"]},
+        "features": { "$ref": "#/definitions/features" },
+        "missing": { "$ref": "#/definitions/features" },
+        "properties": { "type": "array",
+                       "items": { "type": "string",
+                                  "enum": ["API", "CLI", "STATS", "MULTITHREAD"] },
+                       },
+    },
+    "additionalProperties": False,
+    "definitions": {
+        "featureobject": {
+            "type": "object",
+            "patternProperties": {
+                "^.*$": { "$ref": "#/definitions/features" },
+            },
+        },
+        "features": {
+            "type": "array",
+            "items": {"anyOf": [{ "$ref": "#/definitions/featureobject" },
+                      { "type": "string" },
+            ]},
+            "minItems": 1,
+        },
+    },
+}
+
+
+
+def filelist_from_git_status():
+    filelist = []
+    git_status = 'git status --porcelain */FEATURE.yaml'
+    rv = run(git_status.split(), stdout=PIPE, stderr=PIPE)
+    if rv.returncode != 0:
+        sys.exit(rv.returncode)
+
+    for l in rv.stdout.decode('ascii').split('\n'):
+        if len(l):
+            filelist.append(l.split()[1])
+    return filelist
+
+def filelist_from_git_ls():
+    filelist = []
+    git_ls = 'git ls-files :(top)*/FEATURE.yaml'
+    rv = run(git_ls.split(), stdout=PIPE, stderr=PIPE)
+    if rv.returncode != 0:
+        sys.exit(rv.returncode)
+
+    for l in rv.stdout.decode('ascii').split('\n'):
+        if len(l):
+            filelist.append(l)
+    return filelist
+
+def output_features(indent, fl):
+    for f in fl:
+        if type(f) is dict:
+            for k,v in f.items():
+                print('{}- {}'.format(' ' * indent, k))
+                output_features(indent + 2, v)
+        else:
+            print('{}- {}'.format(' ' * indent, f))
+
+def output_markdown(features):
+    for k,v in features.items():
+        print('# {}'.format(v['name']))
+        print('Maintainer: {}  '.format(v['maintainer']))
+        print('State: {}\n'.format(v['state']))
+        print('{}\n'.format(v['description']))
+        output_features(0, v['features'])
+        if 'missing' in v:
+            print('\n## Missing')
+            output_features(0, v['missing'])
+        print()
+
+def main():
+    parser = argparse.ArgumentParser(description='VPP Feature List.')
+    parser.add_argument('--validate', dest='validate', action='store_true',
+                        help='validate the FEATURE.yaml file')
+    parser.add_argument('--git-status', dest='git_status', action='store_true',
+                        help='Get filelist from git status')
+    parser.add_argument('--all', dest='all', action='store_true',
+                        help='Validate all files in repository')
+    parser.add_argument('--markdown', dest='markdown', action='store_true',
+                        help='Output feature table in markdown')
+    parser.add_argument('infile', nargs='?', type=argparse.FileType('r'),
+                        default=sys.stdin)
+    args = parser.parse_args()
+
+    features = {}
+
+    if args.git_status:
+        filelist = filelist_from_git_status()
+    elif args.all:
+        filelist = filelist_from_git_ls()
+    else:
+        filelist = args.infile
+
+    for featurefile in filelist:
+        featurefile = featurefile.rstrip()
+
+        # Load configuration file
+        with open(featurefile) as f:
+            cfg = yaml.load(f)
+        validate(instance=cfg, schema=schema)
+        features[featurefile] = cfg
+
+    if args.markdown:
+        output_markdown(features)
+
+if __name__ == '__main__':
+    main()

src/vnet/ipip/FEATURE.yaml

@@ -0,0 +1,22 @@
+name: IP in IP tunnelling
+maintainer: Ole Troan <ot@cisco.com>
+features:
+  - IPv4/IPv6 over IPv4/IPv6 encapsulation:
+    - Fragmentation and Reassembly
+    - Configurable MTU
+    - Inner to outer Traffic Class / TOS copy
+    - Configurable Traffic Class / TOS
+  - ICMPv4 / ICMPv6 proxying
+  - 6RD (RFC5969):
+    - Border Relay
+
+description: "Implements IP{v4,v6} over IP{v4,v6} tunnelling as
+              described in RFC2473. This module also implement the border relay of
+	      6RD (RFC5969)."
+
+state: production
+properties: [API, CLI, STATS, MULTITHREAD]
+missing:
+  - Tunnel PMTUD
+  - Tracking of FIB state for tunnel state
+  - IPv6 extension headers (Tunnel encapsulation limit option)