build: vppapigen dependency handling

Add dependency generation to the vppapigen compiler, so that
when an API file depends on another, that's registered as a dependency
with the build system.

Add a build dependency on vppapigen submodules so that all api files
are regenerated if the compiler itself changes.

Type: improvement
Change-Id: I392853754129778ef15532d1b04813786b943b44
Signed-off-by: Ole Troan <otroan@employees.org>

src/cmake/api.cmake

@@ -11,14 +11,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Set the CMP0116 policy
+if(POLICY CMP0116)
+    cmake_policy(SET CMP0116 NEW)
+endif()
+
 ##############################################################################
 # API
 ##############################################################################
 function(vpp_generate_api_c_header file)
   set (output_name ${CMAKE_CURRENT_BINARY_DIR}/${file}.h)
+  set (dependency_file ${CMAKE_CURRENT_BINARY_DIR}/${file}.d)
   get_filename_component(output_dir ${output_name} DIRECTORY)
   if(NOT VPP_APIGEN)
-     set(VPP_APIGEN ${CMAKE_SOURCE_DIR}/tools/vppapigen/vppapigen)
+    set(VPP_APIGEN ${CMAKE_SOURCE_DIR}/tools/vppapigen/vppapigen)
+    set(VPPAPIGEN_SUBMODULES
+      ${CMAKE_SOURCE_DIR}/tools/vppapigen/vppapigen_c.py
+      ${CMAKE_SOURCE_DIR}/tools/vppapigen/vppapigen_json.py
+    )
   endif()
   if (VPP_INCLUDE_DIR)
     set(includedir "--includedir" ${VPP_INCLUDE_DIR})
@@ -35,16 +45,19 @@ function(vpp_generate_api_c_header file)
     "${CMAKE_CURRENT_BINARY_DIR}/${file}_test2.c"
   )
 
+  get_filename_component(barename ${file} NAME)
+
   add_custom_command (
     OUTPUT ${OUTPUT_HEADERS}
     COMMAND mkdir -p ${output_dir}
     COMMAND ${PYENV} ${VPP_APIGEN}
-    ARGS ${includedir} --includedir ${CMAKE_SOURCE_DIR} --input ${CMAKE_CURRENT_SOURCE_DIR}/${file} --outputdir ${output_dir} --output ${output_name}
-    DEPENDS ${VPP_APIGEN} ${CMAKE_CURRENT_SOURCE_DIR}/${file}
+    ARGS ${includedir} --includedir ${CMAKE_SOURCE_DIR} --input ${CMAKE_CURRENT_SOURCE_DIR}/${file} --outputdir ${output_dir} --output ${output_name} -MF ${dependency_file}
+    DEPENDS ${VPP_APIGEN} ${CMAKE_CURRENT_SOURCE_DIR}/${file} ${VPPAPIGEN_SUBMODULES}
     COMMENT "Generating API header ${output_name}"
+    DEPFILE ${dependency_file}
   )
-  get_filename_component(barename ${file} NAME)
   set(t ${barename}_deps)
+
   if (NOT TARGET ${t})
     add_custom_target(${t} ALL DEPENDS ${OUTPUT_HEADERS})
     add_dependencies(api_headers ${t})
@@ -158,6 +171,7 @@ function(vpp_add_api_files name dir component)
     # "vapi". Both by in-source components (e.g. vpp-api/vapi/vapi.c), and
     # out-of-tree plugins use #include <vapi/component.api.vapi.h>.
     # ${file} contains the subdirectory, so strip it here.
+    file(MAKE_DIRECTORY ${VPP_BINARY_DIR}/vpp-api/vapi)
     get_filename_component(name ${file} NAME)
     list(APPEND header_files
       ${file}.h
@@ -175,3 +189,5 @@ endfunction()
 add_custom_target(api_headers
   DEPENDS vlibmemory_api_headers vnet_api_headers vpp_api_headers vlib_api_headers
 )
+add_custom_target(vapi_headers
+)

src/tools/vppapigen/vppapigen.py

@@ -9,6 +9,7 @@ import os
 from subprocess import Popen, PIPE
 import ply.lex as lex
 import ply.yacc as yacc
+from io import TextIOWrapper
 
 assert sys.version_info >= (3, 5), "Not supported Python version: {}".format(
     sys.version
@@ -1166,6 +1167,21 @@ def foldup_crcs(s):
         f.crc = foldup_blocks(f.block, binascii.crc32(f.crc) & 0xFFFFFFFF)
 
 
+def write_dependencies(output_file, dependency_file, imports):
+    r = []
+    for i in imports:
+        for d in dirlist:
+            f = os.path.abspath(os.path.join(d, i.filename))
+            if os.path.exists(f):
+                r.append(f)
+    with open(dependency_file, "w", encoding="utf8") as f:
+        print(f"{output_file}: \\", file=f)
+        for i in r[:-1]:
+            print(f" {i} \\", file=f)
+        if imports:
+            print(f" {r[-1]}", file=f)
+
+
 def run_vppapigen(
     input_file=None,
     output=sys.stdout,
@@ -1176,6 +1192,7 @@ def run_vppapigen(
     outputdir=None,
     pluginpath="",
     git_revision=None,
+    dependency_file=None,
 ):
     # reset globals
     dirlist.clear()
@@ -1256,6 +1273,9 @@ def run_vppapigen(
         imports = parser.process_imports(parsed_objects, False, result)
         s["imported"] = parser.process(imports)
 
+    if dependency_file and isinstance(output, TextIOWrapper):
+        write_dependencies(output.name, dependency_file[0], s["Import"])
+
     # Add msg_id field
     s["Define"] = add_msg_id(s["Define"])
 
@@ -1324,6 +1344,7 @@ def main():
     cliparser.add_argument(
         "--git-revision", help="Git revision to use for opening files"
     )
+    cliparser.add_argument("-MF", nargs=1, help="Dependency file")
     args = cliparser.parse_args()
 
     return run_vppapigen(
@@ -1336,6 +1357,7 @@ def main():
         pluginpath=args.pluginpath,
         git_revision=args.git_revision,
         output=args.output,
+        dependency_file=args.MF,
     )