Eevee: add regression tests.

This adds Eevee render tests using the Cycles files. Currently it must
be enabled by setting WITH_OPENGL_RENDER_TESTS=ON. Once we have reference
images we can enable it by default.

Some of the Cycles and Eevee tests are also currently broken due to
modifier and particle changes.

Differential Revision: https://developer.blender.org/D3182

tests/python/CMakeLists.txt

@@ -531,9 +531,9 @@ function(add_python_test testname testscript)
 	endif()
 endfunction()
 
-if(WITH_CYCLES)
-	if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/render/ctests/shader")
-		macro(add_cycles_render_test subject)
+if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/render/ctests/shader")
+	macro(add_cycles_render_test subject)
+		if(WITH_CYCLES)
 			add_python_test(
 				cycles_${subject}_test
 				${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py
@@ -542,31 +542,39 @@ if(WITH_CYCLES)
 				-idiff "${OPENIMAGEIO_IDIFF}"
 				-outdir "${TEST_OUT_DIR}/cycles"
 			)
-		endmacro()
-		if(WITH_OPENGL_RENDER_TESTS)
-			add_cycles_render_test(opengl)
 		endif()
-		add_cycles_render_test(bake)
-		add_cycles_render_test(bsdf)
-		add_cycles_render_test(denoise)
-		add_cycles_render_test(displacement)
-		add_cycles_render_test(hair)
-		add_cycles_render_test(image_data_types)
-		add_cycles_render_test(image_mapping)
-		add_cycles_render_test(image_texture_limit)
-		add_cycles_render_test(integrator)
-		add_cycles_render_test(light)
-		add_cycles_render_test(mesh)
-		add_cycles_render_test(motion_blur)
-		add_cycles_render_test(render_layer)
-		add_cycles_render_test(reports)
-		add_cycles_render_test(shader)
-		add_cycles_render_test(shadow_catcher)
-		add_cycles_render_test(sss)
-		add_cycles_render_test(volume)
-	else()
-		MESSAGE(STATUS "Disabling Cycles tests because tests folder does not exist")
-	endif()
+
+		if(WITH_OPENGL_RENDER_TESTS AND (NOT ${subject} MATCHES "bake"))
+			add_python_test(
+				eevee_${subject}_test
+				${CMAKE_CURRENT_LIST_DIR}/eevee_render_tests.py
+				-blender "$<TARGET_FILE:blender>"
+				-testdir "${TEST_SRC_DIR}/render/ctests/${subject}"
+				-idiff "${OPENIMAGEIO_IDIFF}"
+				-outdir "${TEST_OUT_DIR}/eevee"
+			)
+		endif()
+	endmacro()
+	add_cycles_render_test(bake)
+	add_cycles_render_test(bsdf)
+	add_cycles_render_test(denoise)
+	add_cycles_render_test(displacement)
+	add_cycles_render_test(hair)
+	add_cycles_render_test(image_data_types)
+	add_cycles_render_test(image_mapping)
+	add_cycles_render_test(image_texture_limit)
+	add_cycles_render_test(integrator)
+	add_cycles_render_test(light)
+	add_cycles_render_test(mesh)
+	add_cycles_render_test(motion_blur)
+	add_cycles_render_test(render_layer)
+	add_cycles_render_test(reports)
+	add_cycles_render_test(shader)
+	add_cycles_render_test(shadow_catcher)
+	add_cycles_render_test(sss)
+	add_cycles_render_test(volume)
+elseif(WITH_CYCLES)
+	MESSAGE(STATUS "Disabling Cycles tests because tests folder does not exist")
 endif()
 
 if(WITH_OPENGL_DRAW_TESTS)

tests/python/cycles_render_tests.py

@@ -77,6 +77,7 @@ def render_file(filepath, output_filepath):
             shutil.copy(frame_filepath, output_filepath)
             os.remove(frame_filepath)
         if VERBOSE:
+            print(" ".join(command))
             print(output.decode("utf-8"))
         return None
     except subprocess.CalledProcessError as e:
@@ -84,6 +85,7 @@ def render_file(filepath, output_filepath):
         if os.path.exists(frame_filepath):
             os.remove(frame_filepath)
         if VERBOSE:
+            print(" ".join(command))
             print(e.output.decode("utf-8"))
         if b"Error: engine not found" in e.output:
             return "NO_ENGINE"
@@ -95,6 +97,7 @@ def render_file(filepath, output_filepath):
         if os.path.exists(frame_filepath):
             os.remove(frame_filepath)
         if VERBOSE:
+            print(" ".join(command))
             print(e)
         return "CRASH"
 

tests/python/eevee_render_tests.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+# Apache License, Version 2.0
+
+import argparse
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+
+def setup():
+    import bpy
+
+    # Enable Eevee features
+    scene = bpy.context.scene
+    eevee = scene.eevee
+
+    eevee.sss_enable = True
+    eevee.ssr_enable = True
+    eevee.ssr_refraction = True
+    eevee.gtao_enable = True
+    eevee.dof_enable = True
+
+    eevee.volumetric_enable = True
+    eevee.volumetric_shadows = True
+    eevee.volumetric_tile_size = '2'
+
+    for mat in bpy.data.materials:
+        mat.use_screen_refraction = True
+        mat.use_screen_subsurface = True
+
+# When run from inside Blender, render and exit.
+try:
+    import bpy
+    inside_blender = True
+except ImportError:
+    inside_blender = False
+
+if inside_blender:
+    try:
+        setup()
+    except Exception as e:
+        print(e)
+        sys.exit(1)
+
+
+def render_file(filepath, output_filepath):
+    dirname = os.path.dirname(filepath)
+    basedir = os.path.dirname(dirname)
+    subject = os.path.basename(dirname)
+
+    frame_filepath = output_filepath + '0001.png'
+
+    command = [
+        BLENDER,
+        "--background",
+        "-noaudio",
+        "--factory-startup",
+        "--enable-autoexec",
+        filepath,
+        "-E", "BLENDER_EEVEE",
+        "-P",
+        os.path.realpath(__file__),
+        "-o", output_filepath,
+        "-F", "PNG",
+        "-f", "1"]
+
+    try:
+        # Success
+        output = subprocess.check_output(command)
+        if os.path.exists(frame_filepath):
+            shutil.copy(frame_filepath, output_filepath)
+            os.remove(frame_filepath)
+        if VERBOSE:
+            print(" ".join(command))
+            print(output.decode("utf-8"))
+        return None
+    except subprocess.CalledProcessError as e:
+        # Error
+        if os.path.exists(frame_filepath):
+            os.remove(frame_filepath)
+        if VERBOSE:
+            print(" ".join(command))
+            print(e.output.decode("utf-8"))
+        if b"Error: engine not found" in e.output:
+            return "NO_ENGINE"
+        elif b"blender probably wont start" in e.output:
+            return "NO_START"
+        return "CRASH"
+    except BaseException as e:
+        # Crash
+        if os.path.exists(frame_filepath):
+            os.remove(frame_filepath)
+        if VERBOSE:
+            print(" ".join(command))
+            print(e)
+        return "CRASH"
+
+
+def create_argparse():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-blender", nargs="+")
+    parser.add_argument("-testdir", nargs=1)
+    parser.add_argument("-outdir", nargs=1)
+    parser.add_argument("-idiff", nargs=1)
+    return parser
+
+
+def main():
+    parser = create_argparse()
+    args = parser.parse_args()
+
+    global BLENDER, VERBOSE
+
+    BLENDER = args.blender[0]
+    VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
+
+    test_dir = args.testdir[0]
+    idiff = args.idiff[0]
+    output_dir = args.outdir[0]
+
+    from modules import render_report
+    report = render_report.Report("Eevee Test Report", output_dir, idiff)
+    report.set_pixelated(True)
+    report.set_reference_dir("eevee_renders")
+    ok = report.run(test_dir, render_file)
+
+    sys.exit(not ok)
+
+
+if not inside_blender and __name__ == "__main__":
+    main()

tests/python/modules/render_report.py

@@ -59,11 +59,11 @@ def test_get_name(filepath):
     filename = os.path.basename(filepath)
     return os.path.splitext(filename)[0]
 
-def test_get_images(output_dir, filepath):
+def test_get_images(output_dir, filepath, reference_dir):
     testname = test_get_name(filepath)
     dirpath = os.path.dirname(filepath)
 
-    old_dirpath = os.path.join(dirpath, "reference_renders")
+    old_dirpath = os.path.join(dirpath, reference_dir)
     old_img = os.path.join(old_dirpath, testname + ".png")
 
     ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref")
@@ -90,6 +90,7 @@ class Report:
     __slots__ = (
         'title',
         'output_dir',
+        'reference_dir',
         'idiff',
         'pixelated',
         'verbose',
@@ -101,6 +102,7 @@ class Report:
     def __init__(self, title, output_dir, idiff):
         self.title = title
         self.output_dir = output_dir
+        self.reference_dir = 'reference_renders'
         self.idiff = idiff
 
         self.pixelated = False
@@ -120,6 +122,9 @@ class Report:
     def set_pixelated(self, pixelated):
         self.pixelated = pixelated
 
+    def set_reference_dir(self, reference_dir):
+        self.reference_dir = reference_dir
+
     def run(self, dirpath, render_cb):
         # Run tests and output report.
         dirname = os.path.basename(dirpath)
@@ -229,7 +234,7 @@ class Report:
         name = test_get_name(filepath)
         name = name.replace('_', ' ')
 
-        old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath)
+        old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir)
 
         status = error if error else ""
         tr_style = """ style="background-color: #f99;" """ if error else ""
@@ -259,7 +264,7 @@ class Report:
 
 
     def _diff_output(self, filepath, tmp_filepath):
-        old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath)
+        old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir)
 
         # Create reference render directory.
         old_dirpath = os.path.dirname(old_img)