Add Asan support for clang on windows.

This will currently only work for the RelWithDebInfo configuration since asan
does not support the debug crt. for source line information in the reports,
you need a copy of llvm-symbolizer in the blender folder or set the
ASAN_SYMBOLIZER_PATH environment variable to point to it. Currently (as of
6.0.0) llvm-symbolizer does not ship with the binary clang/llvm distribution.

Reviewers: campbellbarton

Differential Revision: https://developer.blender.org/D3446

CMakeLists.txt

@@ -539,9 +539,7 @@ if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID MATCHES "Clang")
 -fsanitize=enum \
 -fsanitize=float-cast-overflow \
 -fsanitize=float-divide-by-zero \
--fsanitize=leak \
 -fsanitize=nonnull-attribute \
--fsanitize=object-size \
 -fsanitize=returns-nonnull-attribute \
 -fsanitize=signed-integer-overflow \
 -fsanitize=undefined \
@@ -549,6 +547,9 @@ if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID MATCHES "Clang")
 -fno-sanitize=alignment \
 ")
 
+		if(NOT MSVC) # not all sanitizers are supported with clang-cl, these two however are very vocal about it
+			set(_asan_defaults "${_asan_defaults} -fsanitize=leak -fsanitize=object-size" )
+		endif()
 		set(COMPILER_ASAN_CFLAGS "${_asan_defaults}" CACHE STRING "C flags for address sanitizer")
 		mark_as_advanced(COMPILER_ASAN_CFLAGS)
 		set(COMPILER_ASAN_CXXFLAGS "${_asan_defaults}" CACHE STRING "C++ flags for address sanitizer")
@@ -556,9 +557,15 @@ if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID MATCHES "Clang")
 
 		unset(_asan_defaults)
 
-		find_library(COMPILER_ASAN_LIBRARY asan ${CMAKE_C_IMPLICIT_LINK_DIRECTORIES})
+		if(NOT MSVC)
+			find_library(COMPILER_ASAN_LIBRARY asan ${CMAKE_C_IMPLICIT_LINK_DIRECTORIES})
+		else()
+			find_library( COMPILER_ASAN_LIBRARY NAMES clang_rt.asan-x86_64
+				PATHS 	[HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\LLVM\\LLVM;]/lib/clang/7.0.0/lib/windows
+						[HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\LLVM\\LLVM;]/lib/clang/6.0.0/lib/windows
+						)
+		endif() 
 		mark_as_advanced(COMPILER_ASAN_LIBRARY)
-
 	endif()
 endif()
 
@@ -863,8 +870,12 @@ if(WITH_COMPILER_ASAN)
 
 	set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMPILER_ASAN_CXXFLAGS}")
 	set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${COMPILER_ASAN_CXXFLAGS}")
-
-	set(PLATFORM_LINKFLAGS_DEBUG "${COMPILER_ASAN_LIBRARY}")
+	if(MSVC)
+		set(COMPILER_ASAN_LINKER_FLAGS "/FUNCTIONPADMIN:6") 
+	endif()
+	set(PLATFORM_LINKLIBS "${PLATFORM_LINKLIBS};${COMPILER_ASAN_LIBRARY}")
+	set(PLATFORM_LINKFLAGS "${COMPILER_ASAN_LIBRARY} ${COMPILER_ASAN_LINKER_FLAGS}")
+	set(PLATFORM_LINKFLAGS_DEBUG "${COMPILER_ASAN_LIBRARY} ${COMPILER_ASAN_LINKER_FLAGS}") 
 endif()
 
 #-----------------------------------------------------------------------------

build_files/cmake/macros.cmake

@@ -352,6 +352,11 @@ function(SETUP_LIBDIRS)
 	endif()
 endfunction()
 
+macro(setup_platform_linker_flags)
+	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${PLATFORM_LINKFLAGS}")
+	set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} ${PLATFORM_LINKFLAGS_DEBUG}")
+endmacro()
+
 function(setup_liblinks
 	target
 	)

build_files/cmake/platform/platform_win32.cmake

@@ -149,7 +149,7 @@ set(CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL} /MT")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /MT")
 set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} /MT")
 
-set(PLATFORM_LINKFLAGS "/SUBSYSTEM:CONSOLE /STACK:2097152 /INCREMENTAL:NO ")
+set(PLATFORM_LINKFLAGS "${PLATFORM_LINKFLAGS} /SUBSYSTEM:CONSOLE /STACK:2097152 /INCREMENTAL:NO ")
 set(PLATFORM_LINKFLAGS "${PLATFORM_LINKFLAGS} /NODEFAULTLIB:msvcrt.lib /NODEFAULTLIB:msvcmrt.lib /NODEFAULTLIB:msvcurt.lib /NODEFAULTLIB:msvcrtd.lib ")
 
 # Ignore meaningless for us linker warnings.
@@ -162,7 +162,7 @@ else()
 	set(PLATFORM_LINKFLAGS "/MACHINE:IX86 /LARGEADDRESSAWARE ${PLATFORM_LINKFLAGS}")
 endif()
 
-set(PLATFORM_LINKFLAGS_DEBUG "/IGNORE:4099 /NODEFAULTLIB:libcmt.lib /NODEFAULTLIB:libc.lib")
+set(PLATFORM_LINKFLAGS_DEBUG "${PLATFORM_LINKFLAGS_DEBUG} /IGNORE:4099 /NODEFAULTLIB:libcmt.lib /NODEFAULTLIB:libc.lib")
 
 if(NOT DEFINED LIBDIR)
 

build_files/windows/configure_msbuild.cmd

@@ -12,6 +12,14 @@ if "%BUILD_ARCH%"=="x64" (
 
 if "%WITH_CLANG%"=="1" (
 	set BUILD_CMAKE_ARGS=%BUILD_CMAKE_ARGS% -T"LLVM-vs2017"
+	if "%WITH_ASAN%"=="1" (
+		set BUILD_CMAKE_ARGS=%BUILD_CMAKE_ARGS% -DWITH_COMPILER_ASAN=On
+	)
+) else (
+	if "%WITH_ASAN%"=="1" (
+		echo ASAN is only supported with clang.
+		exit /b 1 
+	)
 )
 
 if NOT EXIST %BUILD_DIR%\nul (

build_files/windows/configure_ninja.cmd

@@ -27,6 +27,16 @@ set LLVM_DIR=
 	rem build and tested against 2017 15.7
 	set CFLAGS=-m64 -fmsc-version=1914
 	set CXXFLAGS=-m64 -fmsc-version=1914
+	if "%WITH_ASAN%"=="1" (
+		set BUILD_CMAKE_ARGS=%BUILD_CMAKE_ARGS% -DWITH_COMPILER_ASAN=On
+	)	
+)
+
+if "%WITH_ASAN%"=="1" (
+	if "%WITH_CLANG%" == "" (
+		echo ASAN is only supported with clang.
+		exit /b 1 
+	)
 )
 
 if NOT "%verbose%" == "" (

build_files/windows/parse_arguments.cmd

@@ -42,6 +42,8 @@ if NOT "%1" == "" (
 	) else if "%1" == "release" (
 		set BUILD_CMAKE_ARGS=%BUILD_CMAKE_ARGS% -C"%BLENDER_DIR%\build_files\cmake\config\blender_release.cmake"
 		set TARGET=Release
+	) else if "%1" == "asan" (
+		set WITH_ASAN=1
 	) else if "%1" == "x86" (
 		set BUILD_ARCH=x86
 	) else if "%1" == "x64" (

build_files/windows/reset_variables.cmd

@@ -21,4 +21,5 @@ set BUILD_UPDATE=
 set BUILD_SHOW_HASHES=
 set SHOW_HELP=
 set BUILD_WITH_NINJA=
-set WITH_CLANG=
+set WITH_CLANG=
+set WITH_ASAN=

source/blender/datatoc/CMakeLists.txt

@@ -37,7 +37,7 @@ if(NOT WITH_HEADLESS)
 	set(SRC
 		datatoc_icon.c
 	)
-
+	setup_platform_linker_flags()
 	if(WIN32)
 		include_directories(
 			../blenlib
@@ -52,8 +52,6 @@ if(NOT WITH_HEADLESS)
 			../../../intern/utfconv/utfconv.c
 		)
 
-		set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${PLATFORM_LINKFLAGS}")
-		set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} ${PLATFORM_LINKFLAGS_DEBUG}")
 	endif()
 
 	include_directories(${PNG_INCLUDE_DIRS})

source/blender/makesdna/intern/CMakeLists.txt

@@ -53,6 +53,7 @@ endif()
 # SRC_DNA_INC is defined in the parent dir
 
 add_cc_flags_custom_test(makesdna)
+setup_platform_linker_flags()
 
 add_executable(makesdna ${SRC} ${SRC_DNA_INC})
 

source/blender/makesrna/intern/CMakeLists.txt

@@ -356,6 +356,7 @@ blender_include_dirs_sys(
 )
 
 add_cc_flags_custom_test(makesrna)
+setup_platform_linker_flags()
 
 add_executable(makesrna ${SRC} ${SRC_RNA_INC} ${SRC_DNA_INC})