Lock-free memory allocator

Release builds will now use lock-free allocator by
default without any internal locks happening.

MemHead is also reduces to as minimum as it's possible.
It still need to be size_t stored in a MemHead in order
to make us keep track on memory we're requesting from
the system, not memory which system is allocating. This
is probably also faster than using a malloc's usable
size function.

Lock-free guarded allocator will say you whether all
the blocks were freed, but wouldn't give you a list
of unfreed blocks list. To have such a list use a
--debug or --debug-memory command line arguments.

Debug builds does have the same behavior as release
builds. This is so tools like valgrind are not
screwed up by guarded allocator as they're currently
are.

--
svn merge -r59941:59942 -r60072:60073 -r60093:60094 \
          -r60095:60096 ^/branches/soc-2013-depsgraph_mt

CMakeLists.txt

@@ -287,9 +287,6 @@ mark_as_advanced(WITH_MEM_VALGRIND)
 option(WITH_CXX_GUARDEDALLOC "Enable GuardedAlloc for C++ memory allocation tracking (only enable for development)" OFF)
 mark_as_advanced(WITH_CXX_GUARDEDALLOC)
 
-option(WITH_GUARDEDALLOC "Enable GuardedAlloc (DISABLE AT OWN RISK!)" ON)
-mark_as_advanced(WITH_GUARDEDALLOC)
-
 option(WITH_ASSERT_ABORT "Call abort() when raising an assertion through BLI_assert()" OFF)
 mark_as_advanced(WITH_ASSERT_ABORT)
 

intern/guardedalloc/CMakeLists.txt

@@ -34,11 +34,11 @@ set(INC_SYS
 
 set(SRC
 	./intern/mallocn.c
+	./intern/mallocn_guarded_impl.c
+	./intern/mallocn_lockfree_impl.c
 
 	MEM_guardedalloc.h
-
+	./intern/mallocn_intern.h
-	# include here since its a header-only
-	../atomic/atomic_ops.h
 )
 
 if(WIN32 AND NOT UNIX)
@@ -49,12 +49,6 @@ if(WIN32 AND NOT UNIX)
 	)
 endif()
 
-if (WITH_GUARDEDALLOC)
-	add_definitions(-DWITH_GUARDEDALLOC)
-else()
-	message(WARNING "Disabling GuardedAlloc is experemental, use at own risk!")
-endif()
-
 blender_add_lib(bf_intern_guardedalloc "${SRC}" "${INC}" "${INC_SYS}")
 
 # Override C++ alloc, optional.

intern/guardedalloc/MEM_guardedalloc.h

@@ -62,11 +62,9 @@
 
 #include <stdio.h>          /* needed for FILE* */
 
-/* needed for uintptr_t, exception, dont use BLI anywhere else in MEM_* */
+/* needed for uintptr_t and attributes, exception, dont use BLI anywhere else in MEM_* */
 #include "../../source/blender/blenlib/BLI_sys_types.h"
-
+#include "../../source/blender/blenlib/BLI_compiler_attrs.h"
-/* some GNU attributes are only available from GCC 4.3 */
-#define MEM_GNU_ATTRIBUTES (defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 403))
 
 #ifdef __cplusplus
 extern "C" {
@@ -75,57 +73,36 @@ extern "C" {
 	/** Returns the length of the allocated memory segment pointed at
 	 * by vmemh. If the pointer was not previously allocated by this
 	 * module, the result is undefined.*/
-	size_t MEM_allocN_len(const void *vmemh)
+	extern size_t (*MEM_allocN_len)(const void *vmemh) ATTR_WARN_UNUSED_RESULT;
-#if MEM_GNU_ATTRIBUTES
-	__attribute__((warn_unused_result))
-#endif
-	;
 
 	/**
 	 * Release memory previously allocatred by this module. 
 	 */
-	void MEM_freeN(void *vmemh);
+	extern void (*MEM_freeN)(void *vmemh);
 
 #if 0  /* UNUSED */
 	/**
 	 * Return zero if memory is not in allocated list
 	 */
-	short MEM_testN(void *vmemh);
+	extern short (*MEM_testN)(void *vmemh);
 #endif
 
 	/**
 	 * Duplicates a block of memory, and returns a pointer to the
 	 * newly allocated block.  */
-	void *MEM_dupallocN(const void *vmemh)
+	extern void *(*MEM_dupallocN)(const void *vmemh) /* ATTR_MALLOC */ ATTR_WARN_UNUSED_RESULT;
-#if MEM_GNU_ATTRIBUTES
-	__attribute__((malloc))
-	__attribute__((warn_unused_result))
-#endif
-	;
 
 	/**
 	 * Reallocates a block of memory, and returns pointer to the newly
 	 * allocated block, the old one is freed. this is not as optimized
 	 * as a system realloc but just makes a new allocation and copies
 	 * over from existing memory. */
-	void *MEM_reallocN_id(void *vmemh, size_t len, const char *str)
+	extern void *(*MEM_reallocN_id)(void *vmemh, size_t len, const char *str) /* ATTR_MALLOC */ ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2);
-#if MEM_GNU_ATTRIBUTES
-	__attribute__((malloc))
-	__attribute__((warn_unused_result))
-	__attribute__((alloc_size(2)))
-#endif
-	;
 
 	/**
 	 * A variant of realloc which zeros new bytes
 	 */
-	void *MEM_recallocN_id(void *vmemh, size_t len, const char *str)
+	extern void *(*MEM_recallocN_id)(void *vmemh, size_t len, const char *str) /* ATTR_MALLOC */ ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2);
-#if MEM_GNU_ATTRIBUTES
-	__attribute__((malloc))
-	__attribute__((warn_unused_result))
-	__attribute__((alloc_size(2)))
-#endif
-	;
 
 #define MEM_reallocN(vmemh, len) MEM_reallocN_id(vmemh, len, __func__)
 #define MEM_recallocN(vmemh, len) MEM_recallocN_id(vmemh, len, __func__)
@@ -134,97 +111,75 @@ extern "C" {
 	 * Allocate a block of memory of size len, with tag name str. The
 	 * memory is cleared. The name must be static, because only a
 	 * pointer to it is stored ! */
-	void *MEM_callocN(size_t len, const char *str)
+	extern void *(*MEM_callocN)(size_t len, const char *str) /* ATTR_MALLOC */ ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
-#if MEM_GNU_ATTRIBUTES
-	__attribute__((malloc))
-	__attribute__((warn_unused_result))
-	__attribute__((nonnull(2)))
-	__attribute__((alloc_size(1)))
-#endif
-	;
 
 	/**
 	 * Allocate a block of memory of size len, with tag name str. The
 	 * name must be a static, because only a pointer to it is stored !
 	 * */
-	void *MEM_mallocN(size_t len, const char *str)
+	extern void *(*MEM_mallocN)(size_t len, const char *str) /* ATTR_MALLOC */ ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
-#if MEM_GNU_ATTRIBUTES
-	__attribute__((malloc))
-	__attribute__((warn_unused_result))
-	__attribute__((nonnull(2)))
-	__attribute__((alloc_size(1)))
-#endif
-	;
 
 	/**
 	 * Same as callocN, clears memory and uses mmap (disk cached) if supported.
 	 * Can be free'd with MEM_freeN as usual.
 	 * */
-	void *MEM_mapallocN(size_t len, const char *str)
+	extern void *(*MEM_mapallocN)(size_t len, const char *str) /* ATTR_MALLOC */ ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
-#if MEM_GNU_ATTRIBUTES
-	__attribute__((malloc))
-	__attribute__((warn_unused_result))
-	__attribute__((nonnull(2)))
-	__attribute__((alloc_size(1)))
-#endif
-	;
 
 	/** Print a list of the names and sizes of all allocated memory
 	 * blocks. as a python dict for easy investigation */ 
-	void MEM_printmemlist_pydict(void);
+	extern void (*MEM_printmemlist_pydict)(void);
 
 	/** Print a list of the names and sizes of all allocated memory
 	 * blocks. */ 
-	void MEM_printmemlist(void);
+	extern void (*MEM_printmemlist)(void);
 
 	/** calls the function on all allocated memory blocks. */
-	void MEM_callbackmemlist(void (*func)(void *));
+	extern void (*MEM_callbackmemlist)(void (*func)(void *));
 
 	/** Print statistics about memory usage */
-	void MEM_printmemlist_stats(void);
+	extern void (*MEM_printmemlist_stats)(void);
 	
 	/** Set the callback function for error output. */
-	void MEM_set_error_callback(void (*func)(const char *));
+	extern void (*MEM_set_error_callback)(void (*func)(const char *));
 
 	/**
 	 * Are the start/end block markers still correct ?
 	 *
 	 * @retval 0 for correct memory, 1 for corrupted memory. */
-	bool MEM_check_memory_integrity(void);
+	extern bool (*MEM_check_memory_integrity)(void);
 
 	/** Set thread locking functions for safe memory allocation from multiple
 	 * threads, pass NULL pointers to disable thread locking again. */
-	void MEM_set_lock_callback(void (*lock)(void), void (*unlock)(void));
+	extern void (*MEM_set_lock_callback)(void (*lock)(void), void (*unlock)(void));
 	
 	/** Attempt to enforce OSX (or other OS's) to have malloc and stack nonzero */
-	void MEM_set_memory_debug(void);
+	extern void (*MEM_set_memory_debug)(void);
 
 	/**
 	 * Memory usage stats
 	 * - MEM_get_memory_in_use is all memory
 	 * - MEM_get_mapped_memory_in_use is a subset of all memory */
-	uintptr_t MEM_get_memory_in_use(void);
+	extern uintptr_t (*MEM_get_memory_in_use)(void);
 	/** Get mapped memory usage. */
-	uintptr_t MEM_get_mapped_memory_in_use(void);
+	extern uintptr_t (*MEM_get_mapped_memory_in_use)(void);
 	/** Get amount of memory blocks in use. */
-	unsigned int MEM_get_memory_blocks_in_use(void);
+	extern unsigned int (*MEM_get_memory_blocks_in_use)(void);
 
 	/** Reset the peak memory statistic to zero. */
-	void MEM_reset_peak_memory(void);
+	extern void (*MEM_reset_peak_memory)(void);
 
 	/** Get the peak memory usage in bytes, including mmap allocations. */
-	size_t MEM_get_peak_memory(void)
+	extern size_t (*MEM_get_peak_memory)(void) ATTR_WARN_UNUSED_RESULT;
-#if MEM_GNU_ATTRIBUTES
-	__attribute__((warn_unused_result))
-#endif
-	;
 
 #define MEM_SAFE_FREE(v) if (v) { MEM_freeN(v); v = NULL; } (void)0
 
 #ifndef NDEBUG
-const char *MEM_name_ptr(void *vmemh);
+extern const char *(*MEM_name_ptr)(void *vmemh);
 #endif
 
+/* Switch allocator to slower but fully guarded mode. */
+void MEM_use_guarded_allocator(void);
+
 #ifdef __cplusplus
 /* alloc funcs for C++ only */
 #define MEM_CXX_CLASS_ALLOC_FUNCS(_id)                                        \

intern/guardedalloc/SConscript

@@ -29,10 +29,12 @@ Import('env')
 
 defs = []
 
-sources = ['intern/mallocn.c', 'intern/mmap_win.c']
+sources = [
-
+    'intern/mallocn.c', 
-# could make this optional
+    'intern/mallocn_guarded_impl.c',
-defs.append('WITH_GUARDEDALLOC')
+	'intern/mallocn_lockfree_impl.c',
+    'intern/mmap_win.c'
+]
 
 if env['WITH_BF_CXX_GUARDEDALLOC']:
     sources.append('cpp/mallocn.cpp')

intern/guardedalloc/intern/mallocn_intern.h

@@ -0,0 +1,129 @@
+/*
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2013 by Blender Foundation.
+ * All rights reserved.
+ *
+ * The Original Code is: all of this file.
+ *
+ * Contributor(s): Sergey Sharybin
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+/** \file guardedalloc/intern/mallocn_intern.h
+ *  \ingroup MEM
+ */
+
+#ifndef __MALLOCN_INTERN_H__
+#define __MALLOCN_INTERN_H__
+
+/* mmap exception */
+#if defined(WIN32)
+#  include "mmap_win.h"
+#else
+#  include <sys/mman.h>
+#endif
+
+#if defined(_MSC_VER)
+#  define __func__ __FUNCTION__
+#endif
+
+#ifdef __GNUC__
+#  define UNUSED(x) UNUSED_ ## x __attribute__((__unused__))
+#else
+#  define UNUSED(x) UNUSED_ ## x
+#endif
+
+#undef HAVE_MALLOC_STATS
+
+#if defined(__linux__)
+#  include <malloc.h>
+#  define HAVE_MALLOC_STATS
+#elif defined(__APPLE__)
+#  define malloc_usable_size malloc_size
+#elif defined(WIN32)
+#  include <malloc.h>
+#  define malloc_usable_size _msize
+#else
+#  error "We don't know how to use malloc_usable_size on your platform"
+#endif
+
+/* Blame Microsoft for LLP64 and no inttypes.h, quick workaround needed: */
+#if defined(WIN64)
+#  define SIZET_FORMAT "%I64u"
+#  define SIZET_ARG(a) ((unsigned long long)(a))
+#else
+#  define SIZET_FORMAT "%lu"
+#  define SIZET_ARG(a) ((unsigned long)(a))
+#endif
+
+#define SIZET_ALIGN_4(len) ((len + 3) & ~(size_t)3)
+
+/* Prototypes for counted allocator functions */
+size_t MEM_lockfree_allocN_len(const void *vmemh) ATTR_WARN_UNUSED_RESULT;
+void MEM_lockfree_freeN(void *vmemh);
+void *MEM_lockfree_dupallocN(const void *vmemh) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT;
+void *MEM_lockfree_reallocN_id(void *vmemh, size_t len, const char *UNUSED(str))  ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2);
+void *MEM_lockfree_recallocN_id(void *vmemh, size_t len, const char *UNUSED(str))  ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2);
+void *MEM_lockfree_callocN(size_t len, const char *UNUSED(str))  ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
+void *MEM_lockfree_mallocN(size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
+void *MEM_lockfree_mapallocN(size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
+void MEM_lockfree_printmemlist_pydict(void);
+void MEM_lockfree_printmemlist(void);
+void MEM_lockfree_callbackmemlist(void (*func)(void *));
+void MEM_lockfree_printmemlist_stats(void);
+void MEM_lockfree_set_error_callback(void (*func)(const char *));
+bool MEM_lockfree_check_memory_integrity(void);
+void MEM_lockfree_set_lock_callback(void (*lock)(void), void (*unlock)(void));
+void MEM_lockfree_set_memory_debug(void);
+uintptr_t MEM_lockfree_get_memory_in_use(void);
+uintptr_t MEM_lockfree_get_mapped_memory_in_use(void);
+unsigned int MEM_lockfree_get_memory_blocks_in_use(void);
+void MEM_lockfree_reset_peak_memory(void);
+uintptr_t MEM_lockfree_get_peak_memory(void) ATTR_WARN_UNUSED_RESULT;
+#ifndef NDEBUG
+const char *MEM_lockfree_name_ptr(void *vmemh);
+#endif
+
+/* Prototypes for fully guarded allocator functions */
+size_t MEM_guarded_allocN_len(const void *vmemh) ATTR_WARN_UNUSED_RESULT;
+void MEM_guarded_freeN(void *vmemh);
+void *MEM_guarded_dupallocN(const void *vmemh) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT;
+void *MEM_guarded_reallocN_id(void *vmemh, size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2);
+void *MEM_guarded_recallocN_id(void *vmemh, size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2);
+void *MEM_guarded_callocN(size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
+void *MEM_guarded_mallocN(size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
+void *MEM_guarded_mapallocN(size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
+void MEM_guarded_printmemlist_pydict(void);
+void MEM_guarded_printmemlist(void);
+void MEM_guarded_callbackmemlist(void (*func)(void *));
+void MEM_guarded_printmemlist_stats(void);
+void MEM_guarded_set_error_callback(void (*func)(const char *));
+bool MEM_guarded_check_memory_integrity(void);
+void MEM_guarded_set_lock_callback(void (*lock)(void), void (*unlock)(void));
+void MEM_guarded_set_memory_debug(void);
+uintptr_t MEM_guarded_get_memory_in_use(void);
+uintptr_t MEM_guarded_get_mapped_memory_in_use(void);
+unsigned int MEM_guarded_get_memory_blocks_in_use(void);
+void MEM_guarded_reset_peak_memory(void);
+uintptr_t MEM_guarded_get_peak_memory(void) ATTR_WARN_UNUSED_RESULT;
+#ifndef NDEBUG
+const char *MEM_guarded_name_ptr(void *vmemh);
+#endif
+
+#endif  /* __MALLOCN_INTERN_H__ */

intern/guardedalloc/intern/mallocn_lockfree_impl.c

@@ -0,0 +1,376 @@
+/*
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
+ * All rights reserved.
+ *
+ * The Original Code is: all of this file.
+ *
+ * Contributor(s): Brecht Van Lommel
+ *                 Campbell Barton
+ *                 Sergey Sharybin
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+/** \file guardedalloc/intern/mallocn.c
+ *  \ingroup MEM
+ *
+ * Memory allocation which keeps track on allocated memory counters
+ */
+
+#include <stdlib.h>
+#include <string.h> /* memcpy */
+#include <stdarg.h>
+#include <sys/types.h>
+
+#include "MEM_guardedalloc.h"
+
+/* to ensure strict conversions */
+#include "../../source/blender/blenlib/BLI_strict_flags.h"
+
+#include "atomic_ops.h"
+#include "mallocn_intern.h"
+
+typedef struct MemHead {
+	/* Length of allocated memory block. */
+	size_t len;
+} MemHead;
+
+static unsigned int totblock = 0;
+static size_t mem_in_use = 0, mmap_in_use = 0, peak_mem = 0;
+static bool malloc_debug_memset = false;
+
+static void (*error_callback)(const char *) = NULL;
+static void (*thread_lock_callback)(void) = NULL;
+static void (*thread_unlock_callback)(void) = NULL;
+
+#define MEMHEAD_FROM_PTR(ptr) (((MemHead*) vmemh) - 1)
+#define PTR_FROM_MEMHEAD(memhead) (memhead + 1)
+
+#ifdef __GNUC__
+__attribute__ ((format(printf, 1, 2)))
+#endif
+static void print_error(const char *str, ...)
+{
+	char buf[512];
+	va_list ap;
+
+	va_start(ap, str);
+	vsnprintf(buf, sizeof(buf), str, ap);
+	va_end(ap);
+	buf[sizeof(buf) - 1] = '\0';
+
+	if (error_callback) {
+		error_callback(buf);
+	}
+}
+
+#if defined(WIN32)
+static void mem_lock_thread(void)
+{
+	if (thread_lock_callback)
+		thread_lock_callback();
+}
+
+static void mem_unlock_thread(void)
+{
+	if (thread_unlock_callback)
+		thread_unlock_callback();
+}
+#endif
+
+size_t MEM_lockfree_allocN_len(const void *vmemh)
+{
+	if (vmemh) {
+		return MEMHEAD_FROM_PTR(vmemh)->len & ~((size_t) 1);
+	}
+	else {
+		return 0;
+	}
+}
+
+void MEM_lockfree_freeN(void *vmemh)
+{
+	MemHead *memh = MEMHEAD_FROM_PTR(vmemh);
+	size_t len = MEM_lockfree_allocN_len(vmemh);
+
+	atomic_sub_u(&totblock, 1);
+	atomic_sub_z(&mem_in_use, len);
+
+	if (memh->len & (size_t) 1) {
+		atomic_sub_z(&mmap_in_use, len);
+#if defined(WIN32)
+		/* our windows mmap implementation is not thread safe */
+		mem_lock_thread();
+#endif
+		if (munmap(memh, memh->len + sizeof(MemHead)))
+			printf("Couldn't unmap memory\n");
+#if defined(WIN32)
+		mem_unlock_thread();
+#endif
+	}
+	else {
+		if (malloc_debug_memset && len) {
+			memset(memh + 1, 255, len);
+		}
+		free(memh);
+	}
+}
+
+void *MEM_lockfree_dupallocN(const void *vmemh)
+{
+	void *newp = NULL;
+	if (vmemh) {
+		const size_t prev_size = MEM_allocN_len(vmemh);
+		newp = MEM_lockfree_mallocN(prev_size, "dupli_malloc");
+		memcpy(newp, vmemh, prev_size);
+	}
+	return newp;
+}
+
+void *MEM_lockfree_reallocN_id(void *vmemh, size_t len, const char *str)
+{
+	void *newp = NULL;
+
+	if (vmemh) {
+		size_t old_len = MEM_allocN_len(vmemh);
+
+		newp = MEM_lockfree_mallocN(len, "realloc");
+		if (newp) {
+			if (len < old_len) {
+				/* shrink */
+				memcpy(newp, vmemh, len);
+			}
+			else {
+				/* grow (or remain same size) */
+				memcpy(newp, vmemh, old_len);
+			}
+		}
+
+		MEM_lockfree_freeN(vmemh);
+	}
+	else {
+		newp = MEM_lockfree_mallocN(len, str);
+	}
+
+	return newp;
+}
+
+void *MEM_lockfree_recallocN_id(void *vmemh, size_t len, const char *str)
+{
+	void *newp = NULL;
+
+	if (vmemh) {
+		size_t old_len = MEM_allocN_len(vmemh);
+
+		newp = MEM_lockfree_mallocN(len, "recalloc");
+		if (newp) {
+			if (len < old_len) {
+				/* shrink */
+				memcpy(newp, vmemh, len);
+			}
+			else {
+				memcpy(newp, vmemh, old_len);
+
+				if (len > old_len) {
+					/* grow */
+					/* zero new bytes */
+					memset(((char *)newp) + old_len, 0, len - old_len);
+				}
+			}
+		}
+
+		MEM_lockfree_freeN(vmemh);
+	}
+	else {
+		newp = MEM_lockfree_callocN(len, str);
+	}
+
+	return newp;
+}
+
+void *MEM_lockfree_callocN(size_t len, const char *str)
+{
+	MemHead *memh;
+
+	len = SIZET_ALIGN_4(len);
+
+	memh = (MemHead *)calloc(1, len + sizeof(MemHead));
+
+	if (memh) {
+		memh->len = len;
+		atomic_add_u(&totblock, 1);
+		atomic_add_z(&mem_in_use, len);
+
+		/* TODO(sergey): Not strictly speaking thread-safe. */
+		peak_mem = mem_in_use > peak_mem ? mem_in_use : peak_mem;
+
+		return PTR_FROM_MEMHEAD(memh);
+	}
+	print_error("Calloc returns null: len=" SIZET_FORMAT " in %s, total %u\n",
+	            SIZET_ARG(len), str, (unsigned int) mem_in_use);
+	return NULL;
+}
+
+void *MEM_lockfree_mallocN(size_t len, const char *str)
+{
+	MemHead *memh;
+
+	len = SIZET_ALIGN_4(len);
+
+	memh = (MemHead *)malloc(len + sizeof(MemHead));
+
+	if (memh) {
+		if (malloc_debug_memset && len) {
+			memset(memh + 1, 255, len);
+		}
+
+		memh->len = len;
+		atomic_add_u(&totblock, 1);
+		atomic_add_z(&mem_in_use, len);
+
+		/* TODO(sergey): Not strictly speaking thread-safe. */
+		peak_mem = mem_in_use > peak_mem ? mem_in_use : peak_mem;
+
+		return PTR_FROM_MEMHEAD(memh);
+	}
+	print_error("Malloc returns null: len=" SIZET_FORMAT " in %s, total %u\n",
+	            SIZET_ARG(len), str, (unsigned int) mem_in_use);
+	return NULL;
+}
+
+void *MEM_lockfree_mapallocN(size_t len, const char *str)
+{
+	MemHead *memh;
+
+	len = SIZET_ALIGN_4(len);
+
+#if defined(WIN32)
+	/* our windows mmap implementation is not thread safe */
+	mem_lock_thread();
+#endif
+	memh = mmap(NULL, len + sizeof(MemHead),
+	            PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0);
+#if defined(WIN32)
+	mem_unlock_thread();
+#endif
+
+	if (memh != (MemHead *)-1) {
+		memh->len = len | (size_t) 1;
+		atomic_add_u(&totblock, 1);
+		atomic_add_z(&mem_in_use, len);
+		atomic_add_z(&mmap_in_use, len);
+
+		/* TODO(sergey): Not strictly speaking thread-safe. */
+		peak_mem = mem_in_use > peak_mem ? mem_in_use : peak_mem;
+		peak_mem = mmap_in_use > peak_mem ? mmap_in_use : peak_mem;
+
+		return PTR_FROM_MEMHEAD(memh);
+	}
+	print_error("Mapalloc returns null, fallback to regular malloc: "
+	            "len=" SIZET_FORMAT " in %s, total %u\n",
+	            SIZET_ARG(len), str, (unsigned int) mmap_in_use);
+	return MEM_lockfree_callocN(len, str);
+}
+
+void MEM_lockfree_printmemlist_pydict(void)
+{
+}
+
+void MEM_lockfree_printmemlist(void)
+{
+}
+
+/* unused */
+void MEM_lockfree_callbackmemlist(void (*func)(void *))
+{
+	(void) func;  /* Ignored. */
+}
+
+void MEM_lockfree_printmemlist_stats(void)
+{
+	printf("\ntotal memory len: %.3f MB\n",
+	       (double)mem_in_use / (double)(1024 * 1024));
+	printf("peak memory len: %.3f MB\n",
+	       (double)peak_mem / (double)(1024 * 1024));
+	printf("\nFor more detailed per-block statistics run Blender with memory debugging command line argument.\n");
+
+#ifdef HAVE_MALLOC_STATS
+	printf("System Statistics:\n");
+	malloc_stats();
+#endif
+}
+
+void MEM_lockfree_set_error_callback(void (*func)(const char *))
+{
+	error_callback = func;
+}
+
+bool MEM_lockfree_check_memory_integrity(void)
+{
+	return true;
+}
+
+void MEM_lockfree_set_lock_callback(void (*lock)(void), void (*unlock)(void))
+{
+	thread_lock_callback = lock;
+	thread_unlock_callback = unlock;
+}
+
+void MEM_lockfree_set_memory_debug(void)
+{
+	malloc_debug_memset = true;
+}
+
+uintptr_t MEM_lockfree_get_memory_in_use(void)
+{
+	return mem_in_use;
+}
+
+uintptr_t MEM_lockfree_get_mapped_memory_in_use(void)
+{
+	return mmap_in_use;
+}
+
+unsigned int MEM_lockfree_get_memory_blocks_in_use(void)
+{
+	return totblock;
+}
+
+/* dummy */
+void MEM_lockfree_reset_peak_memory(void)
+{
+	peak_mem = 0;
+}
+
+uintptr_t MEM_lockfree_get_peak_memory(void)
+{
+	return peak_mem;
+}
+
+#ifndef NDEBUG
+const char *MEM_lockfree_name_ptr(void *vmemh)
+{
+	if (vmemh) {
+		return "unknown block name ptr";
+	}
+	else {
+		return "MEM_lockfree_name_ptr(NULL)";
+	}
+}
+#endif  /* NDEBUG */

source/blender/makesdna/intern/CMakeLists.txt

@@ -40,6 +40,8 @@ blender_include_dirs(
 set(SRC
 	makesdna.c
 	../../../../intern/guardedalloc/intern/mallocn.c
+	../../../../intern/guardedalloc/intern/mallocn_guarded_impl.c
+	../../../../intern/guardedalloc/intern/mallocn_lockfree_impl.c
 )
 
 if(WIN32 AND NOT UNIX)
@@ -50,11 +52,6 @@ endif()
 
 # SRC_DNA_INC is defined in the parent dir
 
-# for mallocn.c
-if (WITH_GUARDEDALLOC)
-	add_definitions(-DWITH_GUARDEDALLOC)
-endif()
-
 add_executable(makesdna ${SRC} ${SRC_DNA_INC})
 
 # Output dna.c

source/blender/makesdna/intern/SConscript

@@ -36,8 +36,6 @@ cflags = ''
 defines = []
 root_build_dir=normpath(env['BF_BUILDDIR'])
 
-defines.append('WITH_GUARDEDALLOC')
-
 source_files = ['makesdna.c']
 header_files = env.Glob('../*.h')
 

source/blender/makesrna/intern/CMakeLists.txt

@@ -137,6 +137,8 @@ set(SRC
 	${DEFSRC}
 	${APISRC}
 	../../../../intern/guardedalloc/intern/mallocn.c
+	../../../../intern/guardedalloc/intern/mallocn_guarded_impl.c
+	../../../../intern/guardedalloc/intern/mallocn_lockfree_impl.c
 	../../../../intern/guardedalloc/intern/mmap_win.c
 )
 
@@ -253,11 +255,6 @@ if(WITH_BULLET)
 	add_definitions(-DWITH_BULLET)
 endif()
 
-# for mallocn.c
-if (WITH_GUARDEDALLOC)
-	add_definitions(-DWITH_GUARDEDALLOC)
-endif()
-
 # Build makesrna executable
 blender_include_dirs(
 	.

source/blender/makesrna/intern/SConscript

@@ -150,8 +150,6 @@ if env['WITH_BF_INTERNATIONAL']:
 if not env['BF_DEBUG']:
     defs.append('NDEBUG')
 
-defs.append('WITH_GUARDEDALLOC')
-
 makesrna_tool.Append(CPPDEFINES=defs)
 
 makesrna_tool.Append (CPPPATH = Split(incs))

source/creator/creator.c

@@ -300,6 +300,7 @@ static int print_help(int UNUSED(argc), const char **UNUSED(argv), void *data)
 #ifdef WITH_LIBMV
 	BLI_argsPrintArgDoc(ba, "--debug-libmv");
 #endif
+	BLI_argsPrintArgDoc(ba, "--debug-memory");
 	BLI_argsPrintArgDoc(ba, "--debug-jobs");
 	BLI_argsPrintArgDoc(ba, "--debug-python");
 
@@ -439,6 +440,12 @@ static int debug_mode_libmv(int UNUSED(argc), const char **UNUSED(argv), void *U
 }
 #endif
 
+static int debug_mode_memory(int UNUSED(argc), const char **UNUSED(argv), void *UNUSED(data))
+{
+	MEM_set_memory_debug();
+	return 0;
+}
+
 static int set_debug_value(int argc, const char **argv, void *UNUSED(data))
 {
 	if (argc > 1) {
@@ -1384,6 +1391,7 @@ static void setupArguments(bContext *C, bArgs *ba, SYS_SystemHandle *syshandle)
 #ifdef WITH_LIBMV
 	BLI_argsAdd(ba, 1, NULL, "--debug-libmv", "\n\tEnable debug messages from libmv library", debug_mode_libmv, NULL);
 #endif
+	BLI_argsAdd(ba, 1, NULL, "--debug-memory", "\n\tEnable fully guarded memory allocation and debugging", debug_mode_memory, NULL);
 
 	BLI_argsAdd(ba, 1, NULL, "--debug-value", "<value>\n\tSet debug value of <value> on startup\n", set_debug_value, NULL);
 	BLI_argsAdd(ba, 1, NULL, "--debug-jobs",  "\n\tEnable time profiling for background jobs.", debug_mode_generic, (void *)G_DEBUG_JOBS);
@@ -1454,23 +1462,50 @@ int main(int argc, const char **UNUSED(argv_c)) /* Do not mess with const */
 int main(int argc, const char **argv)
 #endif
 {
-	bContext *C = CTX_create();
+	bContext *C;
 	SYS_SystemHandle syshandle;
 
 #ifndef WITH_PYTHON_MODULE
 	bArgs *ba;
 #endif
 
-#ifdef WIN32
+#ifdef WIN32 /* Win32 Unicode Args */
+	/* NOTE: cannot use guardedalloc malloc here, as it's not yet initialised 
+	 *       (it depends on the args passed in, which is what we're getting here!)
+	 */
 	wchar_t **argv_16 = CommandLineToArgvW(GetCommandLineW(), &argc);
+	char **argv = malloc(argc * sizeof(char *));
 	int argci = 0;
-	char **argv = MEM_mallocN(argc * sizeof(char *), "argv array");
+	
 	for (argci = 0; argci < argc; argci++) {
 		argv[argci] = alloc_utf_8_from_16(argv_16[argci], 0);
 	}
+	
 	LocalFree(argv_16);
 #endif
 
+	/* NOTE: Special exception for guarded allocator type switch:
+	 *       we need to perform switch from lock-free to fully
+	 *       guarded allocator before any allocation happened.
+	 */
+	{
+		int i;
+		for (i = 0; i < argc; i++) {
+			if (STREQ(argv[i], "--debug") || STREQ(argv[i], "-d") ||
+			    STREQ(argv[i], "--debug-memory"))
+			{
+				printf("Switching to fully guarded memory allocator.\n");
+				MEM_use_guarded_allocator();
+				break;
+			}
+			else if (STREQ(argv[i], "--")) {
+				break;
+			}
+		}
+	}
+
+	C = CTX_create();
+
 #ifdef WITH_PYTHON_MODULE
 #ifdef __APPLE__
 	environ = *_NSGetEnviron();
@@ -1641,7 +1676,7 @@ int main(int argc, const char **argv)
 	while (argci) {
 		free(argv[--argci]);
 	}
-	MEM_freeN(argv);
+	free(argv);
 	argv = NULL;
 #endif