Camera tracking integration

===========================

Attempt to switch moviecache to use CacheLimiter.

Some changes in limiter were necessary:
- Limiter counted mapped memory twice when was chacking
  how many memory is used.
- It was using "global" memory usage not memory usage by
  cached elements. It will cause big problems when there's
  large mesh or plenty of undo steps are in memory nothing
  would be cached in sequencer.
- To solve this problem introduced "callback" to measure
  cached element size. It could be not very accurate in general,
  but it works well for image buffers. And if this callback
  isn't set old-school memory usage check would be used.
- The whole cache used to get freed when memory limit exceeded,
  now it'll drop only as much elements as necessary to reduce
  memory usage.

Sequence cache wasn't switched to use moviecache but
now it's really easy to do. When i'll be sure new caching
scheme works fine.

Now clip editor uses as much memory for cache as it's set in
User Preferences (Preferences -> System -> Sequencer -> Memory
Cache Limit) which si 128Mb by default. Please do not complain
about few cached frames out-of-box and just increase limit
there. Caching fixed amount of frames wasn't so nice indeed.

intern/memutil/MEM_CacheLimiter.h

@@ -126,6 +126,10 @@ class MEM_CacheLimiter {
 public:
 	typedef typename std::list<MEM_CacheLimiterHandle<T> *,
 	  MEM_Allocator<MEM_CacheLimiterHandle<T> *> >::iterator iterator;
+	typedef intptr_t (*MEM_CacheLimiter_DataSize_Func) (void *data);
+	MEM_CacheLimiter(MEM_CacheLimiter_DataSize_Func getDataSize_)
+		: getDataSize(getDataSize_) {
+	}
 	~MEM_CacheLimiter() {
 		for (iterator it = queue.begin(); it != queue.end(); it++) {
 			delete *it;
@@ -144,17 +148,36 @@ public:
 	}
 	void enforce_limits() {
 		intptr_t max = MEM_CacheLimiter_get_maximum();
-		intptr_t mem_in_use= MEM_get_memory_in_use();
-		intptr_t mmap_in_use= MEM_get_mapped_memory_in_use();
+		intptr_t mem_in_use, cur_size;
 
 		if (max == 0) {
 			return;
 		}
+
+		if(getDataSize) {
+			mem_in_use = total_size();
+		} else {
+			mem_in_use = MEM_get_memory_in_use();
+		}
+
 		for (iterator it = queue.begin(); 
-		     it != queue.end() && mem_in_use + mmap_in_use > max;) {
+		     it != queue.end() && mem_in_use > max;) {
 			iterator jt = it;
 			++it;
+
+			if(getDataSize) {
+				cur_size= getDataSize((*jt)->get()->get_data());
+			} else {
+				cur_size= mem_in_use;
+			}
+
 			(*jt)->destroy_if_possible();
+
+			if(getDataSize) {
+				mem_in_use-= cur_size;
+			} else {
+				mem_in_use-= cur_size - MEM_get_memory_in_use();
+			}
 		}
 	}
 	void touch(MEM_CacheLimiterHandle<T> * handle) {
@@ -165,8 +188,17 @@ public:
 		handle->me = it;
 	}
 private:
+	intptr_t total_size() {
+		intptr_t size = 0;
+		for (iterator it = queue.begin(); it != queue.end(); it++) {
+			size+= getDataSize((*it)->get()->get_data());
+		}
+		return size;
+	}
+
 	std::list<MEM_CacheLimiterHandle<T>*,
 	  MEM_Allocator<MEM_CacheLimiterHandle<T> *> > queue;
+	MEM_CacheLimiter_DataSize_Func getDataSize;
 };
 
 #endif // MEM_CACHELIMITER_H

intern/memutil/MEM_CacheLimiterC-Api.h

@@ -42,6 +42,9 @@ typedef struct MEM_CacheLimiterHandle_s MEM_CacheLimiterHandleC;
 /* function used to remove data from memory */
 typedef void(*MEM_CacheLimiter_Destruct_Func)(void*);
 
+/* function used to measure stored data element size */
+typedef intptr_t(*MEM_CacheLimiter_DataSize_Func) (void*);
+
 #ifndef MEM_CACHELIMITER_H
 extern void MEM_CacheLimiter_set_maximum(int m);
 extern int MEM_CacheLimiter_get_maximum(void);
@@ -55,7 +58,8 @@ extern int MEM_CacheLimiter_get_maximum(void);
  */
 
 extern MEM_CacheLimiterC * new_MEM_CacheLimiter(
-	MEM_CacheLimiter_Destruct_Func data_destructor);
+	MEM_CacheLimiter_Destruct_Func data_destructor,
+	MEM_CacheLimiter_DataSize_Func data_size);
 
 /** 
  * Delete MEM_CacheLimiter

intern/memutil/intern/MEM_CacheLimiterC-Api.cpp

@@ -54,8 +54,8 @@ typedef std::list<MEM_CacheLimiterHandleCClass*,
 
 class MEM_CacheLimiterCClass {
 public:
-	MEM_CacheLimiterCClass(MEM_CacheLimiter_Destruct_Func data_destructor_)
-		: data_destructor(data_destructor_) { 
+	MEM_CacheLimiterCClass(MEM_CacheLimiter_Destruct_Func data_destructor_, MEM_CacheLimiter_DataSize_Func data_size)
+		: data_destructor(data_destructor_), cache(data_size) {
 	}
         ~MEM_CacheLimiterCClass();
 	
@@ -142,10 +142,12 @@ static inline handle_t* cast(MEM_CacheLimiterHandleC * l)
 }
 
 MEM_CacheLimiterC * new_MEM_CacheLimiter(
-	MEM_CacheLimiter_Destruct_Func data_destructor)
+	MEM_CacheLimiter_Destruct_Func data_destructor,
+	MEM_CacheLimiter_DataSize_Func data_size)
 {
 	return (MEM_CacheLimiterC*) new MEM_CacheLimiterCClass(
-		data_destructor);
+		data_destructor,
+		data_size);
 }
 
 void delete_MEM_CacheLimiter(MEM_CacheLimiterC * This)

source/blender/blenkernel/BKE_moviecache.h

@@ -45,6 +45,9 @@ struct MovieCache;
 
 typedef void (*MovieCacheGetKeyDataFP) (void *userkey, int *framenr);
 
+void BKE_moviecache_init(void);
+void BKE_moviecache_destruct(void);
+
 struct MovieCache *BKE_moviecache_create(int keysize, GHashHashFP hashfp, GHashCmpFP cmpfp, MovieCacheGetKeyDataFP getdatafp);
 void BKE_moviecache_put(struct MovieCache *cache, void *userkey, struct ImBuf *ibuf);
 struct ImBuf* BKE_moviecache_get(struct MovieCache *cache, void *userkey);

source/blender/blenkernel/intern/blender.c

@@ -82,6 +82,7 @@
 #include "BKE_scene.h"
 #include "BKE_screen.h"
 #include "BKE_sequencer.h"
+#include "BKE_moviecache.h"
 
 
 #include "BLO_undofile.h"
@@ -115,6 +116,7 @@ void free_blender(void)
 	BLI_cb_finalize();
 
 	seq_stripelem_cache_destruct();
+	BKE_moviecache_destruct();
 	
 	free_nodesystem();	
 }

source/blender/blenkernel/intern/moviecache.c

@@ -32,6 +32,7 @@
  */
 
 #include "MEM_guardedalloc.h"
+#include "MEM_CacheLimiterC-Api.h"
 
 #include "BLI_utildefines.h"
 #include "BLI_ghash.h"
@@ -42,6 +43,8 @@
 #include "IMB_imbuf_types.h"
 #include "IMB_imbuf.h"
 
+static MEM_CacheLimiterC *limitor= NULL;
+
 typedef struct MovieCache {
 	GHash *hash;
 	GHashHashFP hashfp;
@@ -66,6 +69,7 @@ typedef struct MovieCacheKey {
 typedef struct MovieCacheItem {
 	MovieCache *cache_owner;
 	ImBuf *ibuf;
+	MEM_CacheLimiterHandleC * c_handle;
 	unsigned long last_access;
 } MovieCacheItem;
 
@@ -95,33 +99,30 @@ static void moviecache_valfree(void *val)
 {
 	MovieCacheItem *item= (MovieCacheItem*)val;
 
-	IMB_freeImBuf(item->ibuf);
+	if (item->ibuf) {
+		MEM_CacheLimiter_unmanage(item->c_handle);
+		IMB_freeImBuf(item->ibuf);
+	}
 
 	BLI_mempool_free(item->cache_owner->items_pool, item);
 }
 
-static MovieCacheKey *get_lru_key(MovieCache *cache)
+static void check_unused_keys(MovieCache *cache)
 {
 	GHashIterator *iter;
-	MovieCacheKey *lru_key= NULL;
-	MovieCacheItem *lru_item= NULL;
 
 	iter= BLI_ghashIterator_new(cache->hash);
 	while(!BLI_ghashIterator_isDone(iter)) {
 		MovieCacheKey *key= BLI_ghashIterator_getKey(iter);
 		MovieCacheItem *item= BLI_ghashIterator_getValue(iter);
 
-		if(lru_item==NULL || item->last_access<lru_item->last_access) {
-			lru_key= key;
-			lru_item= item;
-		}
-
 		BLI_ghashIterator_step(iter);
+
+		if(!item->ibuf)
+			BLI_ghash_remove(cache->hash, key, moviecache_keyfree, moviecache_valfree);
 	}
 
 	BLI_ghashIterator_free(iter);
-
-	return lru_key;
 }
 
 static int compare_int(const void *av, const void *bv)
@@ -131,6 +132,70 @@ static int compare_int(const void *av, const void *bv)
 	return *a-*b;
 }
 
+static void IMB_moviecache_destructor(void *p)
+{
+	MovieCacheItem *item= (MovieCacheItem *) p;
+
+	if (item && item->ibuf) {
+		IMB_freeImBuf(item->ibuf);
+
+		item->ibuf= NULL;
+		item->c_handle= NULL;
+	}
+}
+
+/* approximate size of ImBuf in memory */
+static intptr_t IMB_get_size_in_memory(ImBuf *ibuf)
+{
+	int a;
+	intptr_t size= 0, channel_size= 0;
+
+	size+= sizeof(ImBuf);
+
+	if(ibuf->rect)
+		channel_size+= sizeof(char);
+
+	if(ibuf->rect_float)
+		channel_size= sizeof(float);
+
+	size+= channel_size*ibuf->x*ibuf->y*ibuf->channels;
+
+	if(ibuf->miptot) {
+		for(a= 0; a<ibuf->miptot; a++) {
+			if(ibuf->mipmap[a])
+				size+= IMB_get_size_in_memory(ibuf->mipmap[a]);
+		}
+	}
+
+	if(ibuf->tiles) {
+		size+= sizeof(unsigned int)*ibuf->ytiles*ibuf->xtiles;
+	}
+
+	return size;
+}
+
+static intptr_t get_item_size (void *p)
+{
+	intptr_t size= sizeof(MovieCacheItem);
+	MovieCacheItem *item= (MovieCacheItem *) p;
+
+	if(item->ibuf)
+		size+= IMB_get_size_in_memory(item->ibuf);
+
+	return size;
+}
+
+void BKE_moviecache_init(void)
+{
+	limitor= new_MEM_CacheLimiter(IMB_moviecache_destructor, get_item_size);
+}
+
+void BKE_moviecache_destruct(void)
+{
+	if(limitor)
+		delete_MEM_CacheLimiter(limitor);
+}
+
 struct MovieCache *BKE_moviecache_create(int keysize, GHashHashFP hashfp, GHashCmpFP cmpfp,
 		MovieCacheGetKeyDataFP getdatafp)
 {
@@ -155,11 +220,8 @@ void BKE_moviecache_put(MovieCache *cache, void *userkey, ImBuf *ibuf)
 	MovieCacheKey *key;
 	MovieCacheItem *item;
 
-	/* TODO: implement better limiters */
-	if(BLI_ghash_size(cache->hash) > 250) {
-		MovieCacheKey *lru_key= get_lru_key(cache);
-		BLI_ghash_remove(cache->hash, lru_key, moviecache_keyfree, moviecache_valfree);
-	}
+	if(!limitor)
+		BKE_moviecache_init();
 
 	IMB_refImBuf(ibuf);
 
@@ -172,10 +234,20 @@ void BKE_moviecache_put(MovieCache *cache, void *userkey, ImBuf *ibuf)
 	item->ibuf= ibuf;
 	item->cache_owner= cache;
 	item->last_access= cache->curtime++;
+	item->c_handle= NULL;
 
 	BLI_ghash_remove(cache->hash, key, moviecache_keyfree, moviecache_valfree);
 	BLI_ghash_insert(cache->hash, key, item);
 
+	item->c_handle= MEM_CacheLimiter_insert(limitor, item);
+
+	MEM_CacheLimiter_ref(item->c_handle);
+	MEM_CacheLimiter_enforce_limits(limitor);
+	MEM_CacheLimiter_unref(item->c_handle);
+
+	/* cache limiter can't remove unused keys which points to destoryed values */
+	check_unused_keys(cache);
+
 	if(cache->points) {
 		MEM_freeN(cache->points);
 		cache->points= NULL;
@@ -195,6 +267,7 @@ ImBuf* BKE_moviecache_get(MovieCache *cache, void *userkey)
 		item->last_access= cache->curtime++;
 
 		if(item->ibuf) {
+			MEM_CacheLimiter_touch(item->c_handle);
 			IMB_refImBuf(item->ibuf);
 			return item->ibuf;
 		}
@@ -239,11 +312,14 @@ void BKE_moviecache_get_cache_segments(MovieCache *cache, int *totseg_r, int **p
 		a= 0;
 		while(!BLI_ghashIterator_isDone(iter)) {
 			MovieCacheKey *key= BLI_ghashIterator_getKey(iter);
+			MovieCacheItem *item= BLI_ghashIterator_getValue(iter);
 			int framenr;
 
-			cache->getdatafp(key->userkey, &framenr);
+			if(item->ibuf) {
+				cache->getdatafp(key->userkey, &framenr);
 
-			frames[a++]= framenr;
+				frames[a++]= framenr;
+			}
 
 			BLI_ghashIterator_step(iter);
 		}

source/blender/blenkernel/intern/seqcache.c

@@ -148,7 +148,7 @@ static void IMB_seq_cache_destructor(void * p)
 void seq_stripelem_cache_init(void)
 {
 	hash = BLI_ghash_new(HashHash, HashCmp, "seq stripelem cache hash");
-	limitor = new_MEM_CacheLimiter( IMB_seq_cache_destructor );
+	limitor = new_MEM_CacheLimiter( IMB_seq_cache_destructor, NULL );
 
 	entrypool = BLI_mempool_create(sizeof(seqCacheEntry), 64, 64, 0);
 	keypool = BLI_mempool_create(sizeof(seqCacheKey), 64, 64, 0);

source/blender/imbuf/intern/allocimbuf.c

@@ -455,7 +455,7 @@ static MEM_CacheLimiterC **get_imbuf_cache_limiter(void)
 	static MEM_CacheLimiterC *c = NULL;
 
 	if(!c)
-		c = new_MEM_CacheLimiter(imbuf_cache_destructor);
+		c = new_MEM_CacheLimiter(imbuf_cache_destructor, NULL);
 
 	return &c;
 }