Camera tracking integration
authorSergey Sharybin <sergey.vfx@gmail.com>
Wed, 27 Jul 2011 12:53:39 +0000 (12:53 +0000)
committerSergey Sharybin <sergey.vfx@gmail.com>
Wed, 27 Jul 2011 12:53:39 +0000 (12:53 +0000)
===========================

Attempt to switch moviecache to use CacheLimiter.

Some changes in limiter were necessary:
- Limiter counted mapped memory twice when was chacking
  how many memory is used.
- It was using "global" memory usage not memory usage by
  cached elements. It will cause big problems when there's
  large mesh or plenty of undo steps are in memory nothing
  would be cached in sequencer.
- To solve this problem introduced "callback" to measure
  cached element size. It could be not very accurate in general,
  but it works well for image buffers. And if this callback
  isn't set old-school memory usage check would be used.
- The whole cache used to get freed when memory limit exceeded,
  now it'll drop only as much elements as necessary to reduce
  memory usage.

Sequence cache wasn't switched to use moviecache but
now it's really easy to do. When i'll be sure new caching
scheme works fine.

Now clip editor uses as much memory for cache as it's set in
User Preferences (Preferences -> System -> Sequencer -> Memory
Cache Limit) which si 128Mb by default. Please do not complain
about few cached frames out-of-box and just increase limit
there. Caching fixed amount of frames wasn't so nice indeed.

intern/memutil/MEM_CacheLimiter.h
intern/memutil/MEM_CacheLimiterC-Api.h
intern/memutil/intern/MEM_CacheLimiterC-Api.cpp
source/blender/blenkernel/BKE_moviecache.h
source/blender/blenkernel/intern/blender.c
source/blender/blenkernel/intern/moviecache.c
source/blender/blenkernel/intern/seqcache.c
source/blender/imbuf/intern/allocimbuf.c

index 0b657104a054e1b63c962e7dc41196bdb8fd8a17..498e6d29381448c3058c6fee0400a4bd98ee5715 100644 (file)
@@ -126,6 +126,10 @@ class MEM_CacheLimiter {
 public:
        typedef typename std::list<MEM_CacheLimiterHandle<T> *,
          MEM_Allocator<MEM_CacheLimiterHandle<T> *> >::iterator iterator;
+       typedef intptr_t (*MEM_CacheLimiter_DataSize_Func) (void *data);
+       MEM_CacheLimiter(MEM_CacheLimiter_DataSize_Func getDataSize_)
+               : getDataSize(getDataSize_) {
+       }
        ~MEM_CacheLimiter() {
                for (iterator it = queue.begin(); it != queue.end(); it++) {
                        delete *it;
@@ -144,17 +148,36 @@ public:
        }
        void enforce_limits() {
                intptr_t max = MEM_CacheLimiter_get_maximum();
-               intptr_t mem_in_use= MEM_get_memory_in_use();
-               intptr_t mmap_in_use= MEM_get_mapped_memory_in_use();
+               intptr_t mem_in_use, cur_size;
 
                if (max == 0) {
                        return;
                }
+
+               if(getDataSize) {
+                       mem_in_use = total_size();
+               } else {
+                       mem_in_use = MEM_get_memory_in_use();
+               }
+
                for (iterator it = queue.begin(); 
-                    it != queue.end() && mem_in_use + mmap_in_use > max;) {
+                    it != queue.end() && mem_in_use > max;) {
                        iterator jt = it;
                        ++it;
+
+                       if(getDataSize) {
+                               cur_size= getDataSize((*jt)->get()->get_data());
+                       } else {
+                               cur_size= mem_in_use;
+                       }
+
                        (*jt)->destroy_if_possible();
+
+                       if(getDataSize) {
+                               mem_in_use-= cur_size;
+                       } else {
+                               mem_in_use-= cur_size - MEM_get_memory_in_use();
+                       }
                }
        }
        void touch(MEM_CacheLimiterHandle<T> * handle) {
@@ -165,8 +188,17 @@ public:
                handle->me = it;
        }
 private:
+       intptr_t total_size() {
+               intptr_t size = 0;
+               for (iterator it = queue.begin(); it != queue.end(); it++) {
+                       size+= getDataSize((*it)->get()->get_data());
+               }
+               return size;
+       }
+
        std::list<MEM_CacheLimiterHandle<T>*,
          MEM_Allocator<MEM_CacheLimiterHandle<T> *> > queue;
+       MEM_CacheLimiter_DataSize_Func getDataSize;
 };
 
 #endif // MEM_CACHELIMITER_H
index 4f267f7ddf0994c2177fb3ca5784212c94d67641..b3e67c9434da3f00db59cd90628a91bb6fa5bc9d 100644 (file)
@@ -42,6 +42,9 @@ typedef struct MEM_CacheLimiterHandle_s MEM_CacheLimiterHandleC;
 /* function used to remove data from memory */
 typedef void(*MEM_CacheLimiter_Destruct_Func)(void*);
 
+/* function used to measure stored data element size */
+typedef intptr_t(*MEM_CacheLimiter_DataSize_Func) (void*);
+
 #ifndef MEM_CACHELIMITER_H
 extern void MEM_CacheLimiter_set_maximum(int m);
 extern int MEM_CacheLimiter_get_maximum(void);
@@ -55,7 +58,8 @@ extern int MEM_CacheLimiter_get_maximum(void);
  */
 
 extern MEM_CacheLimiterC * new_MEM_CacheLimiter(
-       MEM_CacheLimiter_Destruct_Func data_destructor);
+       MEM_CacheLimiter_Destruct_Func data_destructor,
+       MEM_CacheLimiter_DataSize_Func data_size);
 
 /** 
  * Delete MEM_CacheLimiter
index 1bc011a5be0d75d18d8064f82b9114e9a579384a..cc8d2497f37747aa812365ddd3bcf6144664c118 100644 (file)
@@ -54,8 +54,8 @@ typedef std::list<MEM_CacheLimiterHandleCClass*,
 
 class MEM_CacheLimiterCClass {
 public:
-       MEM_CacheLimiterCClass(MEM_CacheLimiter_Destruct_Func data_destructor_)
-               : data_destructor(data_destructor_) { 
+       MEM_CacheLimiterCClass(MEM_CacheLimiter_Destruct_Func data_destructor_, MEM_CacheLimiter_DataSize_Func data_size)
+               : data_destructor(data_destructor_), cache(data_size) {
        }
         ~MEM_CacheLimiterCClass();
        
@@ -142,10 +142,12 @@ static inline handle_t* cast(MEM_CacheLimiterHandleC * l)
 }
 
 MEM_CacheLimiterC * new_MEM_CacheLimiter(
-       MEM_CacheLimiter_Destruct_Func data_destructor)
+       MEM_CacheLimiter_Destruct_Func data_destructor,
+       MEM_CacheLimiter_DataSize_Func data_size)
 {
        return (MEM_CacheLimiterC*) new MEM_CacheLimiterCClass(
-               data_destructor);
+               data_destructor,
+               data_size);
 }
 
 void delete_MEM_CacheLimiter(MEM_CacheLimiterC * This)
index 804751d6a0cd3afe191b731ecd7eac6ae6079143..8ae3443284cdb448aa1b45618ae583e636115ee0 100644 (file)
@@ -45,6 +45,9 @@ struct MovieCache;
 
 typedef void (*MovieCacheGetKeyDataFP) (void *userkey, int *framenr);
 
+void BKE_moviecache_init(void);
+void BKE_moviecache_destruct(void);
+
 struct MovieCache *BKE_moviecache_create(int keysize, GHashHashFP hashfp, GHashCmpFP cmpfp, MovieCacheGetKeyDataFP getdatafp);
 void BKE_moviecache_put(struct MovieCache *cache, void *userkey, struct ImBuf *ibuf);
 struct ImBuf* BKE_moviecache_get(struct MovieCache *cache, void *userkey);
index 8b4bbbd3c834ddb203978eefb3208fb747980e6d..f50affc1bb5f9013cea20eb4856c7184795dd7c4 100644 (file)
@@ -82,6 +82,7 @@
 #include "BKE_scene.h"
 #include "BKE_screen.h"
 #include "BKE_sequencer.h"
+#include "BKE_moviecache.h"
 
 
 #include "BLO_undofile.h"
@@ -115,6 +116,7 @@ void free_blender(void)
        BLI_cb_finalize();
 
        seq_stripelem_cache_destruct();
+       BKE_moviecache_destruct();
        
        free_nodesystem();      
 }
index dea297bc500d6b13909b1b26ca5233c4c8bdefd4..aa02fd7d71187f2df6246bdb949df43c3f8b9e1a 100644 (file)
@@ -32,6 +32,7 @@
  */
 
 #include "MEM_guardedalloc.h"
+#include "MEM_CacheLimiterC-Api.h"
 
 #include "BLI_utildefines.h"
 #include "BLI_ghash.h"
@@ -42,6 +43,8 @@
 #include "IMB_imbuf_types.h"
 #include "IMB_imbuf.h"
 
+static MEM_CacheLimiterC *limitor= NULL;
+
 typedef struct MovieCache {
        GHash *hash;
        GHashHashFP hashfp;
@@ -66,6 +69,7 @@ typedef struct MovieCacheKey {
 typedef struct MovieCacheItem {
        MovieCache *cache_owner;
        ImBuf *ibuf;
+       MEM_CacheLimiterHandleC * c_handle;
        unsigned long last_access;
 } MovieCacheItem;
 
@@ -95,33 +99,30 @@ static void moviecache_valfree(void *val)
 {
        MovieCacheItem *item= (MovieCacheItem*)val;
 
-       IMB_freeImBuf(item->ibuf);
+       if (item->ibuf) {
+               MEM_CacheLimiter_unmanage(item->c_handle);
+               IMB_freeImBuf(item->ibuf);
+       }
 
        BLI_mempool_free(item->cache_owner->items_pool, item);
 }
 
-static MovieCacheKey *get_lru_key(MovieCache *cache)
+static void check_unused_keys(MovieCache *cache)
 {
        GHashIterator *iter;
-       MovieCacheKey *lru_key= NULL;
-       MovieCacheItem *lru_item= NULL;
 
        iter= BLI_ghashIterator_new(cache->hash);
        while(!BLI_ghashIterator_isDone(iter)) {
                MovieCacheKey *key= BLI_ghashIterator_getKey(iter);
                MovieCacheItem *item= BLI_ghashIterator_getValue(iter);
 
-               if(lru_item==NULL || item->last_access<lru_item->last_access) {
-                       lru_key= key;
-                       lru_item= item;
-               }
-
                BLI_ghashIterator_step(iter);
+
+               if(!item->ibuf)
+                       BLI_ghash_remove(cache->hash, key, moviecache_keyfree, moviecache_valfree);
        }
 
        BLI_ghashIterator_free(iter);
-
-       return lru_key;
 }
 
 static int compare_int(const void *av, const void *bv)
@@ -131,6 +132,70 @@ static int compare_int(const void *av, const void *bv)
        return *a-*b;
 }
 
+static void IMB_moviecache_destructor(void *p)
+{
+       MovieCacheItem *item= (MovieCacheItem *) p;
+
+       if (item && item->ibuf) {
+               IMB_freeImBuf(item->ibuf);
+
+               item->ibuf= NULL;
+               item->c_handle= NULL;
+       }
+}
+
+/* approximate size of ImBuf in memory */
+static intptr_t IMB_get_size_in_memory(ImBuf *ibuf)
+{
+       int a;
+       intptr_t size= 0, channel_size= 0;
+
+       size+= sizeof(ImBuf);
+
+       if(ibuf->rect)
+               channel_size+= sizeof(char);
+
+       if(ibuf->rect_float)
+               channel_size= sizeof(float);
+
+       size+= channel_size*ibuf->x*ibuf->y*ibuf->channels;
+
+       if(ibuf->miptot) {
+               for(a= 0; a<ibuf->miptot; a++) {
+                       if(ibuf->mipmap[a])
+                               size+= IMB_get_size_in_memory(ibuf->mipmap[a]);
+               }
+       }
+
+       if(ibuf->tiles) {
+               size+= sizeof(unsigned int)*ibuf->ytiles*ibuf->xtiles;
+       }
+
+       return size;
+}
+
+static intptr_t get_item_size (void *p)
+{
+       intptr_t size= sizeof(MovieCacheItem);
+       MovieCacheItem *item= (MovieCacheItem *) p;
+
+       if(item->ibuf)
+               size+= IMB_get_size_in_memory(item->ibuf);
+
+       return size;
+}
+
+void BKE_moviecache_init(void)
+{
+       limitor= new_MEM_CacheLimiter(IMB_moviecache_destructor, get_item_size);
+}
+
+void BKE_moviecache_destruct(void)
+{
+       if(limitor)
+               delete_MEM_CacheLimiter(limitor);
+}
+
 struct MovieCache *BKE_moviecache_create(int keysize, GHashHashFP hashfp, GHashCmpFP cmpfp,
                MovieCacheGetKeyDataFP getdatafp)
 {
@@ -155,11 +220,8 @@ void BKE_moviecache_put(MovieCache *cache, void *userkey, ImBuf *ibuf)
        MovieCacheKey *key;
        MovieCacheItem *item;
 
-       /* TODO: implement better limiters */
-       if(BLI_ghash_size(cache->hash) > 250) {
-               MovieCacheKey *lru_key= get_lru_key(cache);
-               BLI_ghash_remove(cache->hash, lru_key, moviecache_keyfree, moviecache_valfree);
-       }
+       if(!limitor)
+               BKE_moviecache_init();
 
        IMB_refImBuf(ibuf);
 
@@ -172,10 +234,20 @@ void BKE_moviecache_put(MovieCache *cache, void *userkey, ImBuf *ibuf)
        item->ibuf= ibuf;
        item->cache_owner= cache;
        item->last_access= cache->curtime++;
+       item->c_handle= NULL;
 
        BLI_ghash_remove(cache->hash, key, moviecache_keyfree, moviecache_valfree);
        BLI_ghash_insert(cache->hash, key, item);
 
+       item->c_handle= MEM_CacheLimiter_insert(limitor, item);
+
+       MEM_CacheLimiter_ref(item->c_handle);
+       MEM_CacheLimiter_enforce_limits(limitor);
+       MEM_CacheLimiter_unref(item->c_handle);
+
+       /* cache limiter can't remove unused keys which points to destoryed values */
+       check_unused_keys(cache);
+
        if(cache->points) {
                MEM_freeN(cache->points);
                cache->points= NULL;
@@ -195,6 +267,7 @@ ImBuf* BKE_moviecache_get(MovieCache *cache, void *userkey)
                item->last_access= cache->curtime++;
 
                if(item->ibuf) {
+                       MEM_CacheLimiter_touch(item->c_handle);
                        IMB_refImBuf(item->ibuf);
                        return item->ibuf;
                }
@@ -239,11 +312,14 @@ void BKE_moviecache_get_cache_segments(MovieCache *cache, int *totseg_r, int **p
                a= 0;
                while(!BLI_ghashIterator_isDone(iter)) {
                        MovieCacheKey *key= BLI_ghashIterator_getKey(iter);
+                       MovieCacheItem *item= BLI_ghashIterator_getValue(iter);
                        int framenr;
 
-                       cache->getdatafp(key->userkey, &framenr);
+                       if(item->ibuf) {
+                               cache->getdatafp(key->userkey, &framenr);
 
-                       frames[a++]= framenr;
+                               frames[a++]= framenr;
+                       }
 
                        BLI_ghashIterator_step(iter);
                }
index 00f88fb6202f773dd4d05b773e85da14f82aadd6..0077c52b9f8b504d94480a5f2ed40d4aa9191e36 100644 (file)
@@ -148,7 +148,7 @@ static void IMB_seq_cache_destructor(void * p)
 void seq_stripelem_cache_init(void)
 {
        hash = BLI_ghash_new(HashHash, HashCmp, "seq stripelem cache hash");
-       limitor = new_MEM_CacheLimiter( IMB_seq_cache_destructor );
+       limitor = new_MEM_CacheLimiter( IMB_seq_cache_destructor, NULL );
 
        entrypool = BLI_mempool_create(sizeof(seqCacheEntry), 64, 64, 0);
        keypool = BLI_mempool_create(sizeof(seqCacheKey), 64, 64, 0);
index 59772771f3b3359c61899c07a9401f60d4ce39fe..f71235b0a9de2cbe6af74cdee8a7c3eee808cf77 100644 (file)
@@ -455,7 +455,7 @@ static MEM_CacheLimiterC **get_imbuf_cache_limiter(void)
        static MEM_CacheLimiterC *c = NULL;
 
        if(!c)
-               c = new_MEM_CacheLimiter(imbuf_cache_destructor);
+               c = new_MEM_CacheLimiter(imbuf_cache_destructor, NULL);
 
        return &c;
 }