Merge branch 'master' into blender2.8
[blender.git] / source / blender / blenkernel / intern / pointcache.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
19  * All rights reserved.
20  *
21  * Contributor(s): Campbell Barton <ideasman42@gmail.com>
22  *
23  * ***** END GPL LICENSE BLOCK *****
24  */
25
26 /** \file blender/blenkernel/intern/pointcache.c
27  *  \ingroup bke
28  */
29
30
31 #include <stdlib.h>
32 #include <stdio.h>
33 #include <string.h>
34 #include <sys/stat.h>
35 #include <sys/types.h>
36
37 #include "MEM_guardedalloc.h"
38
39 #include "DNA_ID.h"
40 #include "DNA_dynamicpaint_types.h"
41 #include "DNA_group_types.h"
42 #include "DNA_modifier_types.h"
43 #include "DNA_object_types.h"
44 #include "DNA_object_force_types.h"
45 #include "DNA_particle_types.h"
46 #include "DNA_rigidbody_types.h"
47 #include "DNA_scene_types.h"
48 #include "DNA_smoke_types.h"
49
50 #include "BLI_blenlib.h"
51 #include "BLI_threads.h"
52 #include "BLI_math.h"
53 #include "BLI_string.h"
54 #include "BLI_utildefines.h"
55
56 #include "BLT_translation.h"
57
58 #include "PIL_time.h"
59
60 #include "BKE_appdir.h"
61 #include "BKE_anim.h"
62 #include "BKE_cloth.h"
63 #include "BKE_collection.h"
64 #include "BKE_dynamicpaint.h"
65 #include "BKE_global.h"
66 #include "BKE_main.h"
67 #include "BKE_modifier.h"
68 #include "BKE_object.h"
69 #include "BKE_particle.h"
70 #include "BKE_pointcache.h"
71 #include "BKE_scene.h"
72 #include "BKE_smoke.h"
73 #include "BKE_softbody.h"
74
75 #include "BIK_api.h"
76
77 #include "DEG_depsgraph.h"
78
79 #ifdef WITH_BULLET
80 #  include "RBI_api.h"
81 #endif
82
83 /* both in intern */
84 #ifdef WITH_SMOKE
85 #include "smoke_API.h"
86 #endif
87
88 #ifdef WITH_OPENVDB
89 #include "openvdb_capi.h"
90 #endif
91
92 #ifdef WITH_LZO
93 #  ifdef WITH_SYSTEM_LZO
94 #    include <lzo/lzo1x.h>
95 #  else
96 #    include "minilzo.h"
97 #  endif
98 #  define LZO_HEAP_ALLOC(var,size) \
99         lzo_align_t __LZO_MMODEL var [ ((size) + (sizeof(lzo_align_t) - 1)) / sizeof(lzo_align_t) ]
100 #endif
101
102 #define LZO_OUT_LEN(size)     ((size) + (size) / 16 + 64 + 3)
103
104 #ifdef WITH_LZMA
105 #include "LzmaLib.h"
106 #endif
107
108 /* needed for directory lookup */
109 #ifndef WIN32
110 #  include <dirent.h>
111 #else
112 #  include "BLI_winstuff.h"
113 #endif
114
115 #define PTCACHE_DATA_FROM(data, type, from)  \
116         if (data[type]) { \
117                 memcpy(data[type], from, ptcache_data_size[type]); \
118         } (void)0
119
120 #define PTCACHE_DATA_TO(data, type, index, to)  \
121         if (data[type]) { \
122                 memcpy(to, (char *)(data)[type] + ((index) ? (index) * ptcache_data_size[type] : 0), ptcache_data_size[type]); \
123         } (void)0
124
125 /* could be made into a pointcache option */
126 #define DURIAN_POINTCACHE_LIB_OK 1
127
128 static int ptcache_data_size[] = {      
129                 sizeof(unsigned int), // BPHYS_DATA_INDEX
130                 3 * sizeof(float), // BPHYS_DATA_LOCATION
131                 3 * sizeof(float), // BPHYS_DATA_VELOCITY
132                 4 * sizeof(float), // BPHYS_DATA_ROTATION
133                 3 * sizeof(float), // BPHYS_DATA_AVELOCITY / BPHYS_DATA_XCONST
134                 sizeof(float), // BPHYS_DATA_SIZE
135                 3 * sizeof(float), // BPHYS_DATA_TIMES
136                 sizeof(BoidData) // case BPHYS_DATA_BOIDS
137 };
138
139 static int ptcache_extra_datasize[] = {
140         0,
141         sizeof(ParticleSpring)
142 };
143
144 /* forward declerations */
145 static int ptcache_file_compressed_read(PTCacheFile *pf, unsigned char *result, unsigned int len);
146 static int ptcache_file_compressed_write(PTCacheFile *pf, unsigned char *in, unsigned int in_len, unsigned char *out, int mode);
147 static int ptcache_file_write(PTCacheFile *pf, const void *f, unsigned int tot, unsigned int size);
148 static int ptcache_file_read(PTCacheFile *pf, void *f, unsigned int tot, unsigned int size);
149
150 /* Common functions */
151 static int ptcache_basic_header_read(PTCacheFile *pf)
152 {
153         int error=0;
154
155         /* Custom functions should read these basic elements too! */
156         if (!error && !fread(&pf->totpoint, sizeof(unsigned int), 1, pf->fp))
157                 error = 1;
158         
159         if (!error && !fread(&pf->data_types, sizeof(unsigned int), 1, pf->fp))
160                 error = 1;
161
162         return !error;
163 }
164 static int ptcache_basic_header_write(PTCacheFile *pf)
165 {
166         /* Custom functions should write these basic elements too! */
167         if (!fwrite(&pf->totpoint, sizeof(unsigned int), 1, pf->fp))
168                 return 0;
169         
170         if (!fwrite(&pf->data_types, sizeof(unsigned int), 1, pf->fp))
171                 return 0;
172
173         return 1;
174 }
175 /* Softbody functions */
176 static int  ptcache_softbody_write(int index, void *soft_v, void **data, int UNUSED(cfra))
177 {
178         SoftBody *soft= soft_v;
179         BodyPoint *bp = soft->bpoint + index;
180
181         PTCACHE_DATA_FROM(data, BPHYS_DATA_LOCATION, bp->pos);
182         PTCACHE_DATA_FROM(data, BPHYS_DATA_VELOCITY, bp->vec);
183
184         return 1;
185 }
186 static void ptcache_softbody_read(int index, void *soft_v, void **data, float UNUSED(cfra), float *old_data)
187 {
188         SoftBody *soft= soft_v;
189         BodyPoint *bp = soft->bpoint + index;
190
191         if (old_data) {
192                 memcpy(bp->pos, data, 3 * sizeof(float));
193                 memcpy(bp->vec, data + 3, 3 * sizeof(float));
194         }
195         else {
196                 PTCACHE_DATA_TO(data, BPHYS_DATA_LOCATION, 0, bp->pos);
197                 PTCACHE_DATA_TO(data, BPHYS_DATA_VELOCITY, 0, bp->vec);
198         }
199 }
200 static void ptcache_softbody_interpolate(int index, void *soft_v, void **data, float cfra, float cfra1, float cfra2, float *old_data)
201 {
202         SoftBody *soft= soft_v;
203         BodyPoint *bp = soft->bpoint + index;
204         ParticleKey keys[4];
205         float dfra;
206
207         if (cfra1 == cfra2)
208                 return;
209
210         copy_v3_v3(keys[1].co, bp->pos);
211         copy_v3_v3(keys[1].vel, bp->vec);
212
213         if (old_data) {
214                 memcpy(keys[2].co, old_data, 3 * sizeof(float));
215                 memcpy(keys[2].vel, old_data + 3, 3 * sizeof(float));
216         }
217         else
218                 BKE_ptcache_make_particle_key(keys+2, 0, data, cfra2);
219
220         dfra = cfra2 - cfra1;
221
222         mul_v3_fl(keys[1].vel, dfra);
223         mul_v3_fl(keys[2].vel, dfra);
224
225         psys_interpolate_particle(-1, keys, (cfra - cfra1) / dfra, keys, 1);
226
227         mul_v3_fl(keys->vel, 1.0f / dfra);
228
229         copy_v3_v3(bp->pos, keys->co);
230         copy_v3_v3(bp->vec, keys->vel);
231 }
232 static int  ptcache_softbody_totpoint(void *soft_v, int UNUSED(cfra))
233 {
234         SoftBody *soft= soft_v;
235         return soft->totpoint;
236 }
237 static void ptcache_softbody_error(void *UNUSED(soft_v), const char *UNUSED(message))
238 {
239         /* ignored for now */
240 }
241
242 /* Particle functions */
243 void BKE_ptcache_make_particle_key(ParticleKey *key, int index, void **data, float time)
244 {
245         PTCACHE_DATA_TO(data, BPHYS_DATA_LOCATION, index, key->co);
246         PTCACHE_DATA_TO(data, BPHYS_DATA_VELOCITY, index, key->vel);
247         
248         /* no rotation info, so make something nice up */
249         if (data[BPHYS_DATA_ROTATION]==NULL) {
250                 vec_to_quat(key->rot, key->vel, OB_NEGX, OB_POSZ);
251         }
252         else {
253                 PTCACHE_DATA_TO(data, BPHYS_DATA_ROTATION, index, key->rot);
254         }
255
256         PTCACHE_DATA_TO(data, BPHYS_DATA_AVELOCITY, index, key->ave);
257         key->time = time;
258 }
259 static int  ptcache_particle_write(int index, void *psys_v, void **data, int cfra)
260 {
261         ParticleSystem *psys= psys_v;
262         ParticleData *pa = psys->particles + index;
263         BoidParticle *boid = (psys->part->phystype == PART_PHYS_BOIDS) ? pa->boid : NULL;
264         float times[3];
265         int step = psys->pointcache->step;
266
267         /* No need to store unborn or died particles outside cache step bounds */
268         if (data[BPHYS_DATA_INDEX] && (cfra < pa->time - step || cfra > pa->dietime + step))
269                 return 0;
270
271         times[0] = pa->time;
272         times[1] = pa->dietime;
273         times[2] = pa->lifetime;
274
275         PTCACHE_DATA_FROM(data, BPHYS_DATA_INDEX, &index);
276         PTCACHE_DATA_FROM(data, BPHYS_DATA_LOCATION, pa->state.co);
277         PTCACHE_DATA_FROM(data, BPHYS_DATA_VELOCITY, pa->state.vel);
278         PTCACHE_DATA_FROM(data, BPHYS_DATA_ROTATION, pa->state.rot);
279         PTCACHE_DATA_FROM(data, BPHYS_DATA_AVELOCITY, pa->state.ave);
280         PTCACHE_DATA_FROM(data, BPHYS_DATA_SIZE, &pa->size);
281         PTCACHE_DATA_FROM(data, BPHYS_DATA_TIMES, times);
282
283         if (boid) {
284                 PTCACHE_DATA_FROM(data, BPHYS_DATA_BOIDS, &boid->data);
285         }
286
287         /* return flag 1+1=2 for newly born particles to copy exact birth location to previously cached frame */
288         return 1 + (pa->state.time >= pa->time && pa->prev_state.time <= pa->time);
289 }
290 static void ptcache_particle_read(int index, void *psys_v, void **data, float cfra, float *old_data)
291 {
292         ParticleSystem *psys= psys_v;
293         ParticleData *pa;
294         BoidParticle *boid;
295         float timestep = 0.04f * psys->part->timetweak;
296
297         if (index >= psys->totpart)
298                 return;
299
300         pa = psys->particles + index;
301         boid = (psys->part->phystype == PART_PHYS_BOIDS) ? pa->boid : NULL;
302
303         if (cfra > pa->state.time)
304                 memcpy(&pa->prev_state, &pa->state, sizeof(ParticleKey));
305
306         if (old_data) {
307                 /* old format cache */
308                 memcpy(&pa->state, old_data, sizeof(ParticleKey));
309                 return;
310         }
311
312         BKE_ptcache_make_particle_key(&pa->state, 0, data, cfra);
313
314         /* set frames cached before birth to birth time */
315         if (cfra < pa->time)
316                 pa->state.time = pa->time;
317         else if (cfra > pa->dietime)
318                 pa->state.time = pa->dietime;
319
320         if (data[BPHYS_DATA_SIZE]) {
321                 PTCACHE_DATA_TO(data, BPHYS_DATA_SIZE, 0, &pa->size);
322         }
323         
324         if (data[BPHYS_DATA_TIMES]) {
325                 float times[3];
326                 PTCACHE_DATA_TO(data, BPHYS_DATA_TIMES, 0, &times);
327                 pa->time = times[0];
328                 pa->dietime = times[1];
329                 pa->lifetime = times[2];
330         }
331
332         if (boid) {
333                 PTCACHE_DATA_TO(data, BPHYS_DATA_BOIDS, 0, &boid->data);
334         }
335
336         /* determine velocity from previous location */
337         if (data[BPHYS_DATA_LOCATION] && !data[BPHYS_DATA_VELOCITY]) {
338                 if (cfra > pa->prev_state.time) {
339                         sub_v3_v3v3(pa->state.vel, pa->state.co, pa->prev_state.co);
340                         mul_v3_fl(pa->state.vel, (cfra - pa->prev_state.time) * timestep);
341                 }
342                 else {
343                         sub_v3_v3v3(pa->state.vel, pa->prev_state.co, pa->state.co);
344                         mul_v3_fl(pa->state.vel, (pa->prev_state.time - cfra) * timestep);
345                 }
346         }
347
348         /* default to no rotation */
349         if (data[BPHYS_DATA_LOCATION] && !data[BPHYS_DATA_ROTATION]) {
350                 unit_qt(pa->state.rot);
351         }
352 }
353 static void ptcache_particle_interpolate(int index, void *psys_v, void **data, float cfra, float cfra1, float cfra2, float *old_data)
354 {
355         ParticleSystem *psys= psys_v;
356         ParticleData *pa;
357         ParticleKey keys[4];
358         float dfra, timestep = 0.04f * psys->part->timetweak;
359
360         if (index >= psys->totpart)
361                 return;
362
363         pa = psys->particles + index;
364
365         /* particle wasn't read from first cache so can't interpolate */
366         if ((int)cfra1 < pa->time - psys->pointcache->step || (int)cfra1 > pa->dietime + psys->pointcache->step)
367                 return;
368
369         cfra = MIN2(cfra, pa->dietime);
370         cfra1 = MIN2(cfra1, pa->dietime);
371         cfra2 = MIN2(cfra2, pa->dietime);
372
373         if (cfra1 == cfra2)
374                 return;
375
376         memcpy(keys+1, &pa->state, sizeof(ParticleKey));
377         if (old_data)
378                 memcpy(keys+2, old_data, sizeof(ParticleKey));
379         else
380                 BKE_ptcache_make_particle_key(keys+2, 0, data, cfra2);
381
382         /* determine velocity from previous location */
383         if (data[BPHYS_DATA_LOCATION] && !data[BPHYS_DATA_VELOCITY]) {
384                 if (keys[1].time > keys[2].time) {
385                         sub_v3_v3v3(keys[2].vel, keys[1].co, keys[2].co);
386                         mul_v3_fl(keys[2].vel, (keys[1].time - keys[2].time) * timestep);
387                 }
388                 else {
389                         sub_v3_v3v3(keys[2].vel, keys[2].co, keys[1].co);
390                         mul_v3_fl(keys[2].vel, (keys[2].time - keys[1].time) * timestep);
391                 }
392         }
393
394         /* default to no rotation */
395         if (data[BPHYS_DATA_LOCATION] && !data[BPHYS_DATA_ROTATION]) {
396                 unit_qt(keys[2].rot);
397         }
398
399         if (cfra > pa->time)
400                 cfra1 = MAX2(cfra1, pa->time);
401
402         dfra = cfra2 - cfra1;
403
404         mul_v3_fl(keys[1].vel, dfra * timestep);
405         mul_v3_fl(keys[2].vel, dfra * timestep);
406
407         psys_interpolate_particle(-1, keys, (cfra - cfra1) / dfra, &pa->state, 1);
408         interp_qt_qtqt(pa->state.rot, keys[1].rot, keys[2].rot, (cfra - cfra1) / dfra);
409
410         mul_v3_fl(pa->state.vel, 1.f / (dfra * timestep));
411
412         pa->state.time = cfra;
413 }
414
415 static int  ptcache_particle_totpoint(void *psys_v, int UNUSED(cfra))
416 {
417         ParticleSystem *psys = psys_v;
418         return psys->totpart;
419 }
420
421 static void ptcache_particle_error(void *UNUSED(psys_v), const char *UNUSED(message))
422 {
423         /* ignored for now */
424 }
425
426 static int  ptcache_particle_totwrite(void *psys_v, int cfra)
427 {
428         ParticleSystem *psys = psys_v;
429         ParticleData *pa= psys->particles;
430         int p, step = psys->pointcache->step;
431         int totwrite = 0;
432
433         if (cfra == 0)
434                 return psys->totpart;
435
436         for (p=0; p<psys->totpart; p++, pa++)
437                 totwrite += (cfra >= pa->time - step && cfra <= pa->dietime + step);
438
439         return totwrite;
440 }
441
442 static void ptcache_particle_extra_write(void *psys_v, PTCacheMem *pm, int UNUSED(cfra))
443 {
444         ParticleSystem *psys = psys_v;
445         PTCacheExtra *extra = NULL;
446
447         if (psys->part->phystype == PART_PHYS_FLUID &&
448             psys->part->fluid && psys->part->fluid->flag & SPH_VISCOELASTIC_SPRINGS &&
449             psys->tot_fluidsprings && psys->fluid_springs)
450         {
451                 extra = MEM_callocN(sizeof(PTCacheExtra), "Point cache: fluid extra data");
452
453                 extra->type = BPHYS_EXTRA_FLUID_SPRINGS;
454                 extra->totdata = psys->tot_fluidsprings;
455
456                 extra->data = MEM_callocN(extra->totdata * ptcache_extra_datasize[extra->type], "Point cache: extra data");
457                 memcpy(extra->data, psys->fluid_springs, extra->totdata * ptcache_extra_datasize[extra->type]);
458
459                 BLI_addtail(&pm->extradata, extra);
460         }
461 }
462
463 static void ptcache_particle_extra_read(void *psys_v, PTCacheMem *pm, float UNUSED(cfra))
464 {
465         ParticleSystem *psys = psys_v;
466         PTCacheExtra *extra = pm->extradata.first;
467
468         for (; extra; extra=extra->next) {
469                 switch (extra->type) {
470                         case BPHYS_EXTRA_FLUID_SPRINGS:
471                         {
472                                 if (psys->fluid_springs)
473                                         MEM_freeN(psys->fluid_springs);
474
475                                 psys->fluid_springs = MEM_dupallocN(extra->data);
476                                 psys->tot_fluidsprings = psys->alloc_fluidsprings = extra->totdata;
477                                 break;
478                         }
479                 }
480         }
481 }
482
483 /* Cloth functions */
484 static int  ptcache_cloth_write(int index, void *cloth_v, void **data, int UNUSED(cfra))
485 {
486         ClothModifierData *clmd= cloth_v;
487         Cloth *cloth= clmd->clothObject;
488         ClothVertex *vert = cloth->verts + index;
489
490         PTCACHE_DATA_FROM(data, BPHYS_DATA_LOCATION, vert->x);
491         PTCACHE_DATA_FROM(data, BPHYS_DATA_VELOCITY, vert->v);
492         PTCACHE_DATA_FROM(data, BPHYS_DATA_XCONST, vert->xconst);
493
494         return 1;
495 }
496 static void ptcache_cloth_read(int index, void *cloth_v, void **data, float UNUSED(cfra), float *old_data)
497 {
498         ClothModifierData *clmd= cloth_v;
499         Cloth *cloth= clmd->clothObject;
500         ClothVertex *vert = cloth->verts + index;
501         
502         if (old_data) {
503                 memcpy(vert->x, data, 3 * sizeof(float));
504                 memcpy(vert->xconst, data + 3, 3 * sizeof(float));
505                 memcpy(vert->v, data + 6, 3 * sizeof(float));
506         }
507         else {
508                 PTCACHE_DATA_TO(data, BPHYS_DATA_LOCATION, 0, vert->x);
509                 PTCACHE_DATA_TO(data, BPHYS_DATA_VELOCITY, 0, vert->v);
510                 PTCACHE_DATA_TO(data, BPHYS_DATA_XCONST, 0, vert->xconst);
511         }
512 }
513 static void ptcache_cloth_interpolate(int index, void *cloth_v, void **data, float cfra, float cfra1, float cfra2, float *old_data)
514 {
515         ClothModifierData *clmd= cloth_v;
516         Cloth *cloth= clmd->clothObject;
517         ClothVertex *vert = cloth->verts + index;
518         ParticleKey keys[4];
519         float dfra;
520
521         if (cfra1 == cfra2)
522                 return;
523
524         copy_v3_v3(keys[1].co, vert->x);
525         copy_v3_v3(keys[1].vel, vert->v);
526
527         if (old_data) {
528                 memcpy(keys[2].co, old_data, 3 * sizeof(float));
529                 memcpy(keys[2].vel, old_data + 6, 3 * sizeof(float));
530         }
531         else
532                 BKE_ptcache_make_particle_key(keys+2, 0, data, cfra2);
533
534         dfra = cfra2 - cfra1;
535
536         mul_v3_fl(keys[1].vel, dfra);
537         mul_v3_fl(keys[2].vel, dfra);
538
539         psys_interpolate_particle(-1, keys, (cfra - cfra1) / dfra, keys, 1);
540
541         mul_v3_fl(keys->vel, 1.0f / dfra);
542
543         copy_v3_v3(vert->x, keys->co);
544         copy_v3_v3(vert->v, keys->vel);
545
546         /* should vert->xconst be interpolated somehow too? - jahka */
547 }
548
549 static int  ptcache_cloth_totpoint(void *cloth_v, int UNUSED(cfra))
550 {
551         ClothModifierData *clmd= cloth_v;
552         return clmd->clothObject ? clmd->clothObject->mvert_num : 0;
553 }
554
555 static void ptcache_cloth_error(void *cloth_v, const char *message)
556 {
557         ClothModifierData *clmd= cloth_v;
558         modifier_setError(&clmd->modifier, "%s", message);
559 }
560
561 #ifdef WITH_SMOKE
562 /* Smoke functions */
563 static int  ptcache_smoke_totpoint(void *smoke_v, int UNUSED(cfra))
564 {
565         SmokeModifierData *smd= (SmokeModifierData *)smoke_v;
566         SmokeDomainSettings *sds = smd->domain;
567         
568         if (sds->fluid) {
569                 return sds->base_res[0]*sds->base_res[1]*sds->base_res[2];
570         }
571         else
572                 return 0;
573 }
574
575 static void ptcache_smoke_error(void *smoke_v, const char *message)
576 {
577         SmokeModifierData *smd= (SmokeModifierData *)smoke_v;
578         modifier_setError(&smd->modifier, "%s", message);
579 }
580
581 #define SMOKE_CACHE_VERSION "1.04"
582
583 static int  ptcache_smoke_write(PTCacheFile *pf, void *smoke_v)
584 {       
585         SmokeModifierData *smd= (SmokeModifierData *)smoke_v;
586         SmokeDomainSettings *sds = smd->domain;
587         int ret = 0;
588         int fluid_fields = smoke_get_data_flags(sds);
589
590         /* version header */
591         ptcache_file_write(pf, SMOKE_CACHE_VERSION, 4, sizeof(char));
592         ptcache_file_write(pf, &fluid_fields, 1, sizeof(int));
593         ptcache_file_write(pf, &sds->active_fields, 1, sizeof(int));
594         ptcache_file_write(pf, &sds->res, 3, sizeof(int));
595         ptcache_file_write(pf, &sds->dx, 1, sizeof(float));
596         
597         if (sds->fluid) {
598                 size_t res = sds->res[0]*sds->res[1]*sds->res[2];
599                 float dt, dx, *dens, *react, *fuel, *flame, *heat, *heatold, *vx, *vy, *vz, *r, *g, *b;
600                 unsigned char *obstacles;
601                 unsigned int in_len = sizeof(float)*(unsigned int)res;
602                 unsigned char *out = (unsigned char *)MEM_callocN(LZO_OUT_LEN(in_len) * 4, "pointcache_lzo_buffer");
603                 //int mode = res >= 1000000 ? 2 : 1;
604                 int mode=1;             // light
605                 if (sds->cache_comp == SM_CACHE_HEAVY) mode=2;  // heavy
606
607                 smoke_export(sds->fluid, &dt, &dx, &dens, &react, &flame, &fuel, &heat, &heatold, &vx, &vy, &vz, &r, &g, &b, &obstacles);
608
609                 ptcache_file_compressed_write(pf, (unsigned char *)sds->shadow, in_len, out, mode);
610                 ptcache_file_compressed_write(pf, (unsigned char *)dens, in_len, out, mode);
611                 if (fluid_fields & SM_ACTIVE_HEAT) {
612                         ptcache_file_compressed_write(pf, (unsigned char *)heat, in_len, out, mode);
613                         ptcache_file_compressed_write(pf, (unsigned char *)heatold, in_len, out, mode);
614                 }
615                 if (fluid_fields & SM_ACTIVE_FIRE) {
616                         ptcache_file_compressed_write(pf, (unsigned char *)flame, in_len, out, mode);
617                         ptcache_file_compressed_write(pf, (unsigned char *)fuel, in_len, out, mode);
618                         ptcache_file_compressed_write(pf, (unsigned char *)react, in_len, out, mode);
619                 }
620                 if (fluid_fields & SM_ACTIVE_COLORS) {
621                         ptcache_file_compressed_write(pf, (unsigned char *)r, in_len, out, mode);
622                         ptcache_file_compressed_write(pf, (unsigned char *)g, in_len, out, mode);
623                         ptcache_file_compressed_write(pf, (unsigned char *)b, in_len, out, mode);
624                 }
625                 ptcache_file_compressed_write(pf, (unsigned char *)vx, in_len, out, mode);
626                 ptcache_file_compressed_write(pf, (unsigned char *)vy, in_len, out, mode);
627                 ptcache_file_compressed_write(pf, (unsigned char *)vz, in_len, out, mode);
628                 ptcache_file_compressed_write(pf, (unsigned char *)obstacles, (unsigned int)res, out, mode);
629                 ptcache_file_write(pf, &dt, 1, sizeof(float));
630                 ptcache_file_write(pf, &dx, 1, sizeof(float));
631                 ptcache_file_write(pf, &sds->p0, 3, sizeof(float));
632                 ptcache_file_write(pf, &sds->p1, 3, sizeof(float));
633                 ptcache_file_write(pf, &sds->dp0, 3, sizeof(float));
634                 ptcache_file_write(pf, &sds->shift, 3, sizeof(int));
635                 ptcache_file_write(pf, &sds->obj_shift_f, 3, sizeof(float));
636                 ptcache_file_write(pf, &sds->obmat, 16, sizeof(float));
637                 ptcache_file_write(pf, &sds->base_res, 3, sizeof(int));
638                 ptcache_file_write(pf, &sds->res_min, 3, sizeof(int));
639                 ptcache_file_write(pf, &sds->res_max, 3, sizeof(int));
640                 ptcache_file_write(pf, &sds->active_color, 3, sizeof(float));
641
642                 MEM_freeN(out);
643                 
644                 ret = 1;
645         }
646
647         if (sds->wt) {
648                 int res_big_array[3];
649                 int res_big;
650                 int res = sds->res[0]*sds->res[1]*sds->res[2];
651                 float *dens, *react, *fuel, *flame, *tcu, *tcv, *tcw, *r, *g, *b;
652                 unsigned int in_len = sizeof(float)*(unsigned int)res;
653                 unsigned int in_len_big;
654                 unsigned char *out;
655                 int mode;
656
657                 smoke_turbulence_get_res(sds->wt, res_big_array);
658                 res_big = res_big_array[0]*res_big_array[1]*res_big_array[2];
659                 //mode =  res_big >= 1000000 ? 2 : 1;
660                 mode = 1;       // light
661                 if (sds->cache_high_comp == SM_CACHE_HEAVY) mode=2;     // heavy
662
663                 in_len_big = sizeof(float) * (unsigned int)res_big;
664
665                 smoke_turbulence_export(sds->wt, &dens, &react, &flame, &fuel, &r, &g, &b, &tcu, &tcv, &tcw);
666
667                 out = (unsigned char *)MEM_callocN(LZO_OUT_LEN(in_len_big), "pointcache_lzo_buffer");
668                 ptcache_file_compressed_write(pf, (unsigned char *)dens, in_len_big, out, mode);
669                 if (fluid_fields & SM_ACTIVE_FIRE) {
670                         ptcache_file_compressed_write(pf, (unsigned char *)flame, in_len_big, out, mode);
671                         ptcache_file_compressed_write(pf, (unsigned char *)fuel, in_len_big, out, mode);
672                         ptcache_file_compressed_write(pf, (unsigned char *)react, in_len_big, out, mode);
673                 }
674                 if (fluid_fields & SM_ACTIVE_COLORS) {
675                         ptcache_file_compressed_write(pf, (unsigned char *)r, in_len_big, out, mode);
676                         ptcache_file_compressed_write(pf, (unsigned char *)g, in_len_big, out, mode);
677                         ptcache_file_compressed_write(pf, (unsigned char *)b, in_len_big, out, mode);
678                 }
679                 MEM_freeN(out);
680
681                 out = (unsigned char *)MEM_callocN(LZO_OUT_LEN(in_len), "pointcache_lzo_buffer");
682                 ptcache_file_compressed_write(pf, (unsigned char *)tcu, in_len, out, mode);
683                 ptcache_file_compressed_write(pf, (unsigned char *)tcv, in_len, out, mode);
684                 ptcache_file_compressed_write(pf, (unsigned char *)tcw, in_len, out, mode);
685                 MEM_freeN(out);
686                 
687                 ret = 1;
688         }
689
690         return ret;
691 }
692
693 /* read old smoke cache from 2.64 */
694 static int ptcache_smoke_read_old(PTCacheFile *pf, void *smoke_v)
695 {
696         SmokeModifierData *smd= (SmokeModifierData *)smoke_v;
697         SmokeDomainSettings *sds = smd->domain;
698         
699         if (sds->fluid) {
700                 const size_t res = sds->res[0] * sds->res[1] * sds->res[2];
701                 const unsigned int out_len = (unsigned int)res * sizeof(float);
702                 float dt, dx, *dens, *heat, *heatold, *vx, *vy, *vz;
703                 unsigned char *obstacles;
704                 float *tmp_array = MEM_callocN(out_len, "Smoke old cache tmp");
705
706                 int fluid_fields = smoke_get_data_flags(sds);
707
708                 /* Part part of the new cache header */
709                 sds->active_color[0] = 0.7f;
710                 sds->active_color[1] = 0.7f;
711                 sds->active_color[2] = 0.7f;
712                 
713                 smoke_export(sds->fluid, &dt, &dx, &dens, NULL, NULL, NULL, &heat, &heatold, &vx, &vy, &vz, NULL, NULL, NULL, &obstacles);
714
715                 ptcache_file_compressed_read(pf, (unsigned char *)sds->shadow, out_len);
716                 ptcache_file_compressed_read(pf, (unsigned char*)dens, out_len);
717                 ptcache_file_compressed_read(pf, (unsigned char*)tmp_array, out_len);
718
719                 if (fluid_fields & SM_ACTIVE_HEAT)
720                 {
721                         ptcache_file_compressed_read(pf, (unsigned char*)heat, out_len);
722                         ptcache_file_compressed_read(pf, (unsigned char*)heatold, out_len);
723                 }
724                 else
725                 {
726                         ptcache_file_compressed_read(pf, (unsigned char*)tmp_array, out_len);
727                         ptcache_file_compressed_read(pf, (unsigned char*)tmp_array, out_len);
728                 }
729                 ptcache_file_compressed_read(pf, (unsigned char*)vx, out_len);
730                 ptcache_file_compressed_read(pf, (unsigned char*)vy, out_len);
731                 ptcache_file_compressed_read(pf, (unsigned char*)vz, out_len);
732                 ptcache_file_compressed_read(pf, (unsigned char*)tmp_array, out_len);
733                 ptcache_file_compressed_read(pf, (unsigned char*)tmp_array, out_len);
734                 ptcache_file_compressed_read(pf, (unsigned char*)tmp_array, out_len);
735                 ptcache_file_compressed_read(pf, (unsigned char*)obstacles, (unsigned int)res);
736                 ptcache_file_read(pf, &dt, 1, sizeof(float));
737                 ptcache_file_read(pf, &dx, 1, sizeof(float));
738
739                 MEM_freeN(tmp_array);
740
741                 if (pf->data_types & (1<<BPHYS_DATA_SMOKE_HIGH) && sds->wt) {
742                         int res_big, res_big_array[3];
743                         float *tcu, *tcv, *tcw;
744                         unsigned int out_len_big;
745                         unsigned char *tmp_array_big;
746                         smoke_turbulence_get_res(sds->wt, res_big_array);
747                         res_big = res_big_array[0]*res_big_array[1]*res_big_array[2];
748                         out_len_big = sizeof(float) * (unsigned int)res_big;
749
750                         tmp_array_big = MEM_callocN(out_len_big, "Smoke old cache tmp");
751
752                         smoke_turbulence_export(sds->wt, &dens, NULL, NULL, NULL, NULL, NULL, NULL, &tcu, &tcv, &tcw);
753
754                         ptcache_file_compressed_read(pf, (unsigned char*)dens, out_len_big);
755                         ptcache_file_compressed_read(pf, (unsigned char*)tmp_array_big, out_len_big);
756
757                         ptcache_file_compressed_read(pf, (unsigned char*)tcu, out_len);
758                         ptcache_file_compressed_read(pf, (unsigned char*)tcv, out_len);
759                         ptcache_file_compressed_read(pf, (unsigned char*)tcw, out_len);
760
761                         MEM_freeN(tmp_array_big);
762                 }
763         }
764
765         return 1;       
766 }
767
768 static int ptcache_smoke_read(PTCacheFile *pf, void *smoke_v)
769 {
770         SmokeModifierData *smd= (SmokeModifierData *)smoke_v;
771         SmokeDomainSettings *sds = smd->domain;
772         char version[4];
773         int ch_res[3];
774         float ch_dx;
775         int fluid_fields = smoke_get_data_flags(sds);
776         int cache_fields = 0;
777         int active_fields = 0;
778         int reallocate = 0;
779
780         /* version header */
781         ptcache_file_read(pf, version, 4, sizeof(char));
782         if (!STREQLEN(version, SMOKE_CACHE_VERSION, 4))
783         {
784                 /* reset file pointer */
785                 fseek(pf->fp, -4, SEEK_CUR);
786                 return ptcache_smoke_read_old(pf, smoke_v);
787         }
788
789         /* fluid info */
790         ptcache_file_read(pf, &cache_fields, 1, sizeof(int));
791         ptcache_file_read(pf, &active_fields, 1, sizeof(int));
792         ptcache_file_read(pf, &ch_res, 3, sizeof(int));
793         ptcache_file_read(pf, &ch_dx, 1, sizeof(float));
794
795         /* check if resolution has changed */
796         if (sds->res[0] != ch_res[0] ||
797             sds->res[1] != ch_res[1] ||
798             sds->res[2] != ch_res[2])
799         {
800                 if (sds->flags & MOD_SMOKE_ADAPTIVE_DOMAIN)
801                         reallocate = 1;
802                 else
803                         return 0;
804         }
805         /* check if active fields have changed */
806         if (fluid_fields != cache_fields ||
807                 active_fields != sds->active_fields)
808                 reallocate = 1;
809
810         /* reallocate fluid if needed*/
811         if (reallocate) {
812                 sds->active_fields = active_fields | cache_fields;
813                 smoke_reallocate_fluid(sds, ch_dx, ch_res, 1);
814                 sds->dx = ch_dx;
815                 VECCOPY(sds->res, ch_res);
816                 sds->total_cells = ch_res[0]*ch_res[1]*ch_res[2];
817                 if (sds->flags & MOD_SMOKE_HIGHRES) {
818                         smoke_reallocate_highres_fluid(sds, ch_dx, ch_res, 1);
819                 }
820         }
821         
822         if (sds->fluid) {
823                 size_t res = sds->res[0]*sds->res[1]*sds->res[2];
824                 float dt, dx, *dens, *react, *fuel, *flame, *heat, *heatold, *vx, *vy, *vz, *r, *g, *b;
825                 unsigned char *obstacles;
826                 unsigned int out_len = (unsigned int)res * sizeof(float);
827                 
828                 smoke_export(sds->fluid, &dt, &dx, &dens, &react, &flame, &fuel, &heat, &heatold, &vx, &vy, &vz, &r, &g, &b, &obstacles);
829
830                 ptcache_file_compressed_read(pf, (unsigned char *)sds->shadow, out_len);
831                 ptcache_file_compressed_read(pf, (unsigned char *)dens, out_len);
832                 if (cache_fields & SM_ACTIVE_HEAT) {
833                         ptcache_file_compressed_read(pf, (unsigned char *)heat, out_len);
834                         ptcache_file_compressed_read(pf, (unsigned char *)heatold, out_len);
835                 }
836                 if (cache_fields & SM_ACTIVE_FIRE) {
837                         ptcache_file_compressed_read(pf, (unsigned char *)flame, out_len);
838                         ptcache_file_compressed_read(pf, (unsigned char *)fuel, out_len);
839                         ptcache_file_compressed_read(pf, (unsigned char *)react, out_len);
840                 }
841                 if (cache_fields & SM_ACTIVE_COLORS) {
842                         ptcache_file_compressed_read(pf, (unsigned char *)r, out_len);
843                         ptcache_file_compressed_read(pf, (unsigned char *)g, out_len);
844                         ptcache_file_compressed_read(pf, (unsigned char *)b, out_len);
845                 }
846                 ptcache_file_compressed_read(pf, (unsigned char *)vx, out_len);
847                 ptcache_file_compressed_read(pf, (unsigned char *)vy, out_len);
848                 ptcache_file_compressed_read(pf, (unsigned char *)vz, out_len);
849                 ptcache_file_compressed_read(pf, (unsigned char *)obstacles, (unsigned int)res);
850                 ptcache_file_read(pf, &dt, 1, sizeof(float));
851                 ptcache_file_read(pf, &dx, 1, sizeof(float));
852                 ptcache_file_read(pf, &sds->p0, 3, sizeof(float));
853                 ptcache_file_read(pf, &sds->p1, 3, sizeof(float));
854                 ptcache_file_read(pf, &sds->dp0, 3, sizeof(float));
855                 ptcache_file_read(pf, &sds->shift, 3, sizeof(int));
856                 ptcache_file_read(pf, &sds->obj_shift_f, 3, sizeof(float));
857                 ptcache_file_read(pf, &sds->obmat, 16, sizeof(float));
858                 ptcache_file_read(pf, &sds->base_res, 3, sizeof(int));
859                 ptcache_file_read(pf, &sds->res_min, 3, sizeof(int));
860                 ptcache_file_read(pf, &sds->res_max, 3, sizeof(int));
861                 ptcache_file_read(pf, &sds->active_color, 3, sizeof(float));
862         }
863
864         if (pf->data_types & (1<<BPHYS_DATA_SMOKE_HIGH) && sds->wt) {
865                         int res = sds->res[0]*sds->res[1]*sds->res[2];
866                         int res_big, res_big_array[3];
867                         float *dens, *react, *fuel, *flame, *tcu, *tcv, *tcw, *r, *g, *b;
868                         unsigned int out_len = sizeof(float)*(unsigned int)res;
869                         unsigned int out_len_big;
870
871                         smoke_turbulence_get_res(sds->wt, res_big_array);
872                         res_big = res_big_array[0]*res_big_array[1]*res_big_array[2];
873                         out_len_big = sizeof(float) * (unsigned int)res_big;
874
875                         smoke_turbulence_export(sds->wt, &dens, &react, &flame, &fuel, &r, &g, &b, &tcu, &tcv, &tcw);
876
877                         ptcache_file_compressed_read(pf, (unsigned char *)dens, out_len_big);
878                         if (cache_fields & SM_ACTIVE_FIRE) {
879                                 ptcache_file_compressed_read(pf, (unsigned char *)flame, out_len_big);
880                                 ptcache_file_compressed_read(pf, (unsigned char *)fuel, out_len_big);
881                                 ptcache_file_compressed_read(pf, (unsigned char *)react, out_len_big);
882                         }
883                         if (cache_fields & SM_ACTIVE_COLORS) {
884                                 ptcache_file_compressed_read(pf, (unsigned char *)r, out_len_big);
885                                 ptcache_file_compressed_read(pf, (unsigned char *)g, out_len_big);
886                                 ptcache_file_compressed_read(pf, (unsigned char *)b, out_len_big);
887                         }
888
889                         ptcache_file_compressed_read(pf, (unsigned char *)tcu, out_len);
890                         ptcache_file_compressed_read(pf, (unsigned char *)tcv, out_len);
891                         ptcache_file_compressed_read(pf, (unsigned char *)tcw, out_len);
892                 }
893
894         return 1;
895 }
896
897 #ifdef WITH_OPENVDB
898 /**
899  * Construct matrices which represent the fluid object, for low and high res:
900  * <pre>
901  * vs 0  0  0
902  * 0  vs 0  0
903  * 0  0  vs 0
904  * px py pz 1
905  * </pre>
906  *
907  * with `vs` = voxel size, and `px, py, pz`,
908  * the min position of the domain's bounding box.
909  */
910 static void compute_fluid_matrices(SmokeDomainSettings *sds)
911 {
912         float bbox_min[3];
913
914         copy_v3_v3(bbox_min, sds->p0);
915
916         if (sds->flags & MOD_SMOKE_ADAPTIVE_DOMAIN) {
917                 bbox_min[0] += (sds->cell_size[0] * (float)sds->res_min[0]);
918                 bbox_min[1] += (sds->cell_size[1] * (float)sds->res_min[1]);
919                 bbox_min[2] += (sds->cell_size[2] * (float)sds->res_min[2]);
920                 add_v3_v3(bbox_min, sds->obj_shift_f);
921         }
922
923         /* construct low res matrix */
924         size_to_mat4(sds->fluidmat, sds->cell_size);
925         copy_v3_v3(sds->fluidmat[3], bbox_min);
926
927         /* The smoke simulator stores voxels cell-centered, whilst VDB is node
928          * centered, so we offset the matrix by half a voxel to compensate. */
929         madd_v3_v3fl(sds->fluidmat[3], sds->cell_size, 0.5f);
930
931         mul_m4_m4m4(sds->fluidmat, sds->obmat, sds->fluidmat);
932
933         if (sds->wt) {
934                 float voxel_size_high[3];
935                 /* construct high res matrix */
936                 mul_v3_v3fl(voxel_size_high, sds->cell_size, 1.0f / (float)(sds->amplify + 1));
937                 size_to_mat4(sds->fluidmat_wt, voxel_size_high);
938                 copy_v3_v3(sds->fluidmat_wt[3], bbox_min);
939
940                 /* Same here, add half a voxel to adjust the position of the fluid. */
941                 madd_v3_v3fl(sds->fluidmat_wt[3], voxel_size_high, 0.5f);
942
943                 mul_m4_m4m4(sds->fluidmat_wt, sds->obmat, sds->fluidmat_wt);
944         }
945 }
946
947 static int ptcache_smoke_openvdb_write(struct OpenVDBWriter *writer, void *smoke_v)
948 {
949         SmokeModifierData *smd = (SmokeModifierData *)smoke_v;
950         SmokeDomainSettings *sds = smd->domain;
951
952         OpenVDBWriter_set_flags(writer, sds->openvdb_comp, (sds->data_depth == 16));
953
954         OpenVDBWriter_add_meta_int(writer, "blender/smoke/active_fields", sds->active_fields);
955         OpenVDBWriter_add_meta_v3_int(writer, "blender/smoke/resolution", sds->res);
956         OpenVDBWriter_add_meta_v3_int(writer, "blender/smoke/min_resolution", sds->res_min);
957         OpenVDBWriter_add_meta_v3_int(writer, "blender/smoke/max_resolution", sds->res_max);
958         OpenVDBWriter_add_meta_v3_int(writer, "blender/smoke/base_resolution", sds->base_res);
959         OpenVDBWriter_add_meta_v3(writer, "blender/smoke/min_bbox", sds->p0);
960         OpenVDBWriter_add_meta_v3(writer, "blender/smoke/max_bbox", sds->p1);
961         OpenVDBWriter_add_meta_v3(writer, "blender/smoke/dp0", sds->dp0);
962         OpenVDBWriter_add_meta_v3_int(writer, "blender/smoke/shift", sds->shift);
963         OpenVDBWriter_add_meta_v3(writer, "blender/smoke/obj_shift_f", sds->obj_shift_f);
964         OpenVDBWriter_add_meta_v3(writer, "blender/smoke/active_color", sds->active_color);
965         OpenVDBWriter_add_meta_mat4(writer, "blender/smoke/obmat", sds->obmat);
966
967         int fluid_fields = smoke_get_data_flags(sds);
968
969         struct OpenVDBFloatGrid *clip_grid = NULL;
970
971         compute_fluid_matrices(sds);
972
973         OpenVDBWriter_add_meta_int(writer, "blender/smoke/fluid_fields", fluid_fields);
974
975         if (sds->wt) {
976                 struct OpenVDBFloatGrid *wt_density_grid;
977                 float *dens, *react, *fuel, *flame, *tcu, *tcv, *tcw, *r, *g, *b;
978
979                 smoke_turbulence_export(sds->wt, &dens, &react, &flame, &fuel, &r, &g, &b, &tcu, &tcv, &tcw);
980
981                 wt_density_grid = OpenVDB_export_grid_fl(writer, "density", dens, sds->res_wt, sds->fluidmat_wt, sds->clipping, NULL);
982                 clip_grid = wt_density_grid;
983
984                 if (fluid_fields & SM_ACTIVE_FIRE) {
985                         OpenVDB_export_grid_fl(writer, "flame", flame, sds->res_wt, sds->fluidmat_wt, sds->clipping, wt_density_grid);
986                         OpenVDB_export_grid_fl(writer, "fuel", fuel, sds->res_wt, sds->fluidmat_wt, sds->clipping, wt_density_grid);
987                         OpenVDB_export_grid_fl(writer, "react", react, sds->res_wt, sds->fluidmat_wt, sds->clipping, wt_density_grid);
988                 }
989
990                 if (fluid_fields & SM_ACTIVE_COLORS) {
991                         OpenVDB_export_grid_vec(writer, "color", r, g, b, sds->res_wt, sds->fluidmat_wt, VEC_INVARIANT, true, sds->clipping, wt_density_grid);
992                 }
993
994                 OpenVDB_export_grid_vec(writer, "texture coordinates", tcu, tcv, tcw, sds->res, sds->fluidmat, VEC_INVARIANT, false, sds->clipping, wt_density_grid);
995         }
996
997         if (sds->fluid) {
998                 struct OpenVDBFloatGrid *density_grid;
999                 float dt, dx, *dens, *react, *fuel, *flame, *heat, *heatold, *vx, *vy, *vz, *r, *g, *b;
1000                 unsigned char *obstacles;
1001
1002                 smoke_export(sds->fluid, &dt, &dx, &dens, &react, &flame, &fuel, &heat,
1003                              &heatold, &vx, &vy, &vz, &r, &g, &b, &obstacles);
1004
1005                 OpenVDBWriter_add_meta_fl(writer, "blender/smoke/dx", dx);
1006                 OpenVDBWriter_add_meta_fl(writer, "blender/smoke/dt", dt);
1007
1008                 const char *name = (!sds->wt) ? "density" : "density_low";
1009                 density_grid = OpenVDB_export_grid_fl(writer, name, dens, sds->res, sds->fluidmat, sds->clipping, NULL);
1010                 clip_grid = sds->wt ? clip_grid : density_grid;
1011
1012                 OpenVDB_export_grid_fl(writer, "shadow", sds->shadow, sds->res, sds->fluidmat, sds->clipping, NULL);
1013
1014                 if (fluid_fields & SM_ACTIVE_HEAT) {
1015                         OpenVDB_export_grid_fl(writer, "heat", heat, sds->res, sds->fluidmat, sds->clipping, clip_grid);
1016                         OpenVDB_export_grid_fl(writer, "heat_old", heatold, sds->res, sds->fluidmat, sds->clipping, clip_grid);
1017                 }
1018
1019                 if (fluid_fields & SM_ACTIVE_FIRE) {
1020                         name = (!sds->wt) ? "flame" : "flame_low";
1021                         OpenVDB_export_grid_fl(writer, name, flame, sds->res, sds->fluidmat, sds->clipping, density_grid);
1022                         name = (!sds->wt) ? "fuel" : "fuel_low";
1023                         OpenVDB_export_grid_fl(writer, name, fuel, sds->res, sds->fluidmat, sds->clipping, density_grid);
1024                         name = (!sds->wt) ? "react" : "react_low";
1025                         OpenVDB_export_grid_fl(writer, name, react, sds->res, sds->fluidmat, sds->clipping, density_grid);
1026                 }
1027
1028                 if (fluid_fields & SM_ACTIVE_COLORS) {
1029                         name = (!sds->wt) ? "color" : "color_low";
1030                         OpenVDB_export_grid_vec(writer, name, r, g, b, sds->res, sds->fluidmat, VEC_INVARIANT, true, sds->clipping, density_grid);
1031                 }
1032
1033                 OpenVDB_export_grid_vec(writer, "velocity", vx, vy, vz, sds->res, sds->fluidmat, VEC_CONTRAVARIANT_RELATIVE, false, sds->clipping, clip_grid);
1034                 OpenVDB_export_grid_ch(writer, "obstacles", obstacles, sds->res, sds->fluidmat, sds->clipping, NULL);
1035         }
1036
1037         return 1;
1038 }
1039
1040 static int ptcache_smoke_openvdb_read(struct OpenVDBReader *reader, void *smoke_v)
1041 {
1042         SmokeModifierData *smd = (SmokeModifierData *)smoke_v;
1043
1044         if (!smd) {
1045                 return 0;
1046         }
1047
1048         SmokeDomainSettings *sds = smd->domain;
1049
1050         int fluid_fields = smoke_get_data_flags(sds);
1051         int active_fields, cache_fields = 0;
1052         int cache_res[3];
1053         float cache_dx;
1054         bool reallocate = false;
1055
1056         OpenVDBReader_get_meta_v3_int(reader, "blender/smoke/min_resolution", sds->res_min);
1057         OpenVDBReader_get_meta_v3_int(reader, "blender/smoke/max_resolution", sds->res_max);
1058         OpenVDBReader_get_meta_v3_int(reader, "blender/smoke/base_resolution", sds->base_res);
1059         OpenVDBReader_get_meta_v3(reader, "blender/smoke/min_bbox", sds->p0);
1060         OpenVDBReader_get_meta_v3(reader, "blender/smoke/max_bbox", sds->p1);
1061         OpenVDBReader_get_meta_v3(reader, "blender/smoke/dp0", sds->dp0);
1062         OpenVDBReader_get_meta_v3_int(reader, "blender/smoke/shift", sds->shift);
1063         OpenVDBReader_get_meta_v3(reader, "blender/smoke/obj_shift_f", sds->obj_shift_f);
1064         OpenVDBReader_get_meta_v3(reader, "blender/smoke/active_color", sds->active_color);
1065         OpenVDBReader_get_meta_mat4(reader, "blender/smoke/obmat", sds->obmat);
1066         OpenVDBReader_get_meta_int(reader, "blender/smoke/fluid_fields", &cache_fields);
1067         OpenVDBReader_get_meta_int(reader, "blender/smoke/active_fields", &active_fields);
1068         OpenVDBReader_get_meta_fl(reader, "blender/smoke/dx", &cache_dx);
1069         OpenVDBReader_get_meta_v3_int(reader, "blender/smoke/resolution", cache_res);
1070
1071         /* check if resolution has changed */
1072         if (sds->res[0] != cache_res[0] ||
1073                 sds->res[1] != cache_res[1] ||
1074                 sds->res[2] != cache_res[2])
1075         {
1076                 if (sds->flags & MOD_SMOKE_ADAPTIVE_DOMAIN) {
1077                         reallocate = true;
1078                 }
1079                 else {
1080                         return 0;
1081                 }
1082         }
1083
1084         /* check if active fields have changed */
1085         if ((fluid_fields != cache_fields) || (active_fields != sds->active_fields)) {
1086                 reallocate = true;
1087         }
1088
1089         /* reallocate fluid if needed*/
1090         if (reallocate) {
1091                 sds->active_fields = active_fields | cache_fields;
1092                 smoke_reallocate_fluid(sds, cache_dx, cache_res, 1);
1093                 sds->dx = cache_dx;
1094                 copy_v3_v3_int(sds->res, cache_res);
1095                 sds->total_cells = cache_res[0] * cache_res[1] * cache_res[2];
1096
1097                 if (sds->flags & MOD_SMOKE_HIGHRES) {
1098                         smoke_reallocate_highres_fluid(sds, cache_dx, cache_res, 1);
1099                 }
1100         }
1101
1102         if (sds->fluid) {
1103                 float dt, dx, *dens, *react, *fuel, *flame, *heat, *heatold, *vx, *vy, *vz, *r, *g, *b;
1104                 unsigned char *obstacles;
1105
1106                 smoke_export(sds->fluid, &dt, &dx, &dens, &react, &flame, &fuel, &heat,
1107                              &heatold, &vx, &vy, &vz, &r, &g, &b, &obstacles);
1108
1109                 OpenVDBReader_get_meta_fl(reader, "blender/smoke/dt", &dt);
1110
1111                 OpenVDB_import_grid_fl(reader, "shadow", &sds->shadow, sds->res);
1112
1113                 const char *name = (!sds->wt) ? "density" : "density_low";
1114                 OpenVDB_import_grid_fl(reader, name, &dens, sds->res);
1115
1116                 if (cache_fields & SM_ACTIVE_HEAT) {
1117                         OpenVDB_import_grid_fl(reader, "heat", &heat, sds->res);
1118                         OpenVDB_import_grid_fl(reader, "heat_old", &heatold, sds->res);
1119                 }
1120
1121                 if (cache_fields & SM_ACTIVE_FIRE) {
1122                         name = (!sds->wt) ? "flame" : "flame_low";
1123                         OpenVDB_import_grid_fl(reader, name, &flame, sds->res);
1124                         name = (!sds->wt) ? "fuel" : "fuel_low";
1125                         OpenVDB_import_grid_fl(reader, name, &fuel, sds->res);
1126                         name = (!sds->wt) ? "react" : "react_low";
1127                         OpenVDB_import_grid_fl(reader, name, &react, sds->res);
1128                 }
1129
1130                 if (cache_fields & SM_ACTIVE_COLORS) {
1131                         name = (!sds->wt) ? "color" : "color_low";
1132                         OpenVDB_import_grid_vec(reader, name, &r, &g, &b, sds->res);
1133                 }
1134
1135                 OpenVDB_import_grid_vec(reader, "velocity", &vx, &vy, &vz, sds->res);
1136                 OpenVDB_import_grid_ch(reader, "obstacles", &obstacles, sds->res);
1137         }
1138
1139         if (sds->wt) {
1140                 float *dens, *react, *fuel, *flame, *tcu, *tcv, *tcw, *r, *g, *b;
1141
1142                 smoke_turbulence_export(sds->wt, &dens, &react, &flame, &fuel, &r, &g, &b, &tcu, &tcv, &tcw);
1143
1144                 OpenVDB_import_grid_fl(reader, "density", &dens, sds->res_wt);
1145
1146                 if (cache_fields & SM_ACTIVE_FIRE) {
1147                         OpenVDB_import_grid_fl(reader, "flame", &flame, sds->res_wt);
1148                         OpenVDB_import_grid_fl(reader, "fuel", &fuel, sds->res_wt);
1149                         OpenVDB_import_grid_fl(reader, "react", &react, sds->res_wt);
1150                 }
1151
1152                 if (cache_fields & SM_ACTIVE_COLORS) {
1153                         OpenVDB_import_grid_vec(reader, "color", &r, &g, &b, sds->res_wt);
1154                 }
1155
1156                 OpenVDB_import_grid_vec(reader, "texture coordinates", &tcu, &tcv, &tcw, sds->res);
1157         }
1158
1159         OpenVDBReader_free(reader);
1160
1161         return 1;
1162 }
1163 #endif
1164
1165 #else // WITH_SMOKE
1166 static int  ptcache_smoke_totpoint(void *UNUSED(smoke_v), int UNUSED(cfra)) { return 0; }
1167 static void ptcache_smoke_error(void *UNUSED(smoke_v), const char *UNUSED(message)) { }
1168 static int  ptcache_smoke_read(PTCacheFile *UNUSED(pf), void *UNUSED(smoke_v)) { return 0; }
1169 static int  ptcache_smoke_write(PTCacheFile *UNUSED(pf), void *UNUSED(smoke_v)) { return 0; }
1170 #endif // WITH_SMOKE
1171
1172 #if !defined(WITH_SMOKE) || !defined(WITH_OPENVDB)
1173 static int ptcache_smoke_openvdb_write(struct OpenVDBWriter *writer, void *smoke_v)
1174 {
1175         UNUSED_VARS(writer, smoke_v);
1176         return 0;
1177 }
1178
1179 static int ptcache_smoke_openvdb_read(struct OpenVDBReader *reader, void *smoke_v)
1180 {
1181         UNUSED_VARS(reader, smoke_v);
1182         return 0;
1183 }
1184 #endif
1185
1186 static int ptcache_dynamicpaint_totpoint(void *sd, int UNUSED(cfra))
1187 {
1188         DynamicPaintSurface *surface = (DynamicPaintSurface*)sd;
1189
1190         if (!surface->data) return 0;
1191         else return surface->data->total_points;
1192 }
1193
1194 static void ptcache_dynamicpaint_error(void *UNUSED(sd), const char *UNUSED(message))
1195 {
1196         /* ignored for now */
1197 }
1198
1199 #define DPAINT_CACHE_VERSION "1.01"
1200
1201 static int  ptcache_dynamicpaint_write(PTCacheFile *pf, void *dp_v)
1202 {       
1203         DynamicPaintSurface *surface = (DynamicPaintSurface*)dp_v;
1204         int cache_compress = 1;
1205
1206         /* version header */
1207         ptcache_file_write(pf, DPAINT_CACHE_VERSION, 1, sizeof(char) * 4);
1208
1209         if (surface->format != MOD_DPAINT_SURFACE_F_IMAGESEQ && surface->data) {
1210                 int total_points=surface->data->total_points;
1211                 unsigned int in_len;
1212                 unsigned char *out;
1213
1214                 /* cache type */
1215                 ptcache_file_write(pf, &surface->type, 1, sizeof(int));
1216
1217                 if (surface->type == MOD_DPAINT_SURFACE_T_PAINT) {
1218                         in_len = sizeof(PaintPoint) * total_points;
1219                 }
1220                 else if (surface->type == MOD_DPAINT_SURFACE_T_DISPLACE ||
1221                          surface->type == MOD_DPAINT_SURFACE_T_WEIGHT)
1222                 {
1223                         in_len = sizeof(float) * total_points;
1224                 }
1225                 else if (surface->type == MOD_DPAINT_SURFACE_T_WAVE) {
1226                         in_len = sizeof(PaintWavePoint) * total_points;
1227                 }
1228                 else {
1229                         return 0;
1230                 }
1231
1232                 out = (unsigned char *)MEM_callocN(LZO_OUT_LEN(in_len), "pointcache_lzo_buffer");
1233
1234                 ptcache_file_compressed_write(pf, (unsigned char *)surface->data->type_data, in_len, out, cache_compress);
1235                 MEM_freeN(out);
1236
1237         }
1238         return 1;
1239 }
1240 static int ptcache_dynamicpaint_read(PTCacheFile *pf, void *dp_v)
1241 {
1242         DynamicPaintSurface *surface = (DynamicPaintSurface*)dp_v;
1243         char version[4];
1244         
1245         /* version header */
1246         ptcache_file_read(pf, version, 1, sizeof(char) * 4);
1247         if (!STREQLEN(version, DPAINT_CACHE_VERSION, 4)) {
1248                 printf("Dynamic Paint: Invalid cache version: '%c%c%c%c'!\n", UNPACK4(version));
1249                 return 0;
1250         }
1251
1252         if (surface->format != MOD_DPAINT_SURFACE_F_IMAGESEQ && surface->data) {
1253                 unsigned int data_len;
1254                 int surface_type;
1255
1256                 /* cache type */
1257                 ptcache_file_read(pf, &surface_type, 1, sizeof(int));
1258
1259                 if (surface_type != surface->type)
1260                         return 0;
1261
1262                 /* read surface data */
1263                 if (surface->type == MOD_DPAINT_SURFACE_T_PAINT) {
1264                         data_len = sizeof(PaintPoint);
1265                 }
1266                 else if (surface->type == MOD_DPAINT_SURFACE_T_DISPLACE ||
1267                          surface->type == MOD_DPAINT_SURFACE_T_WEIGHT)
1268                 {
1269                         data_len = sizeof(float);
1270                 }
1271                 else if (surface->type == MOD_DPAINT_SURFACE_T_WAVE) {
1272                         data_len = sizeof(PaintWavePoint);
1273                 }
1274                 else {
1275                         return 0;
1276                 }
1277
1278                 ptcache_file_compressed_read(pf, (unsigned char *)surface->data->type_data, data_len*surface->data->total_points);
1279
1280         }
1281         return 1;
1282 }
1283
1284 /* Rigid Body functions */
1285 static int  ptcache_rigidbody_write(int index, void *rb_v, void **data, int UNUSED(cfra))
1286 {
1287         RigidBodyWorld *rbw = rb_v;
1288         Object *ob = NULL;
1289         
1290         if (rbw->objects)
1291                 ob = rbw->objects[index];
1292         
1293         if (ob && ob->rigidbody_object) {
1294                 RigidBodyOb *rbo = ob->rigidbody_object;
1295                 
1296                 if (rbo->type == RBO_TYPE_ACTIVE) {
1297 #ifdef WITH_BULLET
1298                         RB_body_get_position(rbo->physics_object, rbo->pos);
1299                         RB_body_get_orientation(rbo->physics_object, rbo->orn);
1300 #endif
1301                         PTCACHE_DATA_FROM(data, BPHYS_DATA_LOCATION, rbo->pos);
1302                         PTCACHE_DATA_FROM(data, BPHYS_DATA_ROTATION, rbo->orn);
1303                 }
1304         }
1305
1306         return 1;
1307 }
1308 static void ptcache_rigidbody_read(int index, void *rb_v, void **data, float UNUSED(cfra), float *old_data)
1309 {
1310         RigidBodyWorld *rbw = rb_v;
1311         Object *ob = NULL;
1312         
1313         if (rbw->objects)
1314                 ob = rbw->objects[index];
1315         
1316         if (ob && ob->rigidbody_object) {
1317                 RigidBodyOb *rbo = ob->rigidbody_object;
1318                 
1319                 if (rbo->type == RBO_TYPE_ACTIVE) {
1320                         
1321                         if (old_data) {
1322                                 memcpy(rbo->pos, data, 3 * sizeof(float));
1323                                 memcpy(rbo->orn, data + 3, 4 * sizeof(float));
1324                         }
1325                         else {
1326                                 PTCACHE_DATA_TO(data, BPHYS_DATA_LOCATION, 0, rbo->pos);
1327                                 PTCACHE_DATA_TO(data, BPHYS_DATA_ROTATION, 0, rbo->orn);
1328                         }
1329                 }
1330         }
1331 }
1332 static void ptcache_rigidbody_interpolate(int index, void *rb_v, void **data, float cfra, float cfra1, float cfra2, float *old_data)
1333 {
1334         RigidBodyWorld *rbw = rb_v;
1335         Object *ob = NULL;
1336         
1337         if (rbw->objects)
1338                 ob = rbw->objects[index];
1339         
1340         if (ob && ob->rigidbody_object) {
1341                 RigidBodyOb *rbo = ob->rigidbody_object;
1342                 
1343                 if (rbo->type == RBO_TYPE_ACTIVE) {
1344                         ParticleKey keys[4];
1345                         ParticleKey result;
1346                         float dfra;
1347                         
1348                         memset(keys, 0, sizeof(keys));
1349                         
1350                         copy_v3_v3(keys[1].co, rbo->pos);
1351                         copy_qt_qt(keys[1].rot, rbo->orn);
1352                         
1353                         if (old_data) {
1354                                 memcpy(keys[2].co, data, 3 * sizeof(float));
1355                                 memcpy(keys[2].rot, data + 3, 4 * sizeof(float));
1356                         }
1357                         else {
1358                                 BKE_ptcache_make_particle_key(&keys[2], 0, data, cfra2);
1359                         }
1360                         
1361                         dfra = cfra2 - cfra1;
1362                 
1363                         /* note: keys[0] and keys[3] unused for type < 1 (crappy) */
1364                         psys_interpolate_particle(-1, keys, (cfra - cfra1) / dfra, &result, true);
1365                         interp_qt_qtqt(result.rot, keys[1].rot, keys[2].rot, (cfra - cfra1) / dfra);
1366                         
1367                         copy_v3_v3(rbo->pos, result.co);
1368                         copy_qt_qt(rbo->orn, result.rot);
1369                 }
1370         }
1371 }
1372 static int ptcache_rigidbody_totpoint(void *rb_v, int UNUSED(cfra))
1373 {
1374         RigidBodyWorld *rbw = rb_v;
1375         
1376         return rbw->numbodies;
1377 }
1378
1379 static void ptcache_rigidbody_error(void *UNUSED(rb_v), const char *UNUSED(message))
1380 {
1381         /* ignored for now */
1382 }
1383
1384 /* Creating ID's */
1385 void BKE_ptcache_id_from_softbody(PTCacheID *pid, Object *ob, SoftBody *sb)
1386 {
1387         memset(pid, 0, sizeof(PTCacheID));
1388
1389         pid->ob= ob;
1390         pid->calldata= sb;
1391         pid->type= PTCACHE_TYPE_SOFTBODY;
1392         pid->cache= sb->pointcache;
1393         pid->cache_ptr= &sb->pointcache;
1394         pid->ptcaches= &sb->ptcaches;
1395         pid->totpoint= pid->totwrite= ptcache_softbody_totpoint;
1396         pid->error                                      = ptcache_softbody_error;
1397
1398         pid->write_point                        = ptcache_softbody_write;
1399         pid->read_point                         = ptcache_softbody_read;
1400         pid->interpolate_point          = ptcache_softbody_interpolate;
1401
1402         pid->write_stream                       = NULL;
1403         pid->read_stream                        = NULL;
1404
1405         pid->write_openvdb_stream       = NULL;
1406         pid->read_openvdb_stream        = NULL;
1407
1408         pid->write_extra_data           = NULL;
1409         pid->read_extra_data            = NULL;
1410         pid->interpolate_extra_data     = NULL;
1411
1412         pid->write_header                       = ptcache_basic_header_write;
1413         pid->read_header                        = ptcache_basic_header_read;
1414
1415         pid->data_types= (1<<BPHYS_DATA_LOCATION) | (1<<BPHYS_DATA_VELOCITY);
1416         pid->info_types= 0;
1417
1418         pid->stack_index = pid->cache->index;
1419
1420         pid->default_step = 10;
1421         pid->max_step = 20;
1422         pid->file_type = PTCACHE_FILE_PTCACHE;
1423 }
1424 void BKE_ptcache_id_from_particles(PTCacheID *pid, Object *ob, ParticleSystem *psys)
1425 {
1426         memset(pid, 0, sizeof(PTCacheID));
1427
1428         pid->ob= ob;
1429         pid->calldata= psys;
1430         pid->type= PTCACHE_TYPE_PARTICLES;
1431         pid->stack_index= psys->pointcache->index;
1432         pid->cache= psys->pointcache;
1433         pid->cache_ptr= &psys->pointcache;
1434         pid->ptcaches= &psys->ptcaches;
1435
1436         if (psys->part->type != PART_HAIR)
1437                 pid->flag |= PTCACHE_VEL_PER_SEC;
1438
1439         pid->totpoint                           = ptcache_particle_totpoint;
1440         pid->totwrite                           = ptcache_particle_totwrite;
1441         pid->error                                      = ptcache_particle_error;
1442
1443         pid->write_point                                = ptcache_particle_write;
1444         pid->read_point                         = ptcache_particle_read;
1445         pid->interpolate_point          = ptcache_particle_interpolate;
1446
1447         pid->write_stream                       = NULL;
1448         pid->read_stream                        = NULL;
1449
1450         pid->write_openvdb_stream       = NULL;
1451         pid->read_openvdb_stream        = NULL;
1452
1453         pid->write_extra_data           = NULL;
1454         pid->read_extra_data            = NULL;
1455         pid->interpolate_extra_data     = NULL;
1456
1457         pid->write_header                       = ptcache_basic_header_write;
1458         pid->read_header                        = ptcache_basic_header_read;
1459
1460         pid->data_types = (1<<BPHYS_DATA_LOCATION) | (1<<BPHYS_DATA_VELOCITY) | (1<<BPHYS_DATA_INDEX);
1461
1462         if (psys->part->phystype == PART_PHYS_BOIDS)
1463                 pid->data_types|= (1<<BPHYS_DATA_AVELOCITY) | (1<<BPHYS_DATA_ROTATION) | (1<<BPHYS_DATA_BOIDS);
1464         else if (psys->part->phystype == PART_PHYS_FLUID && psys->part->fluid && psys->part->fluid->flag & SPH_VISCOELASTIC_SPRINGS) {
1465                 pid->write_extra_data = ptcache_particle_extra_write;
1466                 pid->read_extra_data = ptcache_particle_extra_read;
1467         }
1468
1469         if (psys->part->flag & PART_ROTATIONS) {
1470                 pid->data_types|= (1<<BPHYS_DATA_ROTATION);
1471
1472                 if (psys->part->rotmode != PART_ROT_VEL  ||
1473                     psys->part->avemode == PART_AVE_RAND ||
1474                     psys->part->avefac != 0.0f)
1475                 {
1476                         pid->data_types |= (1 << BPHYS_DATA_AVELOCITY);
1477                 }
1478         }
1479
1480         pid->info_types= (1<<BPHYS_DATA_TIMES);
1481
1482         pid->default_step = 10;
1483         pid->max_step = 20;
1484         pid->file_type = PTCACHE_FILE_PTCACHE;
1485 }
1486 void BKE_ptcache_id_from_cloth(PTCacheID *pid, Object *ob, ClothModifierData *clmd)
1487 {
1488         memset(pid, 0, sizeof(PTCacheID));
1489
1490         pid->ob= ob;
1491         pid->calldata= clmd;
1492         pid->type= PTCACHE_TYPE_CLOTH;
1493         pid->stack_index= clmd->point_cache->index;
1494         pid->cache= clmd->point_cache;
1495         pid->cache_ptr= &clmd->point_cache;
1496         pid->ptcaches= &clmd->ptcaches;
1497         pid->totpoint= pid->totwrite= ptcache_cloth_totpoint;
1498         pid->error                                      = ptcache_cloth_error;
1499
1500         pid->write_point                        = ptcache_cloth_write;
1501         pid->read_point                         = ptcache_cloth_read;
1502         pid->interpolate_point          = ptcache_cloth_interpolate;
1503
1504         pid->write_openvdb_stream       = NULL;
1505         pid->read_openvdb_stream        = NULL;
1506
1507         pid->write_stream                       = NULL;
1508         pid->read_stream                        = NULL;
1509
1510         pid->write_extra_data           = NULL;
1511         pid->read_extra_data            = NULL;
1512         pid->interpolate_extra_data     = NULL;
1513
1514         pid->write_header                       = ptcache_basic_header_write;
1515         pid->read_header                        = ptcache_basic_header_read;
1516
1517         pid->data_types= (1<<BPHYS_DATA_LOCATION) | (1<<BPHYS_DATA_VELOCITY) | (1<<BPHYS_DATA_XCONST);
1518         pid->info_types= 0;
1519
1520         pid->default_step = 1;
1521         pid->max_step = 1;
1522         pid->file_type = PTCACHE_FILE_PTCACHE;
1523 }
1524 void BKE_ptcache_id_from_smoke(PTCacheID *pid, struct Object *ob, struct SmokeModifierData *smd)
1525 {
1526         SmokeDomainSettings *sds = smd->domain;
1527
1528         memset(pid, 0, sizeof(PTCacheID));
1529
1530         pid->ob= ob;
1531         pid->calldata= smd;
1532         
1533         pid->type= PTCACHE_TYPE_SMOKE_DOMAIN;
1534         pid->stack_index= sds->point_cache[0]->index;
1535
1536         pid->cache= sds->point_cache[0];
1537         pid->cache_ptr= &(sds->point_cache[0]);
1538         pid->ptcaches= &(sds->ptcaches[0]);
1539
1540         pid->totpoint= pid->totwrite= ptcache_smoke_totpoint;
1541         pid->error                                      = ptcache_smoke_error;
1542
1543         pid->write_point                        = NULL;
1544         pid->read_point                         = NULL;
1545         pid->interpolate_point          = NULL;
1546
1547         pid->read_stream                        = ptcache_smoke_read;
1548         pid->write_stream                       = ptcache_smoke_write;
1549
1550         pid->write_openvdb_stream       = ptcache_smoke_openvdb_write;
1551         pid->read_openvdb_stream        = ptcache_smoke_openvdb_read;
1552
1553         pid->write_extra_data           = NULL;
1554         pid->read_extra_data            = NULL;
1555         pid->interpolate_extra_data     = NULL;
1556
1557         pid->write_header                       = ptcache_basic_header_write;
1558         pid->read_header                        = ptcache_basic_header_read;
1559
1560         pid->data_types= 0;
1561         pid->info_types= 0;
1562
1563         if (sds->fluid)
1564                 pid->data_types |= (1<<BPHYS_DATA_SMOKE_LOW);
1565         if (sds->wt)
1566                 pid->data_types |= (1<<BPHYS_DATA_SMOKE_HIGH);
1567
1568         pid->default_step = 1;
1569         pid->max_step = 1;
1570         pid->file_type = smd->domain->cache_file_format;
1571 }
1572
1573 void BKE_ptcache_id_from_dynamicpaint(PTCacheID *pid, Object *ob, DynamicPaintSurface *surface)
1574 {
1575
1576         memset(pid, 0, sizeof(PTCacheID));
1577
1578         pid->ob= ob;
1579         pid->calldata= surface;
1580         pid->type= PTCACHE_TYPE_DYNAMICPAINT;
1581         pid->cache= surface->pointcache;
1582         pid->cache_ptr= &surface->pointcache;
1583         pid->ptcaches= &surface->ptcaches;
1584         pid->totpoint= pid->totwrite= ptcache_dynamicpaint_totpoint;
1585         pid->error                                      = ptcache_dynamicpaint_error;
1586
1587         pid->write_point                        = NULL;
1588         pid->read_point                         = NULL;
1589         pid->interpolate_point          = NULL;
1590
1591         pid->write_stream                       = ptcache_dynamicpaint_write;
1592         pid->read_stream                        = ptcache_dynamicpaint_read;
1593
1594         pid->write_openvdb_stream       = NULL;
1595         pid->read_openvdb_stream        = NULL;
1596
1597         pid->write_extra_data           = NULL;
1598         pid->read_extra_data            = NULL;
1599         pid->interpolate_extra_data     = NULL;
1600
1601         pid->write_header                       = ptcache_basic_header_write;
1602         pid->read_header                        = ptcache_basic_header_read;
1603
1604         pid->data_types= BPHYS_DATA_DYNAMICPAINT;
1605         pid->info_types= 0;
1606
1607         pid->stack_index = pid->cache->index;
1608
1609         pid->default_step = 1;
1610         pid->max_step = 1;
1611         pid->file_type = PTCACHE_FILE_PTCACHE;
1612 }
1613
1614 void BKE_ptcache_id_from_rigidbody(PTCacheID *pid, Object *ob, RigidBodyWorld *rbw)
1615 {
1616         
1617         memset(pid, 0, sizeof(PTCacheID));
1618         
1619         pid->ob= ob;
1620         pid->calldata= rbw;
1621         pid->type= PTCACHE_TYPE_RIGIDBODY;
1622         pid->cache= rbw->pointcache;
1623         pid->cache_ptr= &rbw->pointcache;
1624         pid->ptcaches= &rbw->ptcaches;
1625         pid->totpoint= pid->totwrite= ptcache_rigidbody_totpoint;
1626         pid->error                                      = ptcache_rigidbody_error;
1627         
1628         pid->write_point                        = ptcache_rigidbody_write;
1629         pid->read_point                         = ptcache_rigidbody_read;
1630         pid->interpolate_point          = ptcache_rigidbody_interpolate;
1631         
1632         pid->write_stream                       = NULL;
1633         pid->read_stream                        = NULL;
1634
1635         pid->write_openvdb_stream       = NULL;
1636         pid->read_openvdb_stream        = NULL;
1637         
1638         pid->write_extra_data           = NULL;
1639         pid->read_extra_data            = NULL;
1640         pid->interpolate_extra_data     = NULL;
1641         
1642         pid->write_header                       = ptcache_basic_header_write;
1643         pid->read_header                        = ptcache_basic_header_read;
1644         
1645         pid->data_types= (1<<BPHYS_DATA_LOCATION) | (1<<BPHYS_DATA_ROTATION);
1646         pid->info_types= 0;
1647         
1648         pid->stack_index = pid->cache->index;
1649         
1650         pid->default_step = 1;
1651         pid->max_step = 1;
1652         pid->file_type = PTCACHE_FILE_PTCACHE;
1653 }
1654
1655 PTCacheID BKE_ptcache_id_find(Object *ob, Scene *scene, PointCache *cache)
1656 {
1657         PTCacheID result = {0};
1658
1659         ListBase pidlist;
1660         BKE_ptcache_ids_from_object(&pidlist, ob, scene, MAX_DUPLI_RECUR);
1661
1662         for (PTCacheID *pid = pidlist.first; pid; pid = pid->next) {
1663                 if (pid->cache == cache) {
1664                         result = *pid;
1665                         break;
1666                 }
1667         }
1668
1669         BLI_freelistN(&pidlist);
1670
1671         return result;
1672 }
1673
1674 void BKE_ptcache_ids_from_object(ListBase *lb, Object *ob, Scene *scene, int duplis)
1675 {
1676         PTCacheID *pid;
1677         ParticleSystem *psys;
1678         ModifierData *md;
1679
1680         lb->first= lb->last= NULL;
1681
1682         if (ob->soft) {
1683                 pid= MEM_callocN(sizeof(PTCacheID), "PTCacheID");
1684                 BKE_ptcache_id_from_softbody(pid, ob, ob->soft);
1685                 BLI_addtail(lb, pid);
1686         }
1687
1688         for (psys=ob->particlesystem.first; psys; psys=psys->next) {
1689                 if (psys->part==NULL)
1690                         continue;
1691                 
1692                 /* check to make sure point cache is actually used by the particles */
1693                 if (ELEM(psys->part->phystype, PART_PHYS_NO, PART_PHYS_KEYED))
1694                         continue;
1695
1696                 /* hair needs to be included in id-list for cache edit mode to work */
1697                 /* if (psys->part->type == PART_HAIR && (psys->flag & PSYS_HAIR_DYNAMICS)==0) */
1698                 /*      continue; */
1699                         
1700                 if (psys->part->type == PART_FLUID)
1701                         continue;
1702
1703                 pid= MEM_callocN(sizeof(PTCacheID), "PTCacheID");
1704                 BKE_ptcache_id_from_particles(pid, ob, psys);
1705                 BLI_addtail(lb, pid);
1706         }
1707
1708         for (md=ob->modifiers.first; md; md=md->next) {
1709                 if (md->type == eModifierType_Cloth) {
1710                         pid= MEM_callocN(sizeof(PTCacheID), "PTCacheID");
1711                         BKE_ptcache_id_from_cloth(pid, ob, (ClothModifierData*)md);
1712                         BLI_addtail(lb, pid);
1713                 }
1714                 else if (md->type == eModifierType_Smoke) {
1715                         SmokeModifierData *smd = (SmokeModifierData *)md;
1716                         if (smd->type & MOD_SMOKE_TYPE_DOMAIN) {
1717                                 pid= MEM_callocN(sizeof(PTCacheID), "PTCacheID");
1718                                 BKE_ptcache_id_from_smoke(pid, ob, (SmokeModifierData*)md);
1719                                 BLI_addtail(lb, pid);
1720                         }
1721                 }
1722                 else if (md->type == eModifierType_DynamicPaint) {
1723                         DynamicPaintModifierData *pmd = (DynamicPaintModifierData *)md;
1724                         if (pmd->canvas) {
1725                                 DynamicPaintSurface *surface = pmd->canvas->surfaces.first;
1726
1727                                 for (; surface; surface=surface->next) {
1728                                         pid= MEM_callocN(sizeof(PTCacheID), "PTCacheID");
1729                                         BKE_ptcache_id_from_dynamicpaint(pid, ob, surface);
1730                                         BLI_addtail(lb, pid);
1731                                 }
1732                         }
1733                 }
1734         }
1735         
1736         if (scene && ob->rigidbody_object && scene->rigidbody_world) {
1737                 pid = MEM_callocN(sizeof(PTCacheID), "PTCacheID");
1738                 BKE_ptcache_id_from_rigidbody(pid, ob, scene->rigidbody_world);
1739                 BLI_addtail(lb, pid);
1740         }
1741
1742         /* Consider all object in dupli groups to be part of the same object,
1743          * for baking with linking dupligroups. Once we have better overrides
1744          * this can be revisited so users select the local objects directly. */
1745         if (scene && (duplis-- > 0) && (ob->dup_group)) {
1746                 Collection *collection = ob->dup_group;
1747                 Base *base = BKE_collection_object_cache_get(collection).first;
1748
1749                 for (; base; base = base->next) {
1750                         if (base->object != ob) {
1751                                 ListBase lb_dupli_pid;
1752                                 BKE_ptcache_ids_from_object(&lb_dupli_pid, base->object, scene, duplis);
1753                                 BLI_movelisttolist(lb, &lb_dupli_pid);
1754                         }
1755                 }
1756         }
1757 }
1758
1759 /* File handling */
1760
1761 static const char *ptcache_file_extension(const PTCacheID *pid)
1762 {
1763         switch (pid->file_type) {
1764                 default:
1765                 case PTCACHE_FILE_PTCACHE:
1766                         return PTCACHE_EXT;
1767                 case PTCACHE_FILE_OPENVDB:
1768                         return ".vdb";
1769         }
1770 }
1771
1772 /**
1773  * Similar to #BLI_path_frame_get, but takes into account the stack-index which is after the frame.
1774  */
1775 static int ptcache_frame_from_filename(const char *filename, const char *ext)
1776 {
1777         const int frame_len = 6;
1778         const int ext_len = frame_len + strlen(ext);
1779         const int len = strlen(filename);
1780
1781         /* could crash if trying to copy a string out of this range */
1782         if (len > ext_len) {
1783                 /* using frame_len here gives compile error (vla) */
1784                 char num[/* frame_len */6 + 1];
1785                 BLI_strncpy(num, filename + len - ext_len, sizeof(num));
1786
1787                 return atoi(num);
1788         }
1789
1790         return -1;
1791 }
1792
1793 /* Takes an Object ID and returns a unique name
1794  * - id: object id
1795  * - cfra: frame for the cache, can be negative
1796  * - stack_index: index in the modifier stack. we can have cache for more than one stack_index
1797  */
1798
1799 #define MAX_PTCACHE_PATH FILE_MAX
1800 #define MAX_PTCACHE_FILE (FILE_MAX * 2)
1801
1802 static int ptcache_path(PTCacheID *pid, char *filename)
1803 {
1804         Library *lib = (pid->ob) ? pid->ob->id.lib : NULL;
1805         const char *blendfilename= (lib && (pid->cache->flag & PTCACHE_IGNORE_LIBPATH)==0) ? lib->filepath: G.main->name;
1806         size_t i;
1807
1808         if (pid->cache->flag & PTCACHE_EXTERNAL) {
1809                 strcpy(filename, pid->cache->path);
1810
1811                 if (BLI_path_is_rel(filename)) {
1812                         BLI_path_abs(filename, blendfilename);
1813                 }
1814
1815                 return BLI_add_slash(filename); /* new strlen() */
1816         }
1817         else if (G.relbase_valid || lib) {
1818                 char file[MAX_PTCACHE_PATH]; /* we don't want the dir, only the file */
1819
1820                 BLI_split_file_part(blendfilename, file, sizeof(file));
1821                 i = strlen(file);
1822                 
1823                 /* remove .blend */
1824                 if (i > 6)
1825                         file[i-6] = '\0';
1826                 
1827                 BLI_snprintf(filename, MAX_PTCACHE_PATH, "//"PTCACHE_PATH"%s", file); /* add blend file name to pointcache dir */
1828                 BLI_path_abs(filename, blendfilename);
1829                 return BLI_add_slash(filename); /* new strlen() */
1830         }
1831         
1832         /* use the temp path. this is weak but better then not using point cache at all */
1833         /* temporary directory is assumed to exist and ALWAYS has a trailing slash */
1834         BLI_snprintf(filename, MAX_PTCACHE_PATH, "%s"PTCACHE_PATH, BKE_tempdir_session());
1835         
1836         return BLI_add_slash(filename); /* new strlen() */
1837 }
1838
1839 static int ptcache_filename(PTCacheID *pid, char *filename, int cfra, short do_path, short do_ext)
1840 {
1841         int len=0;
1842         char *idname;
1843         char *newname;
1844         filename[0] = '\0';
1845         newname = filename;
1846         
1847         if (!G.relbase_valid && (pid->cache->flag & PTCACHE_EXTERNAL)==0) return 0; /* save blend file before using disk pointcache */
1848         
1849         /* start with temp dir */
1850         if (do_path) {
1851                 len = ptcache_path(pid, filename);
1852                 newname += len;
1853         }
1854         if (pid->cache->name[0] == '\0' && (pid->cache->flag & PTCACHE_EXTERNAL)==0) {
1855                 idname = (pid->ob->id.name + 2);
1856                 /* convert chars to hex so they are always a valid filename */
1857                 while ('\0' != *idname) {
1858                         BLI_snprintf(newname, MAX_PTCACHE_FILE, "%02X", (unsigned int)(*idname++));
1859                         newname+=2;
1860                         len += 2;
1861                 }
1862         }
1863         else {
1864                 int temp = (int)strlen(pid->cache->name); 
1865                 strcpy(newname, pid->cache->name); 
1866                 newname+=temp;
1867                 len += temp;
1868         }
1869
1870         if (do_ext) {
1871                 if (pid->cache->index < 0)
1872                         pid->cache->index =  pid->stack_index = BKE_object_insert_ptcache(pid->ob);
1873
1874                 const char *ext = ptcache_file_extension(pid);
1875
1876                 if (pid->cache->flag & PTCACHE_EXTERNAL) {
1877                         if (pid->cache->index >= 0)
1878                                 BLI_snprintf(newname, MAX_PTCACHE_FILE, "_%06d_%02u%s", cfra, pid->stack_index, ext); /* always 6 chars */
1879                         else
1880                                 BLI_snprintf(newname, MAX_PTCACHE_FILE, "_%06d%s", cfra, ext); /* always 6 chars */
1881                 }
1882                 else {
1883                         BLI_snprintf(newname, MAX_PTCACHE_FILE, "_%06d_%02u%s", cfra, pid->stack_index, ext); /* always 6 chars */
1884                 }
1885                 len += 16;
1886         }
1887         
1888         return len; /* make sure the above string is always 16 chars */
1889 }
1890
1891 /* youll need to close yourself after! */
1892 static PTCacheFile *ptcache_file_open(PTCacheID *pid, int mode, int cfra)
1893 {
1894         PTCacheFile *pf;
1895         FILE *fp = NULL;
1896         char filename[FILE_MAX * 2];
1897
1898 #ifndef DURIAN_POINTCACHE_LIB_OK
1899         /* don't allow writing for linked objects */
1900         if (pid->ob->id.lib && mode == PTCACHE_FILE_WRITE)
1901                 return NULL;
1902 #endif
1903         if (!G.relbase_valid && (pid->cache->flag & PTCACHE_EXTERNAL)==0) return NULL; /* save blend file before using disk pointcache */
1904         
1905         ptcache_filename(pid, filename, cfra, 1, 1);
1906
1907         if (mode==PTCACHE_FILE_READ) {
1908                 fp = BLI_fopen(filename, "rb");
1909         }
1910         else if (mode==PTCACHE_FILE_WRITE) {
1911                 BLI_make_existing_file(filename); /* will create the dir if needs be, same as //textures is created */
1912                 fp = BLI_fopen(filename, "wb");
1913         }
1914         else if (mode==PTCACHE_FILE_UPDATE) {
1915                 BLI_make_existing_file(filename);
1916                 fp = BLI_fopen(filename, "rb+");
1917         }
1918
1919         if (!fp)
1920                 return NULL;
1921
1922         pf= MEM_mallocN(sizeof(PTCacheFile), "PTCacheFile");
1923         pf->fp= fp;
1924         pf->old_format = 0;
1925         pf->frame = cfra;
1926
1927         return pf;
1928 }
1929 static void ptcache_file_close(PTCacheFile *pf)
1930 {
1931         if (pf) {
1932                 fclose(pf->fp);
1933                 MEM_freeN(pf);
1934         }
1935 }
1936
1937 static int ptcache_file_compressed_read(PTCacheFile *pf, unsigned char *result, unsigned int len)
1938 {
1939         int r = 0;
1940         unsigned char compressed = 0;
1941         size_t in_len;
1942 #ifdef WITH_LZO
1943         size_t out_len = len;
1944 #endif
1945         unsigned char *in;
1946         unsigned char *props = MEM_callocN(16 * sizeof(char), "tmp");
1947
1948         ptcache_file_read(pf, &compressed, 1, sizeof(unsigned char));
1949         if (compressed) {
1950                 unsigned int size;
1951                 ptcache_file_read(pf, &size, 1, sizeof(unsigned int));
1952                 in_len = (size_t)size;
1953                 if (in_len==0) {
1954                         /* do nothing */
1955                 }
1956                 else {
1957                         in = (unsigned char *)MEM_callocN(sizeof(unsigned char)*in_len, "pointcache_compressed_buffer");
1958                         ptcache_file_read(pf, in, in_len, sizeof(unsigned char));
1959 #ifdef WITH_LZO
1960                         if (compressed == 1)
1961                                 r = lzo1x_decompress_safe(in, (lzo_uint)in_len, result, (lzo_uint *)&out_len, NULL);
1962 #endif
1963 #ifdef WITH_LZMA
1964                         if (compressed == 2) {
1965                                 size_t sizeOfIt;
1966                                 size_t leni = in_len, leno = len;
1967                                 ptcache_file_read(pf, &size, 1, sizeof(unsigned int));
1968                                 sizeOfIt = (size_t)size;
1969                                 ptcache_file_read(pf, props, sizeOfIt, sizeof(unsigned char));
1970                                 r = LzmaUncompress(result, &leno, in, &leni, props, sizeOfIt);
1971                         }
1972 #endif
1973                         MEM_freeN(in);
1974                 }
1975         }
1976         else {
1977                 ptcache_file_read(pf, result, len, sizeof(unsigned char));
1978         }
1979
1980         MEM_freeN(props);
1981
1982         return r;
1983 }
1984 static int ptcache_file_compressed_write(PTCacheFile *pf, unsigned char *in, unsigned int in_len, unsigned char *out, int mode)
1985 {
1986         int r = 0;
1987         unsigned char compressed = 0;
1988         size_t out_len= 0;
1989         unsigned char *props = MEM_callocN(16 * sizeof(char), "tmp");
1990         size_t sizeOfIt = 5;
1991
1992         (void)mode; /* unused when building w/o compression */
1993
1994 #ifdef WITH_LZO
1995         out_len= LZO_OUT_LEN(in_len);
1996         if (mode == 1) {
1997                 LZO_HEAP_ALLOC(wrkmem, LZO1X_MEM_COMPRESS);
1998                 
1999                 r = lzo1x_1_compress(in, (lzo_uint)in_len, out, (lzo_uint *)&out_len, wrkmem);
2000                 if (!(r == LZO_E_OK) || (out_len >= in_len))
2001                         compressed = 0;
2002                 else
2003                         compressed = 1;
2004         }
2005 #endif
2006 #ifdef WITH_LZMA
2007         if (mode == 2) {
2008                 
2009                 r = LzmaCompress(out, &out_len, in, in_len, //assume sizeof(char)==1....
2010                                  props, &sizeOfIt, 5, 1 << 24, 3, 0, 2, 32, 2);
2011
2012                 if (!(r == SZ_OK) || (out_len >= in_len))
2013                         compressed = 0;
2014                 else
2015                         compressed = 2;
2016         }
2017 #endif
2018         
2019         ptcache_file_write(pf, &compressed, 1, sizeof(unsigned char));
2020         if (compressed) {
2021                 unsigned int size = out_len;
2022                 ptcache_file_write(pf, &size, 1, sizeof(unsigned int));
2023                 ptcache_file_write(pf, out, out_len, sizeof(unsigned char));
2024         }
2025         else
2026                 ptcache_file_write(pf, in, in_len, sizeof(unsigned char));
2027
2028         if (compressed == 2) {
2029                 unsigned int size = sizeOfIt;
2030                 ptcache_file_write(pf, &sizeOfIt, 1, sizeof(unsigned int));
2031                 ptcache_file_write(pf, props, size, sizeof(unsigned char));
2032         }
2033
2034         MEM_freeN(props);
2035
2036         return r;
2037 }
2038 static int ptcache_file_read(PTCacheFile *pf, void *f, unsigned int tot, unsigned int size)
2039 {
2040         return (fread(f, size, tot, pf->fp) == tot);
2041 }
2042 static int ptcache_file_write(PTCacheFile *pf, const void *f, unsigned int tot, unsigned int size)
2043 {
2044         return (fwrite(f, size, tot, pf->fp) == tot);
2045 }
2046 static int ptcache_file_data_read(PTCacheFile *pf)
2047 {
2048         int i;
2049
2050         for (i=0; i<BPHYS_TOT_DATA; i++) {
2051                 if ((pf->data_types & (1<<i)) && !ptcache_file_read(pf, pf->cur[i], 1, ptcache_data_size[i]))
2052                         return 0;
2053         }
2054         
2055         return 1;
2056 }
2057 static int ptcache_file_data_write(PTCacheFile *pf)
2058 {               
2059         int i;
2060
2061         for (i=0; i<BPHYS_TOT_DATA; i++) {
2062                 if ((pf->data_types & (1<<i)) && !ptcache_file_write(pf, pf->cur[i], 1, ptcache_data_size[i]))
2063                         return 0;
2064         }
2065         
2066         return 1;
2067 }
2068 static int ptcache_file_header_begin_read(PTCacheFile *pf)
2069 {
2070         unsigned int typeflag=0;
2071         int error=0;
2072         char bphysics[8];
2073         
2074         pf->data_types = 0;
2075         
2076         if (fread(bphysics, sizeof(char), 8, pf->fp) != 8)
2077                 error = 1;
2078         
2079         if (!error && !STREQLEN(bphysics, "BPHYSICS", 8))
2080                 error = 1;
2081
2082         if (!error && !fread(&typeflag, sizeof(unsigned int), 1, pf->fp))
2083                 error = 1;
2084
2085         pf->type = (typeflag & PTCACHE_TYPEFLAG_TYPEMASK);
2086         pf->flag = (typeflag & PTCACHE_TYPEFLAG_FLAGMASK);
2087         
2088         /* if there was an error set file as it was */
2089         if (error)
2090                 fseek(pf->fp, 0, SEEK_SET);
2091
2092         return !error;
2093 }
2094 static int ptcache_file_header_begin_write(PTCacheFile *pf)
2095 {
2096         const char *bphysics = "BPHYSICS";
2097         unsigned int typeflag = pf->type + pf->flag;
2098         
2099         if (fwrite(bphysics, sizeof(char), 8, pf->fp) != 8)
2100                 return 0;
2101
2102         if (!fwrite(&typeflag, sizeof(unsigned int), 1, pf->fp))
2103                 return 0;
2104         
2105         return 1;
2106 }
2107
2108 /* Data pointer handling */
2109 int BKE_ptcache_data_size(int data_type)
2110 {
2111         return ptcache_data_size[data_type];
2112 }
2113
2114 static void ptcache_file_pointers_init(PTCacheFile *pf)
2115 {
2116         int data_types = pf->data_types;
2117
2118         pf->cur[BPHYS_DATA_INDEX] =             (data_types & (1<<BPHYS_DATA_INDEX))    ?               &pf->data.index : NULL;
2119         pf->cur[BPHYS_DATA_LOCATION] =  (data_types & (1<<BPHYS_DATA_LOCATION)) ?               &pf->data.loc   : NULL;
2120         pf->cur[BPHYS_DATA_VELOCITY] =  (data_types & (1<<BPHYS_DATA_VELOCITY)) ?               &pf->data.vel   : NULL;
2121         pf->cur[BPHYS_DATA_ROTATION] =  (data_types & (1<<BPHYS_DATA_ROTATION)) ?               &pf->data.rot   : NULL;
2122         pf->cur[BPHYS_DATA_AVELOCITY] = (data_types & (1<<BPHYS_DATA_AVELOCITY))?               &pf->data.ave   : NULL;
2123         pf->cur[BPHYS_DATA_SIZE] =              (data_types & (1<<BPHYS_DATA_SIZE))             ?               &pf->data.size  : NULL;
2124         pf->cur[BPHYS_DATA_TIMES] =             (data_types & (1<<BPHYS_DATA_TIMES))    ?               &pf->data.times : NULL;
2125         pf->cur[BPHYS_DATA_BOIDS] =             (data_types & (1<<BPHYS_DATA_BOIDS))    ?               &pf->data.boids : NULL;
2126 }
2127
2128 /* Check to see if point number "index" is in pm, uses binary search for index data. */
2129 int BKE_ptcache_mem_index_find(PTCacheMem *pm, unsigned int index)
2130 {
2131         if (pm->totpoint > 0 && pm->data[BPHYS_DATA_INDEX]) {
2132                 unsigned int *data = pm->data[BPHYS_DATA_INDEX];
2133                 unsigned int mid, low = 0, high = pm->totpoint - 1;
2134
2135                 if (index < *data || index > *(data+high))
2136                         return -1;
2137
2138                 /* check simple case for continuous indexes first */
2139                 if (index-*data < high && data[index-*data] == index)
2140                         return index-*data;
2141
2142                 while (low <= high) {
2143                         mid= (low + high)/2;
2144
2145                         if (data[mid] > index)
2146                                 high = mid - 1;
2147                         else if (data[mid] < index)
2148                                 low = mid + 1;
2149                         else
2150                                 return mid;
2151                 }
2152
2153                 return -1;
2154         }
2155         else {
2156                 return (index < pm->totpoint ? index : -1);
2157         }
2158 }
2159
2160 void BKE_ptcache_mem_pointers_init(PTCacheMem *pm)
2161 {
2162         int data_types = pm->data_types;
2163         int i;
2164
2165         for (i=0; i<BPHYS_TOT_DATA; i++)
2166                 pm->cur[i] = ((data_types & (1<<i)) ? pm->data[i] : NULL);
2167 }
2168
2169 void BKE_ptcache_mem_pointers_incr(PTCacheMem *pm)
2170 {
2171         int i;
2172
2173         for (i=0; i<BPHYS_TOT_DATA; i++) {
2174                 if (pm->cur[i])
2175                         pm->cur[i] = (char *)pm->cur[i] + ptcache_data_size[i];
2176         }
2177 }
2178 int  BKE_ptcache_mem_pointers_seek(int point_index, PTCacheMem *pm)
2179 {
2180         int data_types = pm->data_types;
2181         int i, index = BKE_ptcache_mem_index_find(pm, point_index);
2182
2183         if (index < 0) {
2184                 /* Can't give proper location without reallocation, so don't give any location.
2185                  * Some points will be cached improperly, but this only happens with simulation
2186                  * steps bigger than cache->step, so the cache has to be recalculated anyways
2187                  * at some point.
2188                  */
2189                 return 0;
2190         }
2191
2192         for (i=0; i<BPHYS_TOT_DATA; i++)
2193                 pm->cur[i] = data_types & (1<<i) ? (char *)pm->data[i] + index * ptcache_data_size[i] : NULL;
2194
2195         return 1;
2196 }
2197 static void ptcache_data_alloc(PTCacheMem *pm)
2198 {
2199         int data_types = pm->data_types;
2200         int totpoint = pm->totpoint;
2201         int i;
2202
2203         for (i=0; i<BPHYS_TOT_DATA; i++) {
2204                 if (data_types & (1<<i))
2205                         pm->data[i] = MEM_callocN(totpoint * ptcache_data_size[i], "PTCache Data");
2206         }
2207 }
2208 static void ptcache_data_free(PTCacheMem *pm)
2209 {
2210         void **data = pm->data;
2211         int i;
2212
2213         for (i=0; i<BPHYS_TOT_DATA; i++) {
2214                 if (data[i])
2215                         MEM_freeN(data[i]);
2216         }
2217 }
2218 static void ptcache_data_copy(void *from[], void *to[])
2219 {
2220         int i;
2221         for (i=0; i<BPHYS_TOT_DATA; i++) {
2222                 /* note, durian file 03.4b_comp crashes if to[i] is not tested
2223                  * its NULL, not sure if this should be fixed elsewhere but for now its needed */
2224                 if (from[i] && to[i])
2225                         memcpy(to[i], from[i], ptcache_data_size[i]);
2226         }
2227 }
2228
2229 static void ptcache_extra_free(PTCacheMem *pm)
2230 {
2231         PTCacheExtra *extra = pm->extradata.first;
2232
2233         if (extra) {
2234                 for (; extra; extra=extra->next) {
2235                         if (extra->data)
2236                                 MEM_freeN(extra->data);
2237                 }
2238
2239                 BLI_freelistN(&pm->extradata);
2240         }
2241 }
2242 static int ptcache_old_elemsize(PTCacheID *pid)
2243 {
2244         if (pid->type==PTCACHE_TYPE_SOFTBODY)
2245                 return 6 * sizeof(float);
2246         else if (pid->type==PTCACHE_TYPE_PARTICLES)
2247                 return sizeof(ParticleKey);
2248         else if (pid->type==PTCACHE_TYPE_CLOTH)
2249                 return 9 * sizeof(float);
2250
2251         return 0;
2252 }
2253
2254 static void ptcache_find_frames_around(PTCacheID *pid, unsigned int frame, int *fra1, int *fra2)
2255 {
2256         if (pid->cache->flag & PTCACHE_DISK_CACHE) {
2257                 int cfra1=frame, cfra2=frame+1;
2258
2259                 while (cfra1 >= pid->cache->startframe && !BKE_ptcache_id_exist(pid, cfra1))
2260                         cfra1--;
2261
2262                 if (cfra1 < pid->cache->startframe)
2263                         cfra1 = 0;
2264
2265                 while (cfra2 <= pid->cache->endframe && !BKE_ptcache_id_exist(pid, cfra2))
2266                         cfra2++;
2267
2268                 if (cfra2 > pid->cache->endframe)
2269                         cfra2 = 0;
2270
2271                 if (cfra1 && !cfra2) {
2272                         *fra1 = 0;
2273                         *fra2 = cfra1;
2274                 }
2275                 else {
2276                         *fra1 = cfra1;
2277                         *fra2 = cfra2;
2278                 }
2279         }
2280         else if (pid->cache->mem_cache.first) {
2281                 PTCacheMem *pm = pid->cache->mem_cache.first;
2282                 PTCacheMem *pm2 = pid->cache->mem_cache.last;
2283
2284                 while (pm->next && pm->next->frame <= frame)
2285                         pm= pm->next;
2286
2287                 if (pm2->frame < frame) {
2288                         pm2 = NULL;
2289                 }
2290                 else {
2291                         while (pm2->prev && pm2->prev->frame > frame) {
2292                                 pm2= pm2->prev;
2293                         }
2294                 }
2295
2296                 if (!pm2) {
2297                         *fra1 = 0;
2298                         *fra2 = pm->frame;
2299                 }
2300                 else {
2301                         *fra1 = pm->frame;
2302                         *fra2 = pm2->frame;
2303                 }
2304         }
2305 }
2306
2307 static PTCacheMem *ptcache_disk_frame_to_mem(PTCacheID *pid, int cfra)
2308 {
2309         PTCacheFile *pf = ptcache_file_open(pid, PTCACHE_FILE_READ, cfra);
2310         PTCacheMem *pm = NULL;
2311         unsigned int i, error = 0;
2312
2313         if (pf == NULL)
2314                 return NULL;
2315
2316         if (!ptcache_file_header_begin_read(pf))
2317                 error = 1;
2318
2319         if (!error && (pf->type != pid->type || !pid->read_header(pf)))
2320                 error = 1;
2321
2322         if (!error) {
2323                 pm = MEM_callocN(sizeof(PTCacheMem), "Pointcache mem");
2324
2325                 pm->totpoint = pf->totpoint;
2326                 pm->data_types = pf->data_types;
2327                 pm->frame = pf->frame;
2328
2329                 ptcache_data_alloc(pm);
2330
2331                 if (pf->flag & PTCACHE_TYPEFLAG_COMPRESS) {
2332                         for (i=0; i<BPHYS_TOT_DATA; i++) {
2333                                 unsigned int out_len = pm->totpoint*ptcache_data_size[i];
2334                                 if (pf->data_types & (1<<i))
2335                                         ptcache_file_compressed_read(pf, (unsigned char *)(pm->data[i]), out_len);
2336                         }
2337                 }
2338                 else {
2339                         BKE_ptcache_mem_pointers_init(pm);
2340                         ptcache_file_pointers_init(pf);
2341
2342                         for (i=0; i<pm->totpoint; i++) {
2343                                 if (!ptcache_file_data_read(pf)) {
2344                                         error = 1;
2345                                         break;
2346                                 }
2347                                 ptcache_data_copy(pf->cur, pm->cur);
2348                                 BKE_ptcache_mem_pointers_incr(pm);
2349                         }
2350                 }
2351         }
2352
2353         if (!error && pf->flag & PTCACHE_TYPEFLAG_EXTRADATA) {
2354                 unsigned int extratype = 0;
2355
2356                 while (ptcache_file_read(pf, &extratype, 1, sizeof(unsigned int))) {
2357                         PTCacheExtra *extra = MEM_callocN(sizeof(PTCacheExtra), "Pointcache extradata");
2358
2359                         extra->type = extratype;
2360
2361                         ptcache_file_read(pf, &extra->totdata, 1, sizeof(unsigned int));
2362
2363                         extra->data = MEM_callocN(extra->totdata * ptcache_extra_datasize[extra->type], "Pointcache extradata->data");
2364
2365                         if (pf->flag & PTCACHE_TYPEFLAG_COMPRESS)
2366                                 ptcache_file_compressed_read(pf, (unsigned char *)(extra->data), extra->totdata*ptcache_extra_datasize[extra->type]);
2367                         else
2368                                 ptcache_file_read(pf, extra->data, extra->totdata, ptcache_extra_datasize[extra->type]);
2369
2370                         BLI_addtail(&pm->extradata, extra);
2371                 }
2372         }
2373
2374         if (error && pm) {
2375                 ptcache_data_free(pm);
2376                 ptcache_extra_free(pm);
2377                 MEM_freeN(pm);
2378                 pm = NULL;
2379         }
2380
2381         ptcache_file_close(pf);
2382
2383         if (error && G.debug & G_DEBUG)
2384                 printf("Error reading from disk cache\n");
2385         
2386         return pm;
2387 }
2388 static int ptcache_mem_frame_to_disk(PTCacheID *pid, PTCacheMem *pm)
2389 {
2390         PTCacheFile *pf = NULL;
2391         unsigned int i, error = 0;
2392         
2393         BKE_ptcache_id_clear(pid, PTCACHE_CLEAR_FRAME, pm->frame);
2394
2395         pf = ptcache_file_open(pid, PTCACHE_FILE_WRITE, pm->frame);
2396
2397         if (pf==NULL) {
2398                 if (G.debug & G_DEBUG)
2399                         printf("Error opening disk cache file for writing\n");
2400                 return 0;
2401         }
2402
2403         pf->data_types = pm->data_types;
2404         pf->totpoint = pm->totpoint;
2405         pf->type = pid->type;
2406         pf->flag = 0;
2407         
2408         if (pm->extradata.first)
2409                 pf->flag |= PTCACHE_TYPEFLAG_EXTRADATA;
2410         
2411         if (pid->cache->compression)
2412                 pf->flag |= PTCACHE_TYPEFLAG_COMPRESS;
2413
2414         if (!ptcache_file_header_begin_write(pf) || !pid->write_header(pf))
2415                 error = 1;
2416
2417         if (!error) {
2418                 if (pid->cache->compression) {
2419                         for (i=0; i<BPHYS_TOT_DATA; i++) {
2420                                 if (pm->data[i]) {
2421                                         unsigned int in_len = pm->totpoint*ptcache_data_size[i];
2422                                         unsigned char *out = (unsigned char *)MEM_callocN(LZO_OUT_LEN(in_len) * 4, "pointcache_lzo_buffer");
2423                                         ptcache_file_compressed_write(pf, (unsigned char *)(pm->data[i]), in_len, out, pid->cache->compression);
2424                                         MEM_freeN(out);
2425                                 }
2426                         }
2427                 }
2428                 else {
2429                         BKE_ptcache_mem_pointers_init(pm);
2430                         ptcache_file_pointers_init(pf);
2431
2432                         for (i=0; i<pm->totpoint; i++) {
2433                                 ptcache_data_copy(pm->cur, pf->cur);
2434                                 if (!ptcache_file_data_write(pf)) {
2435                                         error = 1;
2436                                         break;
2437                                 }
2438                                 BKE_ptcache_mem_pointers_incr(pm);
2439                         }
2440                 }
2441         }
2442
2443         if (!error && pm->extradata.first) {
2444                 PTCacheExtra *extra = pm->extradata.first;
2445
2446                 for (; extra; extra=extra->next) {
2447                         if (extra->data == NULL || extra->totdata == 0)
2448                                 continue;
2449
2450                         ptcache_file_write(pf, &extra->type, 1, sizeof(unsigned int));
2451                         ptcache_file_write(pf, &extra->totdata, 1, sizeof(unsigned int));
2452
2453                         if (pid->cache->compression) {
2454                                 unsigned int in_len = extra->totdata * ptcache_extra_datasize[extra->type];
2455                                 unsigned char *out = (unsigned char *)MEM_callocN(LZO_OUT_LEN(in_len) * 4, "pointcache_lzo_buffer");
2456                                 ptcache_file_compressed_write(pf, (unsigned char *)(extra->data), in_len, out, pid->cache->compression);
2457                                 MEM_freeN(out);
2458                         }
2459                         else {
2460                                 ptcache_file_write(pf, extra->data, extra->totdata, ptcache_extra_datasize[extra->type]);
2461                         }
2462                 }
2463         }
2464
2465         ptcache_file_close(pf);
2466         
2467         if (error && G.debug & G_DEBUG)
2468                 printf("Error writing to disk cache\n");
2469
2470         return error==0;
2471 }
2472
2473 static int ptcache_read_stream(PTCacheID *pid, int cfra)
2474 {
2475         PTCacheFile *pf = ptcache_file_open(pid, PTCACHE_FILE_READ, cfra);
2476         int error = 0;
2477
2478         if (pid->read_stream == NULL)
2479                 return 0;
2480
2481         if (pf == NULL) {
2482                 if (G.debug & G_DEBUG)
2483                         printf("Error opening disk cache file for reading\n");
2484                 return 0;
2485         }
2486
2487         if (!ptcache_file_header_begin_read(pf)) {
2488                 pid->error(pid->calldata, "Failed to read point cache file");
2489                 error = 1;
2490         }
2491         else if (pf->type != pid->type) {
2492                 pid->error(pid->calldata, "Point cache file has wrong type");
2493                 error = 1;
2494         }
2495         else if (!pid->read_header(pf)) {
2496                 pid->error(pid->calldata, "Failed to read point cache file header");
2497                 error = 1;
2498         }
2499         else if (pf->totpoint != pid->totpoint(pid->calldata, cfra)) {
2500                 pid->error(pid->calldata, "Number of points in cache does not match mesh");
2501                 error = 1;
2502         }
2503
2504         if (!error) {
2505                 ptcache_file_pointers_init(pf);
2506
2507                 // we have stream reading here
2508                 if (!pid->read_stream(pf, pid->calldata)) {
2509                         pid->error(pid->calldata, "Failed to read point cache file data");
2510                         error = 1;
2511                 }
2512         }
2513
2514         ptcache_file_close(pf);
2515         
2516         return error == 0;
2517 }
2518
2519 static int ptcache_read_openvdb_stream(PTCacheID *pid, int cfra)
2520 {
2521 #ifdef WITH_OPENVDB
2522         char filename[FILE_MAX * 2];
2523
2524         /* save blend file before using disk pointcache */
2525         if (!G.relbase_valid && (pid->cache->flag & PTCACHE_EXTERNAL) == 0)
2526                 return 0;
2527
2528         ptcache_filename(pid, filename, cfra, 1, 1);
2529
2530         if (!BLI_exists(filename)) {
2531                 return 0;
2532         }
2533
2534         struct OpenVDBReader *reader = OpenVDBReader_create();
2535         OpenVDBReader_open(reader, filename);
2536
2537         if (!pid->read_openvdb_stream(reader, pid->calldata)) {
2538                 return 0;
2539         }
2540
2541         return 1;
2542 #else
2543         UNUSED_VARS(pid, cfra);
2544         return 0;
2545 #endif
2546 }
2547
2548 static int ptcache_read(PTCacheID *pid, int cfra)
2549 {
2550         PTCacheMem *pm = NULL;
2551         int i;
2552         int *index = &i;
2553
2554         /* get a memory cache to read from */
2555         if (pid->cache->flag & PTCACHE_DISK_CACHE) {
2556                 pm = ptcache_disk_frame_to_mem(pid, cfra);
2557         }
2558         else {
2559                 pm = pid->cache->mem_cache.first;
2560                 
2561                 while (pm && pm->frame != cfra)
2562                         pm = pm->next;
2563         }
2564
2565         /* read the cache */
2566         if (pm) {
2567                 int totpoint = pm->totpoint;
2568
2569                 if ((pid->data_types & (1<<BPHYS_DATA_INDEX)) == 0) {
2570                         int pid_totpoint = pid->totpoint(pid->calldata, cfra);
2571
2572                         if (totpoint != pid_totpoint) {
2573                                 pid->error(pid->calldata, "Number of points in cache does not match mesh");
2574                                 totpoint = MIN2(totpoint, pid_totpoint);
2575                         }
2576                 }
2577
2578                 BKE_ptcache_mem_pointers_init(pm);
2579
2580                 for (i=0; i<totpoint; i++) {
2581                         if (pm->data_types & (1<<BPHYS_DATA_INDEX))
2582                                 index = pm->cur[BPHYS_DATA_INDEX];
2583
2584                         pid->read_point(*index, pid->calldata, pm->cur, (float)pm->frame, NULL);
2585                 
2586                         BKE_ptcache_mem_pointers_incr(pm);
2587                 }
2588
2589                 if (pid->read_extra_data && pm->extradata.first)
2590                         pid->read_extra_data(pid->calldata, pm, (float)pm->frame);
2591
2592                 /* clean up temporary memory cache */
2593                 if (pid->cache->flag & PTCACHE_DISK_CACHE) {
2594                         ptcache_data_free(pm);
2595                         ptcache_extra_free(pm);
2596                         MEM_freeN(pm);
2597                 }
2598         }
2599
2600         return 1;
2601 }
2602 static int ptcache_interpolate(PTCacheID *pid, float cfra, int cfra1, int cfra2)
2603 {
2604         PTCacheMem *pm = NULL;
2605         int i;
2606         int *index = &i;
2607
2608         /* get a memory cache to read from */
2609         if (pid->cache->flag & PTCACHE_DISK_CACHE) {
2610                 pm = ptcache_disk_frame_to_mem(pid, cfra2);
2611         }
2612         else {
2613                 pm = pid->cache->mem_cache.first;
2614                 
2615                 while (pm && pm->frame != cfra2)
2616                         pm = pm->next;
2617         }
2618
2619         /* read the cache */
2620         if (pm) {
2621                 int totpoint = pm->totpoint;
2622
2623                 if ((pid->data_types & (1<<BPHYS_DATA_INDEX)) == 0) {
2624                         int pid_totpoint = pid->totpoint(pid->calldata, (int)cfra);
2625
2626                         if (totpoint != pid_totpoint) {
2627                                 pid->error(pid->calldata, "Number of points in cache does not match mesh");
2628                                 totpoint = MIN2(totpoint, pid_totpoint);
2629                         }
2630                 }
2631
2632                 BKE_ptcache_mem_pointers_init(pm);
2633
2634                 for (i=0; i<totpoint; i++) {
2635                         if (pm->data_types & (1<<BPHYS_DATA_INDEX))
2636                                 index = pm->cur[BPHYS_DATA_INDEX];
2637
2638                         pid->interpolate_point(*index, pid->calldata, pm->cur, cfra, (float)cfra1, (float)cfra2, NULL);
2639                         BKE_ptcache_mem_pointers_incr(pm);
2640                 }
2641
2642                 if (pid->interpolate_extra_data && pm->extradata.first)
2643                         pid->interpolate_extra_data(pid->calldata, pm, cfra, (float)cfra1, (float)cfra2);
2644
2645                 /* clean up temporary memory cache */
2646                 if (pid->cache->flag & PTCACHE_DISK_CACHE) {
2647                         ptcache_data_free(pm);
2648                         ptcache_extra_free(pm);
2649                         MEM_freeN(pm);
2650                 }
2651         }
2652
2653         return 1;
2654 }
2655 /* reads cache from disk or memory */
2656 /* possible to get old or interpolated result */
2657 int BKE_ptcache_read(PTCacheID *pid, float cfra, bool no_extrapolate_old)
2658 {
2659         int cfrai = (int)floor(cfra), cfra1=0, cfra2=0;
2660         int ret = 0;
2661
2662         /* nothing to read to */
2663         if (pid->totpoint(pid->calldata, cfrai) == 0)
2664                 return 0;
2665
2666         if (pid->cache->flag & PTCACHE_READ_INFO) {
2667                 pid->cache->flag &= ~PTCACHE_READ_INFO;
2668                 ptcache_read(pid, 0);
2669         }
2670
2671         /* first check if we have the actual frame cached */
2672         if (cfra == (float)cfrai && BKE_ptcache_id_exist(pid, cfrai))
2673                 cfra1 = cfrai;
2674
2675         /* no exact cache frame found so try to find cached frames around cfra */
2676         if (cfra1 == 0)
2677                 ptcache_find_frames_around(pid, cfrai, &cfra1, &cfra2);
2678
2679         if (cfra1 == 0 && cfra2 == 0)
2680                 return 0;
2681
2682         /* don't read old cache if already simulated past cached frame */
2683         if (no_extrapolate_old) {
2684                 if (cfra1 == 0 && cfra2 && cfra2 <= pid->cache->simframe)
2685                         return 0;
2686                 if (cfra1 && cfra1 == cfra2)
2687                         return 0;
2688         }
2689         else {
2690                 /* avoid calling interpolate between the same frame values */
2691                 if (cfra1 && cfra1 == cfra2)
2692                         cfra1 = 0;
2693         }
2694
2695         if (cfra1) {
2696                 if (pid->file_type == PTCACHE_FILE_OPENVDB && pid->read_openvdb_stream) {
2697                         if (!ptcache_read_openvdb_stream(pid, cfra1)) {
2698                                 return 0;
2699                         }
2700                 }
2701                 else if (pid->read_stream) {
2702                         if (!ptcache_read_stream(pid, cfra1))
2703                                 return 0;
2704                 }
2705                 else if (pid->read_point)
2706                         ptcache_read(pid, cfra1);
2707         }
2708
2709         if (cfra2) {
2710                 if (pid->file_type == PTCACHE_FILE_OPENVDB && pid->read_openvdb_stream) {
2711                         if (!ptcache_read_openvdb_stream(pid, cfra2)) {
2712                                 return 0;
2713                         }
2714                 }
2715                 else if (pid->read_stream) {
2716                         if (!ptcache_read_stream(pid, cfra2))
2717                                 return 0;
2718                 }
2719                 else if (pid->read_point) {
2720                         if (cfra1 && cfra2 && pid->interpolate_point)
2721                                 ptcache_interpolate(pid, cfra, cfra1, cfra2);
2722                         else
2723                                 ptcache_read(pid, cfra2);
2724                 }
2725         }
2726
2727         if (cfra1)
2728                 ret = (cfra2 ? PTCACHE_READ_INTERPOLATED : PTCACHE_READ_EXACT);
2729         else if (cfra2) {
2730                 ret = PTCACHE_READ_OLD;
2731                 pid->cache->simframe = cfra2;
2732         }
2733
2734         cfrai = (int)cfra;
2735         /* clear invalid cache frames so that better stuff can be simulated */
2736         if (pid->cache->flag & PTCACHE_OUTDATED) {
2737                 BKE_ptcache_id_clear(pid, PTCACHE_CLEAR_AFTER, cfrai);
2738         }
2739         else if (pid->cache->flag & PTCACHE_FRAMES_SKIPPED) {
2740                 if (cfra <= pid->cache->last_exact)
2741                         pid->cache->flag &= ~PTCACHE_FRAMES_SKIPPED;
2742
2743                 BKE_ptcache_id_clear(pid, PTCACHE_CLEAR_AFTER, MAX2(cfrai, pid->cache->last_exact));
2744         }
2745
2746         return ret;
2747 }
2748 static int ptcache_write_stream(PTCacheID *pid, int cfra, int totpoint)
2749 {
2750         PTCacheFile *pf = NULL;
2751         int error = 0;
2752         
2753         BKE_ptcache_id_clear(pid, PTCACHE_CLEAR_FRAME, cfra);
2754
2755         pf = ptcache_file_open(pid, PTCACHE_FILE_WRITE, cfra);
2756
2757         if (pf==NULL) {
2758                 if (G.debug & G_DEBUG)
2759                         printf("Error opening disk cache file for writing\n");
2760                 return 0;
2761         }
2762
2763         pf->data_types = pid->data_types;
2764         pf->totpoint = totpoint;
2765         pf->type = pid->type;
2766         pf->flag = 0;
2767
2768         if (!error && (!ptcache_file_header_begin_write(pf) || !pid->write_header(pf)))
2769                 error = 1;
2770
2771         if (!error && pid->write_stream)
2772                 pid->write_stream(pf, pid->calldata);
2773
2774         ptcache_file_close(pf);
2775
2776         if (error && G.debug & G_DEBUG)
2777                 printf("Error writing to disk cache\n");
2778
2779         return error == 0;
2780 }
2781 static int ptcache_write_openvdb_stream(PTCacheID *pid, int cfra)
2782 {
2783 #ifdef WITH_OPENVDB
2784         struct OpenVDBWriter *writer = OpenVDBWriter_create();
2785         char filename[FILE_MAX * 2];
2786
2787         BKE_ptcache_id_clear(pid, PTCACHE_CLEAR_FRAME, cfra);
2788
2789         ptcache_filename(pid, filename, cfra, 1, 1);
2790         BLI_make_existing_file(filename);
2791
2792         int error = pid->write_openvdb_stream(writer, pid->calldata);
2793
2794         OpenVDBWriter_write(writer, filename);
2795         OpenVDBWriter_free(writer);
2796
2797         return error == 0;
2798 #else
2799         UNUSED_VARS(pid, cfra);
2800         return 0;
2801 #endif
2802 }
2803 static int ptcache_write(PTCacheID *pid, int cfra, int overwrite)
2804 {
2805         PointCache *cache = pid->cache;
2806         PTCacheMem *pm=NULL, *pm2=NULL;
2807         int totpoint = pid->totpoint(pid->calldata, cfra);
2808         int i, error = 0;
2809
2810         pm = MEM_callocN(sizeof(PTCacheMem), "Pointcache mem");
2811
2812         pm->totpoint = pid->totwrite(pid->calldata, cfra);
2813         pm->data_types = cfra ? pid->data_types : pid->info_types;
2814
2815         ptcache_data_alloc(pm);
2816         BKE_ptcache_mem_pointers_init(pm);
2817
2818         if (overwrite) {
2819                 if (cache->flag & PTCACHE_DISK_CACHE) {
2820                         int fra = cfra-1;
2821
2822                         while (fra >= cache->startframe && !BKE_ptcache_id_exist(pid, fra))
2823                                 fra--;
2824                         
2825                         pm2 = ptcache_disk_frame_to_mem(pid, fra);
2826                 }
2827                 else
2828                         pm2 = cache->mem_cache.last;
2829         }
2830
2831         if (pid->write_point) {
2832                 for (i=0; i<totpoint; i++) {
2833                         int write = pid->write_point(i, pid->calldata, pm->cur, cfra);
2834                         if (write) {
2835                                 BKE_ptcache_mem_pointers_incr(pm);
2836
2837                                 /* newly born particles have to be copied to previous cached frame */
2838                                 if (overwrite && write == 2 && pm2 && BKE_ptcache_mem_pointers_seek(i, pm2))
2839                                         pid->write_point(i, pid->calldata, pm2->cur, cfra);
2840                         }
2841                 }
2842         }
2843
2844         if (pid->write_extra_data)
2845                 pid->write_extra_data(pid->calldata, pm, cfra);
2846
2847         pm->frame = cfra;
2848
2849         if (cache->flag & PTCACHE_DISK_CACHE) {
2850                 error += !ptcache_mem_frame_to_disk(pid, pm);
2851
2852                 // if (pm) /* pm is always set */
2853                 {
2854                         ptcache_data_free(pm);
2855                         ptcache_extra_free(pm);
2856                         MEM_freeN(pm);
2857                 }
2858
2859                 if (pm2) {
2860                         error += !ptcache_mem_frame_to_disk(pid, pm2);
2861                         ptcache_data_free(pm2);
2862                         ptcache_extra_free(pm2);
2863                         MEM_freeN(pm2);
2864                 }
2865         }
2866         else {
2867                 BLI_addtail(&cache->mem_cache, pm);
2868         }
2869
2870         return error;
2871 }
2872 static int ptcache_write_needed(PTCacheID *pid, int cfra, int *overwrite)
2873 {
2874         PointCache *cache = pid->cache;
2875         int ofra = 0, efra = cache->endframe;
2876
2877         /* always start from scratch on the first frame */
2878         if (cfra && cfra == cache->startframe) {
2879                 BKE_ptcache_id_clear(pid, PTCACHE_CLEAR_ALL, cfra);
2880                 cache->flag &= ~PTCACHE_REDO_NEEDED;
2881                 return 1;
2882         }
2883
2884         if (pid->cache->flag & PTCACHE_DISK_CACHE) {
2885                 if (cfra==0 && cache->startframe > 0)
2886                         return 1;
2887
2888                                 /* find last cached frame */
2889                 while (efra > cache->startframe && !BKE_ptcache_id_exist(pid, efra))
2890                         efra--;
2891
2892                 /* find second last cached frame */
2893                 ofra = efra-1;
2894                 while (ofra > cache->startframe && !BKE_ptcache_id_exist(pid, ofra))
2895                         ofra--;
2896         }
2897         else {
2898                 PTCacheMem *pm = cache->mem_cache.last;
2899                 /* don't write info file in memory */
2900                 if (cfra == 0)
2901                         return 0;
2902
2903                 if (pm == NULL)
2904                         return 1;
2905
2906                 efra = pm->frame;
2907                 ofra = (pm->prev ? pm->prev->frame : efra - cache->step);
2908         }
2909
2910         if (efra >= cache->startframe && cfra > efra) {
2911                 if (ofra >= cache->startframe && efra - ofra < cache->step) {
2912                         /* overwrite previous frame */
2913                         BKE_ptcache_id_clear(pid, PTCACHE_CLEAR_FRAME, efra);
2914                         *overwrite = 1;
2915                 }
2916                 return 1;
2917         }
2918
2919         return 0;
2920 }
2921 /* writes cache to disk or memory */
2922 int BKE_ptcache_write(PTCacheID *pid, unsigned int cfra)
2923 {
2924         PointCache *cache = pid->cache;
2925         int totpoint = pid->totpoint(pid->calldata, cfra);
2926         int overwrite = 0, error = 0;
2927
2928         if (totpoint == 0 || (cfra ? pid->data_types == 0 : pid->info_types == 0))
2929                 return 0;
2930
2931         if (ptcache_write_needed(pid, cfra, &overwrite)==0)
2932                 return 0;
2933
2934         if (pid->file_type == PTCACHE_FILE_OPENVDB && pid->write_openvdb_stream) {
2935                 ptcache_write_openvdb_stream(pid, cfra);
2936         }
2937         else if (pid->write_stream) {
2938                 ptcache_write_stream(pid, cfra, totpoint);
2939         }
2940         else if (pid->write_point) {
2941                 error += ptcache_write(pid, cfra, overwrite);
2942         }
2943
2944         /* Mark frames skipped if more than 1 frame forwards since last non-skipped frame. */
2945         if (cfra - cache->last_exact == 1 || cfra == cache->startframe) {
2946                 cache->last_exact = cfra;
2947                 cache->flag &= ~PTCACHE_FRAMES_SKIPPED;
2948         }
2949         /* Don't mark skipped when writing info file (frame 0) */
2950         else if (cfra)
2951                 cache->flag |= PTCACHE_FRAMES_SKIPPED;
2952
2953         /* Update timeline cache display */
2954         if (cfra && cache->cached_frames)
2955                 cache->cached_frames[cfra-cache->startframe] = 1;
2956
2957         BKE_ptcache_update_info(pid);
2958
2959         return !error;
2960 }
2961 /* youll need to close yourself after!
2962  * mode - PTCACHE_CLEAR_ALL, 
2963  */
2964
2965 /* Clears & resets */
2966 void BKE_ptcache_id_clear(PTCacheID *pid, int mode, unsigned int cfra)
2967 {
2968         unsigned int len; /* store the length of the string */
2969         unsigned int sta, end;
2970
2971         /* mode is same as fopen's modes */
2972         DIR *dir; 
2973         struct dirent *de;
2974         char path[MAX_PTCACHE_PATH];
2975         char filename[MAX_PTCACHE_FILE];
2976         char path_full[MAX_PTCACHE_FILE];
2977         char ext[MAX_PTCACHE_PATH];
2978
2979         if (!pid || !pid->cache || pid->cache->flag & PTCACHE_BAKED)
2980                 return;
2981
2982         if (pid->cache->flag & PTCACHE_IGNORE_CLEAR)
2983                 return;
2984
2985         sta = pid->cache->startframe;
2986         end = pid->cache->endframe;
2987
2988 #ifndef DURIAN_POINTCACHE_LIB_OK
2989         /* don't allow clearing for linked objects */
2990         if (pid->ob->id.lib)
2991                 return;
2992 #endif
2993
2994         /*if (!G.relbase_valid) return; *//* save blend file before using pointcache */
2995
2996         const char *fext = ptcache_file_extension(pid);
2997
2998         /* clear all files in the temp dir with the prefix of the ID and the ".bphys" suffix */
2999         switch (mode) {
3000         case PTCACHE_CLEAR_ALL:
3001         case PTCACHE_CLEAR_BEFORE:
3002         case PTCACHE_CLEAR_AFTER:
3003                 if (pid->cache->flag & PTCACHE_DISK_CACHE) {
3004                         ptcache_path(pid, path);
3005                         
3006                         dir = opendir(path);
3007                         if (dir==NULL)
3008                                 return;
3009                         
3010                         len = ptcache_filename(pid, filename, cfra, 0, 0); /* no path */
3011                         /* append underscore terminator to ensure we don't match similar names
3012                          * from objects whose names start with the same prefix
3013                          */
3014                         if (len < sizeof(filename) - 2) {
3015                                 BLI_strncpy(filename + len, "_", sizeof(filename) - 2 - len);
3016                                 len += 1;
3017                         }
3018                         
3019                         BLI_snprintf(ext, sizeof(ext), "_%02u%s", pid->stack_index, fext);
3020                         
3021                         while ((de = readdir(dir)) != NULL) {
3022                                 if (strstr(de->d_name, ext)) { /* do we have the right extension?*/
3023                                         if (STREQLEN(filename, de->d_name, len)) { /* do we have the right prefix */
3024                                                 if (mode == PTCACHE_CLEAR_ALL) {
3025                                                         pid->cache->last_exact = MIN2(pid->cache->startframe, 0);
3026                                                         BLI_join_dirfile(path_full, sizeof(path_full), path, de->d_name);
3027                                                         BLI_delete(path_full, false, false);
3028                                                 }
3029                                                 else {
3030                                                         /* read the number of the file */
3031                                                         const int frame = ptcache_frame_from_filename(de->d_name, ext);
3032
3033                                                         if (frame != -1) {
3034                                                                 if ((mode == PTCACHE_CLEAR_BEFORE && frame < cfra) ||
3035                                                                     (mode == PTCACHE_CLEAR_AFTER && frame > cfra))
3036                                                                 {
3037                                                                         
3038                                                                         BLI_join_dirfile(path_full, sizeof(path_full), path, de->d_name);
3039                                                                         BLI_delete(path_full, false, false);
3040                                                                         if (pid->cache->cached_frames && frame >=sta && frame <= end)
3041                                                                                 pid->cache->cached_frames[frame-sta] = 0;
3042                                                                 }
3043                                                         }
3044                                                 }
3045                                         }
3046                                 }
3047                         }
3048                         closedir(dir);
3049
3050                         if (mode == PTCACHE_CLEAR_ALL && pid->cache->cached_frames)
3051                                 memset(pid->cache->cached_frames, 0, MEM_allocN_len(pid->cache->cached_frames));
3052                 }
3053                 else {
3054                         PTCacheMem *pm= pid->cache->mem_cache.first;
3055                         PTCacheMem *link= NULL;
3056
3057                         if (mode == PTCACHE_CLEAR_ALL) {
3058                                 /*we want startframe if the cache starts before zero*/
3059                                 pid->cache->last_exact = MIN2(pid->cache->startframe, 0);
3060                                 for (; pm; pm=pm->next) {
3061                                         ptcache_data_free(pm);
3062                                         ptcache_extra_free(pm);
3063                                 }
3064                                 BLI_freelistN(&pid->cache->mem_cache);
3065
3066                                 if (pid->cache->cached_frames) 
3067                                         memset(pid->cache->cached_frames, 0, MEM_allocN_len(pid->cache->cached_frames));
3068                         }
3069                         else {
3070                                 while (pm) {
3071                                         if ((mode == PTCACHE_CLEAR_BEFORE && pm->frame < cfra) ||
3072                                             (mode == PTCACHE_CLEAR_AFTER && pm->frame > cfra))
3073                                         {
3074                                                 link = pm;
3075                                                 if (pid->cache->cached_frames && pm->frame >=sta && pm->frame <= end)
3076                                                         pid->cache->cached_frames[pm->frame-sta] = 0;
3077                                                 ptcache_data_free(pm);
3078                                                 ptcache_extra_free(pm);
3079                                                 pm = pm->next;
3080                                                 BLI_freelinkN(&pid->cache->mem_cache, link);
3081                                         }
3082                                         else
3083                                                 pm = pm->next;
3084                                 }
3085                         }
3086                 }
3087                 break;
3088                 
3089         case PTCACHE_CLEAR_FRAME:
3090                 if (pid->cache->flag & PTCACHE_DISK_CACHE) {
3091                         if (BKE_ptcache_id_exist(pid, cfra)) {
3092                                 ptcache_filename(pid, filename, cfra, 1, 1); /* no path */
3093                                 BLI_delete(filename, false, false);
3094                         }
3095                 }
3096                 else {
3097                         PTCacheMem *pm = pid->cache->mem_cache.first;
3098
3099                         for (; pm; pm=pm->next) {
3100                                 if (pm->frame == cfra) {
3101                                         ptcache_data_free(pm);
3102                                         ptcache_extra_free(pm);
3103                                         BLI_freelinkN(&pid->cache->mem_cache, pm);
3104                                         break;
3105                                 }
3106                         }
3107                 }
3108                 if (pid->cache->cached_frames && cfra >= sta && cfra <= end)
3109                         pid->cache->cached_frames[cfra-sta] = 0;
3110                 break;
3111         }
3112
3113         BKE_ptcache_update_info(pid);
3114 }
3115 int  BKE_ptcache_id_exist(PTCacheID *pid, int cfra)
3116 {
3117         if (!pid->cache)
3118                 return 0;
3119
3120         if (cfra<pid->cache->startframe || cfra > pid->cache->endframe)
3121                 return 0;
3122
3123         if (pid->cache->cached_frames &&        pid->cache->cached_frames[cfra-pid->cache->startframe]==0)
3124                 return 0;
3125         
3126         if (pid->cache->flag & PTCACHE_DISK_CACHE) {
3127                 char filename[MAX_PTCACHE_FILE];
3128                 
3129                 ptcache_filename(pid, filename, cfra, 1, 1);
3130
3131                 return BLI_exists(filename);
3132         }
3133         else {
3134                 PTCacheMem *pm = pid->cache->mem_cache.first;
3135
3136                 for (; pm; pm=pm->next) {
3137                         if (pm->frame==cfra)
3138                                 return 1;
3139                 }
3140                 return 0;
3141         }
3142 }
3143 void BKE_ptcache_id_time(PTCacheID *pid, Scene *scene, float cfra, int *startframe, int *endframe, float *timescale)
3144 {
3145         /* Object *ob; */ /* UNUSED */
3146         PointCache *cache;
3147         /* float offset; unused for now */
3148         float time, nexttime;
3149
3150         /* TODO: this has to be sorted out once bsystem_time gets redone, */
3151         /*       now caches can handle interpolating etc. too - jahka */
3152
3153         /* time handling for point cache:
3154          * - simulation time is scaled by result of bsystem_time
3155          * - for offsetting time only time offset is taken into account, since
3156          *   that's always the same and can't be animated. a timeoffset which
3157          *   varies over time is not simple to support.
3158          * - field and motion blur offsets are currently ignored, proper solution
3159          *   is probably to interpolate results from two frames for that ..
3160          */
3161
3162         /* ob= pid->ob; */ /* UNUSED */
3163         cache= pid->cache;
3164
3165         if (timescale) {
3166                 time= BKE_scene_frame_get(scene);
3167                 nexttime = BKE_scene_frame_get_from_ctime(scene, CFRA + 1.0f);
3168                 
3169                 *timescale= MAX2(nexttime - time, 0.0f);
3170         }
3171
3172         if (startframe && endframe) {
3173                 *startframe= cache->startframe;
3174                 *endframe= cache->endframe;
3175
3176                 /* TODO: time handling with object offsets and si