Small cleanup from previous commit by Danrae Pray (@spockTheGray)
[blender.git] / source / blender / blenloader / intern / readfile.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
19  * All rights reserved.
20  *
21  *
22  * Contributor(s): Blender Foundation
23  *
24  * ***** END GPL LICENSE BLOCK *****
25  *
26  */
27
28 /** \file blender/blenloader/intern/readfile.c
29  *  \ingroup blenloader
30  */
31
32
33 #include "zlib.h"
34
35 #include <limits.h>
36 #include <stdio.h> // for printf fopen fwrite fclose sprintf FILE
37 #include <stdlib.h> // for getenv atoi
38 #include <stddef.h> // for offsetof
39 #include <fcntl.h> // for open
40 #include <string.h> // for strrchr strncmp strstr
41 #include <math.h> // for fabs
42 #include <stdarg.h> /* for va_start/end */
43 #include <time.h> /* for gmtime */
44
45 #include "BLI_utildefines.h"
46 #ifndef WIN32
47 #  include <unistd.h> // for read close
48 #else
49 #  include <io.h> // for open close read
50 #  include "winsock2.h"
51 #  include "BLI_winstuff.h"
52 #endif
53
54 /* allow readfile to use deprecated functionality */
55 #define DNA_DEPRECATED_ALLOW
56 /* Allow using DNA struct members that are marked as private for read/write.
57  * Note: Each header that uses this needs to define its own way of handling
58  * it. There's no generic implementation, direct use does nothing. */
59 #define DNA_PRIVATE_READ_WRITE_ALLOW
60
61 #include "DNA_anim_types.h"
62 #include "DNA_armature_types.h"
63 #include "DNA_actuator_types.h"
64 #include "DNA_brush_types.h"
65 #include "DNA_camera_types.h"
66 #include "DNA_cachefile_types.h"
67 #include "DNA_cloth_types.h"
68 #include "DNA_controller_types.h"
69 #include "DNA_constraint_types.h"
70 #include "DNA_dynamicpaint_types.h"
71 #include "DNA_effect_types.h"
72 #include "DNA_fileglobal_types.h"
73 #include "DNA_genfile.h"
74 #include "DNA_group_types.h"
75 #include "DNA_gpencil_types.h"
76 #include "DNA_ipo_types.h"
77 #include "DNA_key_types.h"
78 #include "DNA_lattice_types.h"
79 #include "DNA_layer_types.h"
80 #include "DNA_lamp_types.h"
81 #include "DNA_linestyle_types.h"
82 #include "DNA_meta_types.h"
83 #include "DNA_material_types.h"
84 #include "DNA_mesh_types.h"
85 #include "DNA_meshdata_types.h"
86 #include "DNA_nla_types.h"
87 #include "DNA_node_types.h"
88 #include "DNA_object_fluidsim.h" // NT
89 #include "DNA_object_types.h"
90 #include "DNA_packedFile_types.h"
91 #include "DNA_particle_types.h"
92 #include "DNA_lightprobe_types.h"
93 #include "DNA_property_types.h"
94 #include "DNA_rigidbody_types.h"
95 #include "DNA_text_types.h"
96 #include "DNA_view3d_types.h"
97 #include "DNA_screen_types.h"
98 #include "DNA_sensor_types.h"
99 #include "DNA_sdna_types.h"
100 #include "DNA_scene_types.h"
101 #include "DNA_sequence_types.h"
102 #include "DNA_smoke_types.h"
103 #include "DNA_speaker_types.h"
104 #include "DNA_sound_types.h"
105 #include "DNA_space_types.h"
106 #include "DNA_vfont_types.h"
107 #include "DNA_workspace_types.h"
108 #include "DNA_world_types.h"
109 #include "DNA_movieclip_types.h"
110 #include "DNA_mask_types.h"
111
112 #include "RNA_access.h"
113
114 #include "MEM_guardedalloc.h"
115
116 #include "BLI_endian_switch.h"
117 #include "BLI_blenlib.h"
118 #include "BLI_math.h"
119 #include "BLI_threads.h"
120 #include "BLI_mempool.h"
121
122 #include "BLT_translation.h"
123
124 #include "BKE_action.h"
125 #include "BKE_armature.h"
126 #include "BKE_brush.h"
127 #include "BKE_cachefile.h"
128 #include "BKE_cloth.h"
129 #include "BKE_constraint.h"
130 #include "BKE_context.h"
131 #include "BKE_curve.h"
132 #include "BKE_effect.h"
133 #include "BKE_fcurve.h"
134 #include "BKE_global.h" // for G
135 #include "BKE_group.h"
136 #include "BKE_layer.h"
137 #include "BKE_library.h" // for which_libbase
138 #include "BKE_library_idmap.h"
139 #include "BKE_library_override.h"
140 #include "BKE_library_query.h"
141 #include "BKE_idcode.h"
142 #include "BKE_idprop.h"
143 #include "BKE_material.h"
144 #include "BKE_main.h" // for Main
145 #include "BKE_mesh.h" // for ME_ defines (patching)
146 #include "BKE_modifier.h"
147 #include "BKE_multires.h"
148 #include "BKE_node.h" // for tree type defines
149 #include "BKE_object.h"
150 #include "BKE_paint.h"
151 #include "BKE_particle.h"
152 #include "BKE_pointcache.h"
153 #include "BKE_report.h"
154 #include "BKE_sca.h" // for init_actuator
155 #include "BKE_scene.h"
156 #include "BKE_screen.h"
157 #include "BKE_sequencer.h"
158 #include "BKE_outliner_treehash.h"
159 #include "BKE_sound.h"
160 #include "BKE_colortools.h"
161 #include "BKE_workspace.h"
162
163 #include "DEG_depsgraph.h"
164
165 #include "NOD_common.h"
166 #include "NOD_socket.h"
167
168 #include "BLO_readfile.h"
169 #include "BLO_undofile.h"
170 #include "BLO_blend_defs.h"
171
172 #include "RE_engine.h"
173
174 #include "readfile.h"
175
176
177 #include <errno.h>
178
179 /**
180  * READ
181  * ====
182  *
183  * - Existing Library (#Main) push or free
184  * - allocate new #Main
185  * - load file
186  * - read #SDNA
187  * - for each LibBlock
188  *   - read LibBlock
189  *   - if a Library
190  *     - make a new #Main
191  *     - attach ID's to it
192  *   - else
193  *     - read associated 'direct data'
194  *     - link direct data (internal and to LibBlock)
195  * - read #FileGlobal
196  * - read #USER data, only when indicated (file is ``~/X.XX/startup.blend``)
197  * - free file
198  * - per Library (per #Main)
199  *   - read file
200  *   - read #SDNA
201  *   - find LibBlocks and attach #ID's to #Main
202  *     - if external LibBlock
203  *       - search all #Main's
204  *         - or it's already read,
205  *         - or not read yet
206  *         - or make new #Main
207  *   - per LibBlock
208  *     - read recursive
209  *     - read associated direct data
210  *     - link direct data (internal and to LibBlock)
211  *   - free file
212  * - per Library with unread LibBlocks
213  *   - read file
214  *   - read #SDNA
215  *   - per LibBlock
216  *     - read recursive
217  *     - read associated direct data
218  *     - link direct data (internal and to LibBlock)
219  *   - free file
220  * - join all #Main's
221  * - link all LibBlocks and indirect pointers to libblocks
222  * - initialize #FileGlobal and copy pointers to #Global
223  *
224  * \note Still a weak point is the new-address function, that doesnt solve reading from
225  * multiple files at the same time.
226  * (added remark: oh, i thought that was solved? will look at that... (ton).
227  */
228
229 /* use GHash for BHead name-based lookups (speeds up linking) */
230 #define USE_GHASH_BHEAD
231
232 /* Use GHash for restoring pointers by name */
233 #define USE_GHASH_RESTORE_POINTER
234
235 /***/
236
237 typedef struct OldNew {
238         const void *old;
239         void *newp;
240         int nr;
241 } OldNew;
242
243 typedef struct OldNewMap {
244         OldNew *entries;
245         int nentries, entriessize;
246         bool sorted;
247         int lasthit;
248 } OldNewMap;
249
250
251 /* local prototypes */
252 static void *read_struct(FileData *fd, BHead *bh, const char *blockname);
253 static void direct_link_modifiers(FileData *fd, ListBase *lb);
254 static BHead *find_bhead_from_code_name(FileData *fd, const short idcode, const char *name);
255 static BHead *find_bhead_from_idname(FileData *fd, const char *idname);
256 static void expand_scene_collection(FileData *fd, Main *mainvar, SceneCollection *sc);
257 static SceneCollection *get_scene_collection_active_or_create(struct Scene *scene, struct ViewLayer *view_layer, const short flag);
258
259 /* this function ensures that reports are printed,
260  * in the case of libraray linking errors this is important!
261  *
262  * bit kludge but better then doubling up on prints,
263  * we could alternatively have a versions of a report function which forces printing - campbell
264  */
265
266 void blo_reportf_wrap(ReportList *reports, ReportType type, const char *format, ...)
267 {
268         char fixed_buf[1024]; /* should be long enough */
269         
270         va_list args;
271         
272         va_start(args, format);
273         vsnprintf(fixed_buf, sizeof(fixed_buf), format, args);
274         va_end(args);
275         
276         fixed_buf[sizeof(fixed_buf) - 1] = '\0';
277         
278         BKE_report(reports, type, fixed_buf);
279         
280         if (G.background == 0) {
281                 printf("%s: %s\n", BKE_report_type_str(type), fixed_buf);
282         }
283 }
284
285 /* for reporting linking messages */
286 static const char *library_parent_filepath(Library *lib)
287 {
288         return lib->parent ? lib->parent->filepath : "<direct>";
289 }
290
291 static OldNewMap *oldnewmap_new(void) 
292 {
293         OldNewMap *onm= MEM_callocN(sizeof(*onm), "OldNewMap");
294         
295         onm->entriessize = 1024;
296         onm->entries = MEM_mallocN(sizeof(*onm->entries)*onm->entriessize, "OldNewMap.entries");
297         
298         return onm;
299 }
300
301 static int verg_oldnewmap(const void *v1, const void *v2)
302 {
303         const struct OldNew *x1=v1, *x2=v2;
304         
305         if (x1->old > x2->old) return 1;
306         else if (x1->old < x2->old) return -1;
307         return 0;
308 }
309
310
311 static void oldnewmap_sort(FileData *fd) 
312 {
313         BLI_assert(fd->libmap->sorted == false);
314         qsort(fd->libmap->entries, fd->libmap->nentries, sizeof(OldNew), verg_oldnewmap);
315         fd->libmap->sorted = 1;
316 }
317
318 /* nr is zero for data, and ID code for libdata */
319 static void oldnewmap_insert(OldNewMap *onm, const void *oldaddr, void *newaddr, int nr)
320 {
321         OldNew *entry;
322         
323         if (oldaddr==NULL || newaddr==NULL) return;
324         
325         if (UNLIKELY(onm->nentries == onm->entriessize)) {
326                 onm->entriessize *= 2;
327                 onm->entries = MEM_reallocN(onm->entries, sizeof(*onm->entries) * onm->entriessize);
328         }
329
330         entry = &onm->entries[onm->nentries++];
331         entry->old = oldaddr;
332         entry->newp = newaddr;
333         entry->nr = nr;
334 }
335
336 void blo_do_versions_oldnewmap_insert(OldNewMap *onm, const void *oldaddr, void *newaddr, int nr)
337 {
338         oldnewmap_insert(onm, oldaddr, newaddr, nr);
339 }
340
341 /**
342  * Do a full search (no state).
343  *
344  * \param lasthit: Use as a reference position to avoid a full search
345  * from either end of the array, giving more efficient lookups.
346  *
347  * \note This would seem an ideal case for hash or btree lookups.
348  * However the data is written in-order, using the \a lasthit will normally avoid calling this function.
349  * Creating a btree/hash structure adds overhead for the common-case to optimize the corner-case
350  * (since most entries will never be retrieved).
351  * So just keep full lookups as a fall-back.
352  */
353 static int oldnewmap_lookup_entry_full(const OldNewMap *onm, const void *addr, int lasthit)
354 {
355         const int nentries = onm->nentries;
356         const OldNew *entries = onm->entries;
357         int i;
358
359         /* search relative to lasthit where possible */
360         if (lasthit >= 0 && lasthit < nentries) {
361
362                 /* search forwards */
363                 i = lasthit;
364                 while (++i != nentries) {
365                         if (entries[i].old == addr) {
366                                 return i;
367                         }
368                 }
369
370                 /* search backwards */
371                 i = lasthit + 1;
372                 while (i--) {
373                         if (entries[i].old == addr) {
374                                 return i;
375                         }
376                 }
377         }
378         else {
379                 /* search backwards (full) */
380                 i = nentries;
381                 while (i--) {
382                         if (entries[i].old == addr) {
383                                 return i;
384                         }
385                 }
386         }
387
388         return -1;
389 }
390
391 static void *oldnewmap_lookup_and_inc(OldNewMap *onm, const void *addr, bool increase_users)
392 {
393         int i;
394         
395         if (addr == NULL) return NULL;
396         
397         if (onm->lasthit < onm->nentries-1) {
398                 OldNew *entry = &onm->entries[++onm->lasthit];
399                 
400                 if (entry->old == addr) {
401                         if (increase_users)
402                                 entry->nr++;
403                         return entry->newp;
404                 }
405         }
406         
407         i = oldnewmap_lookup_entry_full(onm, addr, onm->lasthit);
408         if (i != -1) {
409                 OldNew *entry = &onm->entries[i];
410                 BLI_assert(entry->old == addr);
411                 onm->lasthit = i;
412                 if (increase_users)
413                         entry->nr++;
414                 return entry->newp;
415         }
416         
417         return NULL;
418 }
419
420 /* for libdata, nr has ID code, no increment */
421 static void *oldnewmap_liblookup(OldNewMap *onm, const void *addr, const void *lib)
422 {
423         if (addr == NULL) {
424                 return NULL;
425         }
426
427         /* lasthit works fine for non-libdata, linking there is done in same sequence as writing */
428         if (onm->sorted) {
429                 const OldNew entry_s = {.old = addr};
430                 OldNew *entry = bsearch(&entry_s, onm->entries, onm->nentries, sizeof(OldNew), verg_oldnewmap);
431                 if (entry) {
432                         ID *id = entry->newp;
433
434                         if (id && (!lib || id->lib)) {
435                                 return id;
436                         }
437                 }
438         }
439         else {
440                 /* note, this can be a bottle neck when loading some files */
441                 const int i = oldnewmap_lookup_entry_full(onm, addr, -1);
442                 if (i != -1) {
443                         OldNew *entry = &onm->entries[i];
444                         ID *id = entry->newp;
445                         BLI_assert(entry->old == addr);
446                         if (id && (!lib || id->lib)) {
447                                 return id;
448                         }
449                 }
450         }
451
452         return NULL;
453 }
454
455 static void oldnewmap_free_unused(OldNewMap *onm) 
456 {
457         int i;
458
459         for (i = 0; i < onm->nentries; i++) {
460                 OldNew *entry = &onm->entries[i];
461                 if (entry->nr == 0) {
462                         MEM_freeN(entry->newp);
463                         entry->newp = NULL;
464                 }
465         }
466 }
467
468 static void oldnewmap_clear(OldNewMap *onm) 
469 {
470         onm->nentries = 0;
471         onm->lasthit = 0;
472 }
473
474 static void oldnewmap_free(OldNewMap *onm) 
475 {
476         MEM_freeN(onm->entries);
477         MEM_freeN(onm);
478 }
479
480 /***/
481
482 static void read_libraries(FileData *basefd, ListBase *mainlist);
483
484 /* ************ help functions ***************** */
485
486 static void add_main_to_main(Main *mainvar, Main *from)
487 {
488         ListBase *lbarray[MAX_LIBARRAY], *fromarray[MAX_LIBARRAY];
489         int a;
490         
491         set_listbasepointers(mainvar, lbarray);
492         a = set_listbasepointers(from, fromarray);
493         while (a--) {
494                 BLI_movelisttolist(lbarray[a], fromarray[a]);
495         }
496 }
497
498 void blo_join_main(ListBase *mainlist)
499 {
500         Main *tojoin, *mainl;
501         
502         mainl = mainlist->first;
503         while ((tojoin = mainl->next)) {
504                 add_main_to_main(mainl, tojoin);
505                 BLI_remlink(mainlist, tojoin);
506                 BKE_main_free(tojoin);
507         }
508 }
509
510 static void split_libdata(ListBase *lb_src, Main **lib_main_array, const unsigned int lib_main_array_len)
511 {
512         for (ID *id = lb_src->first, *idnext; id; id = idnext) {
513                 idnext = id->next;
514
515                 if (id->lib) {
516                         if (((unsigned int)id->lib->temp_index < lib_main_array_len) &&
517                             /* this check should never fail, just incase 'id->lib' is a dangling pointer. */
518                             (lib_main_array[id->lib->temp_index]->curlib == id->lib))
519                         {
520                                 Main *mainvar = lib_main_array[id->lib->temp_index];
521                                 ListBase *lb_dst = which_libbase(mainvar, GS(id->name));
522                                 BLI_remlink(lb_src, id);
523                                 BLI_addtail(lb_dst, id);
524                         }
525                         else {
526                                 printf("%s: invalid library for '%s'\n", __func__, id->name);
527                                 BLI_assert(0);
528                         }
529                 }
530         }
531 }
532
533 void blo_split_main(ListBase *mainlist, Main *main)
534 {
535         mainlist->first = mainlist->last = main;
536         main->next = NULL;
537         
538         if (BLI_listbase_is_empty(&main->library))
539                 return;
540         
541         /* (Library.temp_index -> Main), lookup table */
542         const unsigned int lib_main_array_len = BLI_listbase_count(&main->library);
543         Main             **lib_main_array     = MEM_mallocN(lib_main_array_len * sizeof(*lib_main_array), __func__);
544
545         int i = 0;
546         for (Library *lib = main->library.first; lib; lib = lib->id.next, i++) {
547                 Main *libmain = BKE_main_new();
548                 libmain->curlib = lib;
549                 libmain->versionfile = lib->versionfile;
550                 libmain->subversionfile = lib->subversionfile;
551                 BLI_addtail(mainlist, libmain);
552                 lib->temp_index = i;
553                 lib_main_array[i] = libmain;
554         }
555         
556         ListBase *lbarray[MAX_LIBARRAY];
557         i = set_listbasepointers(main, lbarray);
558         while (i--) {
559                 split_libdata(lbarray[i], lib_main_array, lib_main_array_len);
560         }
561
562         MEM_freeN(lib_main_array);
563 }
564
565 static void read_file_version(FileData *fd, Main *main)
566 {
567         BHead *bhead;
568         
569         for (bhead= blo_firstbhead(fd); bhead; bhead= blo_nextbhead(fd, bhead)) {
570                 if (bhead->code == GLOB) {
571                         FileGlobal *fg= read_struct(fd, bhead, "Global");
572                         if (fg) {
573                                 main->subversionfile= fg->subversion;
574                                 main->minversionfile= fg->minversion;
575                                 main->minsubversionfile= fg->minsubversion;
576                                 MEM_freeN(fg);
577                         }
578                         else if (bhead->code == ENDB)
579                                 break;
580                 }
581         }
582         if (main->curlib) {
583                 main->curlib->versionfile = main->versionfile;
584                 main->curlib->subversionfile = main->subversionfile;
585         }
586 }
587
588 #ifdef USE_GHASH_BHEAD
589 static void read_file_bhead_idname_map_create(FileData *fd)
590 {
591         BHead *bhead;
592
593         /* dummy values */
594         bool is_link = false;
595         int code_prev = ENDB;
596         unsigned int reserve = 0;
597
598         for (bhead = blo_firstbhead(fd); bhead; bhead = blo_nextbhead(fd, bhead)) {
599                 if (code_prev != bhead->code) {
600                         code_prev = bhead->code;
601                         is_link = BKE_idcode_is_valid(code_prev) ? BKE_idcode_is_linkable(code_prev) : false;
602                 }
603
604                 if (is_link) {
605                         reserve += 1;
606                 }
607         }
608
609         BLI_assert(fd->bhead_idname_hash == NULL);
610
611         fd->bhead_idname_hash = BLI_ghash_str_new_ex(__func__, reserve);
612
613         for (bhead = blo_firstbhead(fd); bhead; bhead = blo_nextbhead(fd, bhead)) {
614                 if (code_prev != bhead->code) {
615                         code_prev = bhead->code;
616                         is_link = BKE_idcode_is_valid(code_prev) ? BKE_idcode_is_linkable(code_prev) : false;
617                 }
618
619                 if (is_link) {
620                         BLI_ghash_insert(fd->bhead_idname_hash, (void *)bhead_id_name(fd, bhead), bhead);
621                 }
622         }
623 }
624 #endif
625
626
627 static Main *blo_find_main(FileData *fd, const char *filepath, const char *relabase)
628 {
629         ListBase *mainlist = fd->mainlist;
630         Main *m;
631         Library *lib;
632         char name1[FILE_MAX];
633         
634         BLI_strncpy(name1, filepath, sizeof(name1));
635         BLI_cleanup_path(relabase, name1);
636         
637 //      printf("blo_find_main: relabase  %s\n", relabase);
638 //      printf("blo_find_main: original in  %s\n", filepath);
639 //      printf("blo_find_main: converted to %s\n", name1);
640         
641         for (m = mainlist->first; m; m = m->next) {
642                 const char *libname = (m->curlib) ? m->curlib->filepath : m->name;
643                 
644                 if (BLI_path_cmp(name1, libname) == 0) {
645                         if (G.debug & G_DEBUG) printf("blo_find_main: found library %s\n", libname);
646                         return m;
647                 }
648         }
649         
650         m = BKE_main_new();
651         BLI_addtail(mainlist, m);
652         
653         /* Add library datablock itself to 'main' Main, since libraries are **never** linked data.
654          * Fixes bug where you could end with all ID_LI datablocks having the same name... */
655         lib = BKE_libblock_alloc(mainlist->first, ID_LI, "Lib", 0);
656         lib->id.us = ID_FAKE_USERS(lib);  /* Important, consistency with main ID reading code from read_libblock(). */
657         BLI_strncpy(lib->name, filepath, sizeof(lib->name));
658         BLI_strncpy(lib->filepath, name1, sizeof(lib->filepath));
659         
660         m->curlib = lib;
661         
662         read_file_version(fd, m);
663         
664         if (G.debug & G_DEBUG) printf("blo_find_main: added new lib %s\n", filepath);
665         return m;
666 }
667
668
669 /* ************ FILE PARSING ****************** */
670
671 static void switch_endian_bh4(BHead4 *bhead)
672 {
673         /* the ID_.. codes */
674         if ((bhead->code & 0xFFFF)==0) bhead->code >>= 16;
675         
676         if (bhead->code != ENDB) {
677                 BLI_endian_switch_int32(&bhead->len);
678                 BLI_endian_switch_int32(&bhead->SDNAnr);
679                 BLI_endian_switch_int32(&bhead->nr);
680         }
681 }
682
683 static void switch_endian_bh8(BHead8 *bhead)
684 {
685         /* the ID_.. codes */
686         if ((bhead->code & 0xFFFF)==0) bhead->code >>= 16;
687         
688         if (bhead->code != ENDB) {
689                 BLI_endian_switch_int32(&bhead->len);
690                 BLI_endian_switch_int32(&bhead->SDNAnr);
691                 BLI_endian_switch_int32(&bhead->nr);
692         }
693 }
694
695 static void bh4_from_bh8(BHead *bhead, BHead8 *bhead8, int do_endian_swap)
696 {
697         BHead4 *bhead4 = (BHead4 *) bhead;
698         int64_t old;
699
700         bhead4->code = bhead8->code;
701         bhead4->len = bhead8->len;
702
703         if (bhead4->code != ENDB) {
704                 /* perform a endian swap on 64bit pointers, otherwise the pointer might map to zero
705                  * 0x0000000000000000000012345678 would become 0x12345678000000000000000000000000
706                  */
707                 if (do_endian_swap) {
708                         BLI_endian_switch_int64(&bhead8->old);
709                 }
710                 
711                 /* this patch is to avoid a long long being read from not-eight aligned positions
712                  * is necessary on any modern 64bit architecture) */
713                 memcpy(&old, &bhead8->old, 8);
714                 bhead4->old = (int) (old >> 3);
715                 
716                 bhead4->SDNAnr = bhead8->SDNAnr;
717                 bhead4->nr = bhead8->nr;
718         }
719 }
720
721 static void bh8_from_bh4(BHead *bhead, BHead4 *bhead4)
722 {
723         BHead8 *bhead8 = (BHead8 *) bhead;
724         
725         bhead8->code = bhead4->code;
726         bhead8->len = bhead4->len;
727         
728         if (bhead8->code != ENDB) {
729                 bhead8->old = bhead4->old;
730                 bhead8->SDNAnr = bhead4->SDNAnr;
731                 bhead8->nr= bhead4->nr;
732         }
733 }
734
735 static BHeadN *get_bhead(FileData *fd)
736 {
737         BHeadN *new_bhead = NULL;
738         int readsize;
739         
740         if (fd) {
741                 if (!fd->eof) {
742                         /* initializing to zero isn't strictly needed but shuts valgrind up
743                          * since uninitialized memory gets compared */
744                         BHead8 bhead8 = {0};
745                         BHead4 bhead4 = {0};
746                         BHead  bhead = {0};
747                         
748                         /* First read the bhead structure.
749                          * Depending on the platform the file was written on this can
750                          * be a big or little endian BHead4 or BHead8 structure.
751                          *
752                          * As usual 'ENDB' (the last *partial* bhead of the file)
753                          * needs some special handling. We don't want to EOF just yet.
754                          */
755                         if (fd->flags & FD_FLAGS_FILE_POINTSIZE_IS_4) {
756                                 bhead4.code = DATA;
757                                 readsize = fd->read(fd, &bhead4, sizeof(bhead4));
758                                 
759                                 if (readsize == sizeof(bhead4) || bhead4.code == ENDB) {
760                                         if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
761                                                 switch_endian_bh4(&bhead4);
762                                         }
763                                         
764                                         if (fd->flags & FD_FLAGS_POINTSIZE_DIFFERS) {
765                                                 bh8_from_bh4(&bhead, &bhead4);
766                                         }
767                                         else {
768                                                 memcpy(&bhead, &bhead4, sizeof(bhead));
769                                         }
770                                 }
771                                 else {
772                                         fd->eof = 1;
773                                         bhead.len= 0;
774                                 }
775                         }
776                         else {
777                                 bhead8.code = DATA;
778                                 readsize = fd->read(fd, &bhead8, sizeof(bhead8));
779                                 
780                                 if (readsize == sizeof(bhead8) || bhead8.code == ENDB) {
781                                         if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
782                                                 switch_endian_bh8(&bhead8);
783                                         }
784                                         
785                                         if (fd->flags & FD_FLAGS_POINTSIZE_DIFFERS) {
786                                                 bh4_from_bh8(&bhead, &bhead8, (fd->flags & FD_FLAGS_SWITCH_ENDIAN));
787                                         }
788                                         else {
789                                                 memcpy(&bhead, &bhead8, sizeof(bhead));
790                                         }
791                                 }
792                                 else {
793                                         fd->eof = 1;
794                                         bhead.len= 0;
795                                 }
796                         }
797                         
798                         /* make sure people are not trying to pass bad blend files */
799                         if (bhead.len < 0) fd->eof = 1;
800                         
801                         /* bhead now contains the (converted) bhead structure. Now read
802                          * the associated data and put everything in a BHeadN (creative naming !)
803                          */
804                         if (!fd->eof) {
805                                 new_bhead = MEM_mallocN(sizeof(BHeadN) + bhead.len, "new_bhead");
806                                 if (new_bhead) {
807                                         new_bhead->next = new_bhead->prev = NULL;
808                                         new_bhead->bhead = bhead;
809                                         
810                                         readsize = fd->read(fd, new_bhead + 1, bhead.len);
811                                         
812                                         if (readsize != bhead.len) {
813                                                 fd->eof = 1;
814                                                 MEM_freeN(new_bhead);
815                                                 new_bhead = NULL;
816                                         }
817                                 }
818                                 else {
819                                         fd->eof = 1;
820                                 }
821                         }
822                 }
823         }
824
825         /* We've read a new block. Now add it to the list
826          * of blocks.
827          */
828         if (new_bhead) {
829                 BLI_addtail(&fd->listbase, new_bhead);
830         }
831         
832         return(new_bhead);
833 }
834
835 BHead *blo_firstbhead(FileData *fd)
836 {
837         BHeadN *new_bhead;
838         BHead *bhead = NULL;
839         
840         /* Rewind the file
841          * Read in a new block if necessary
842          */
843         new_bhead = fd->listbase.first;
844         if (new_bhead == NULL) {
845                 new_bhead = get_bhead(fd);
846         }
847         
848         if (new_bhead) {
849                 bhead = &new_bhead->bhead;
850         }
851         
852         return(bhead);
853 }
854
855 BHead *blo_prevbhead(FileData *UNUSED(fd), BHead *thisblock)
856 {
857         BHeadN *bheadn = (BHeadN *)POINTER_OFFSET(thisblock, -offsetof(BHeadN, bhead));
858         BHeadN *prev = bheadn->prev;
859         
860         return (prev) ? &prev->bhead : NULL;
861 }
862
863 BHead *blo_nextbhead(FileData *fd, BHead *thisblock)
864 {
865         BHeadN *new_bhead = NULL;
866         BHead *bhead = NULL;
867         
868         if (thisblock) {
869                 /* bhead is actually a sub part of BHeadN
870                  * We calculate the BHeadN pointer from the BHead pointer below */
871                 new_bhead = (BHeadN *)POINTER_OFFSET(thisblock, -offsetof(BHeadN, bhead));
872                 
873                 /* get the next BHeadN. If it doesn't exist we read in the next one */
874                 new_bhead = new_bhead->next;
875                 if (new_bhead == NULL) {
876                         new_bhead = get_bhead(fd);
877                 }
878         }
879         
880         if (new_bhead) {
881                 /* here we do the reverse:
882                  * go from the BHeadN pointer to the BHead pointer */
883                 bhead = &new_bhead->bhead;
884         }
885         
886         return(bhead);
887 }
888
889 /* Warning! Caller's responsibility to ensure given bhead **is** and ID one! */
890 const char *bhead_id_name(const FileData *fd, const BHead *bhead)
891 {
892         return (const char *)POINTER_OFFSET(bhead, sizeof(*bhead) + fd->id_name_offs);
893 }
894
895 static void decode_blender_header(FileData *fd)
896 {
897         char header[SIZEOFBLENDERHEADER], num[4];
898         int readsize;
899         
900         /* read in the header data */
901         readsize = fd->read(fd, header, sizeof(header));
902         
903         if (readsize == sizeof(header)) {
904                 if (STREQLEN(header, "BLENDER", 7)) {
905                         fd->flags |= FD_FLAGS_FILE_OK;
906                         
907                         /* what size are pointers in the file ? */
908                         if (header[7]=='_') {
909                                 fd->flags |= FD_FLAGS_FILE_POINTSIZE_IS_4;
910                                 if (sizeof(void *) != 4) {
911                                         fd->flags |= FD_FLAGS_POINTSIZE_DIFFERS;
912                                 }
913                         }
914                         else {
915                                 if (sizeof(void *) != 8) {
916                                         fd->flags |= FD_FLAGS_POINTSIZE_DIFFERS;
917                                 }
918                         }
919                         
920                         /* is the file saved in a different endian
921                          * than we need ?
922                          */
923                         if (((header[8] == 'v') ? L_ENDIAN : B_ENDIAN) != ENDIAN_ORDER) {
924                                 fd->flags |= FD_FLAGS_SWITCH_ENDIAN;
925                         }
926                         
927                         /* get the version number */
928                         memcpy(num, header + 9, 3);
929                         num[3] = 0;
930                         fd->fileversion = atoi(num);
931                 }
932         }
933 }
934
935 /**
936  * \return Success if the file is read correctly, else set \a r_error_message.
937  */
938 static bool read_file_dna(FileData *fd, const char **r_error_message)
939 {
940         BHead *bhead;
941         
942         for (bhead = blo_firstbhead(fd); bhead; bhead = blo_nextbhead(fd, bhead)) {
943                 if (bhead->code == DNA1) {
944                         const bool do_endian_swap = (fd->flags & FD_FLAGS_SWITCH_ENDIAN) != 0;
945                         
946                         fd->filesdna = DNA_sdna_from_data(&bhead[1], bhead->len, do_endian_swap, true, r_error_message);
947                         if (fd->filesdna) {
948                                 fd->compflags = DNA_struct_get_compareflags(fd->filesdna, fd->memsdna);
949                                 /* used to retrieve ID names from (bhead+1) */
950                                 fd->id_name_offs = DNA_elem_offset(fd->filesdna, "ID", "char", "name[]");
951
952                                 return true;
953                         }
954                         else {
955                                 return false;
956                         }
957                         
958                 }
959                 else if (bhead->code == ENDB)
960                         break;
961         }
962         
963         *r_error_message = "Missing DNA block";
964         return false;
965 }
966
967 static int *read_file_thumbnail(FileData *fd)
968 {
969         BHead *bhead;
970         int *blend_thumb = NULL;
971
972         for (bhead = blo_firstbhead(fd); bhead; bhead = blo_nextbhead(fd, bhead)) {
973                 if (bhead->code == TEST) {
974                         const bool do_endian_swap = (fd->flags & FD_FLAGS_SWITCH_ENDIAN) != 0;
975                         int *data = (int *)(bhead + 1);
976
977                         if (bhead->len < (2 * sizeof(int))) {
978                                 break;
979                         }
980
981                         if (do_endian_swap) {
982                                 BLI_endian_switch_int32(&data[0]);
983                                 BLI_endian_switch_int32(&data[1]);
984                         }
985
986                         if (bhead->len < BLEN_THUMB_MEMSIZE_FILE(data[0], data[1])) {
987                                 break;
988                         }
989
990                         blend_thumb = data;
991                         break;
992                 }
993                 else if (bhead->code != REND) {
994                         /* Thumbnail is stored in TEST immediately after first REND... */
995                         break;
996                 }
997         }
998
999         return blend_thumb;
1000 }
1001
1002 static int fd_read_from_file(FileData *filedata, void *buffer, unsigned int size)
1003 {
1004         int readsize = read(filedata->filedes, buffer, size);
1005         
1006         if (readsize < 0) {
1007                 readsize = EOF;
1008         }
1009         else {
1010                 filedata->seek += readsize;
1011         }
1012         
1013         return readsize;
1014 }
1015
1016 static int fd_read_gzip_from_file(FileData *filedata, void *buffer, unsigned int size)
1017 {
1018         int readsize = gzread(filedata->gzfiledes, buffer, size);
1019         
1020         if (readsize < 0) {
1021                 readsize = EOF;
1022         }
1023         else {
1024                 filedata->seek += readsize;
1025         }
1026         
1027         return (readsize);
1028 }
1029
1030 static int fd_read_from_memory(FileData *filedata, void *buffer, unsigned int size)
1031 {
1032         /* don't read more bytes then there are available in the buffer */
1033         int readsize = (int)MIN2(size, (unsigned int)(filedata->buffersize - filedata->seek));
1034         
1035         memcpy(buffer, filedata->buffer + filedata->seek, readsize);
1036         filedata->seek += readsize;
1037         
1038         return (readsize);
1039 }
1040
1041 static int fd_read_from_memfile(FileData *filedata, void *buffer, unsigned int size)
1042 {
1043         static unsigned int seek = (1<<30);     /* the current position */
1044         static unsigned int offset = 0;         /* size of previous chunks */
1045         static MemFileChunk *chunk = NULL;
1046         unsigned int chunkoffset, readsize, totread;
1047         
1048         if (size == 0) return 0;
1049         
1050         if (seek != (unsigned int)filedata->seek) {
1051                 chunk = filedata->memfile->chunks.first;
1052                 seek = 0;
1053                 
1054                 while (chunk) {
1055                         if (seek + chunk->size > (unsigned) filedata->seek) break;
1056                         seek += chunk->size;
1057                         chunk = chunk->next;
1058                 }
1059                 offset = seek;
1060                 seek = filedata->seek;
1061         }
1062         
1063         if (chunk) {
1064                 totread = 0;
1065                 
1066                 do {
1067                         /* first check if it's on the end if current chunk */
1068                         if (seek-offset == chunk->size) {
1069                                 offset += chunk->size;
1070                                 chunk = chunk->next;
1071                         }
1072                         
1073                         /* debug, should never happen */
1074                         if (chunk == NULL) {
1075                                 printf("illegal read, chunk zero\n");
1076                                 return 0;
1077                         }
1078                         
1079                         chunkoffset = seek-offset;
1080                         readsize = size-totread;
1081                         
1082                         /* data can be spread over multiple chunks, so clamp size
1083                          * to within this chunk, and then it will read further in
1084                          * the next chunk */
1085                         if (chunkoffset+readsize > chunk->size)
1086                                 readsize= chunk->size-chunkoffset;
1087                         
1088                         memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize);
1089                         totread += readsize;
1090                         filedata->seek += readsize;
1091                         seek += readsize;
1092                 } while (totread < size);
1093                 
1094                 return totread;
1095         }
1096         
1097         return 0;
1098 }
1099
1100 static FileData *filedata_new(void)
1101 {
1102         FileData *fd = MEM_callocN(sizeof(FileData), "FileData");
1103         
1104         fd->filedes = -1;
1105         fd->gzfiledes = NULL;
1106
1107         fd->memsdna = DNA_sdna_current_get();
1108
1109         fd->datamap = oldnewmap_new();
1110         fd->globmap = oldnewmap_new();
1111         fd->libmap = oldnewmap_new();
1112         
1113         return fd;
1114 }
1115
1116 static FileData *blo_decode_and_check(FileData *fd, ReportList *reports)
1117 {
1118         decode_blender_header(fd);
1119         
1120         if (fd->flags & FD_FLAGS_FILE_OK) {
1121                 const char *error_message = NULL;
1122                 if (read_file_dna(fd, &error_message) == false) {
1123                         BKE_reportf(reports, RPT_ERROR,
1124                                     "Failed to read blend file '%s': %s",
1125                                     fd->relabase, error_message);
1126                         blo_freefiledata(fd);
1127                         fd = NULL;
1128                 }
1129         }
1130         else {
1131                 BKE_reportf(reports, RPT_ERROR, "Failed to read blend file '%s', not a blend file", fd->relabase);
1132                 blo_freefiledata(fd);
1133                 fd = NULL;
1134         }
1135         
1136         return fd;
1137 }
1138
1139 /* cannot be called with relative paths anymore! */
1140 /* on each new library added, it now checks for the current FileData and expands relativeness */
1141 FileData *blo_openblenderfile(const char *filepath, ReportList *reports)
1142 {
1143         gzFile gzfile;
1144         errno = 0;
1145         gzfile = BLI_gzopen(filepath, "rb");
1146         
1147         if (gzfile == (gzFile)Z_NULL) {
1148                 BKE_reportf(reports, RPT_WARNING, "Unable to open '%s': %s",
1149                             filepath, errno ? strerror(errno) : TIP_("unknown error reading file"));
1150                 return NULL;
1151         }
1152         else {
1153                 FileData *fd = filedata_new();
1154                 fd->gzfiledes = gzfile;
1155                 fd->read = fd_read_gzip_from_file;
1156                 
1157                 /* needed for library_append and read_libraries */
1158                 BLI_strncpy(fd->relabase, filepath, sizeof(fd->relabase));
1159                 
1160                 return blo_decode_and_check(fd, reports);
1161         }
1162 }
1163
1164 /**
1165  * Same as blo_openblenderfile(), but does not reads DNA data, only header. Use it for light access
1166  * (e.g. thumbnail reading).
1167  */
1168 static FileData *blo_openblenderfile_minimal(const char *filepath)
1169 {
1170         gzFile gzfile;
1171         errno = 0;
1172         gzfile = BLI_gzopen(filepath, "rb");
1173
1174         if (gzfile != (gzFile)Z_NULL) {
1175                 FileData *fd = filedata_new();
1176                 fd->gzfiledes = gzfile;
1177                 fd->read = fd_read_gzip_from_file;
1178
1179                 decode_blender_header(fd);
1180
1181                 if (fd->flags & FD_FLAGS_FILE_OK) {
1182                         return fd;
1183                 }
1184
1185                 blo_freefiledata(fd);
1186         }
1187
1188         return NULL;
1189 }
1190
1191 static int fd_read_gzip_from_memory(FileData *filedata, void *buffer, unsigned int size)
1192 {
1193         int err;
1194
1195         filedata->strm.next_out = (Bytef *) buffer;
1196         filedata->strm.avail_out = size;
1197
1198         // Inflate another chunk.
1199         err = inflate (&filedata->strm, Z_SYNC_FLUSH);
1200
1201         if (err == Z_STREAM_END) {
1202                 return 0;
1203         }
1204         else if (err != Z_OK) {
1205                 printf("fd_read_gzip_from_memory: zlib error\n");
1206                 return 0;
1207         }
1208
1209         filedata->seek += size;
1210
1211         return (size);
1212 }
1213
1214 static int fd_read_gzip_from_memory_init(FileData *fd)
1215 {
1216
1217         fd->strm.next_in = (Bytef *) fd->buffer;
1218         fd->strm.avail_in = fd->buffersize;
1219         fd->strm.total_out = 0;
1220         fd->strm.zalloc = Z_NULL;
1221         fd->strm.zfree = Z_NULL;
1222         
1223         if (inflateInit2(&fd->strm, (16+MAX_WBITS)) != Z_OK)
1224                 return 0;
1225
1226         fd->read = fd_read_gzip_from_memory;
1227         
1228         return 1;
1229 }
1230
1231 FileData *blo_openblendermemory(const void *mem, int memsize, ReportList *reports)
1232 {
1233         if (!mem || memsize<SIZEOFBLENDERHEADER) {
1234                 BKE_report(reports, RPT_WARNING, (mem) ? TIP_("Unable to read"): TIP_("Unable to open"));
1235                 return NULL;
1236         }
1237         else {
1238                 FileData *fd = filedata_new();
1239                 const char *cp = mem;
1240                 
1241                 fd->buffer = mem;
1242                 fd->buffersize = memsize;
1243                 
1244                 /* test if gzip */
1245                 if (cp[0] == 0x1f && cp[1] == 0x8b) {
1246                         if (0 == fd_read_gzip_from_memory_init(fd)) {
1247                                 blo_freefiledata(fd);
1248                                 return NULL;
1249                         }
1250                 }
1251                 else
1252                         fd->read = fd_read_from_memory;
1253                         
1254                 fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
1255
1256                 return blo_decode_and_check(fd, reports);
1257         }
1258 }
1259
1260 FileData *blo_openblendermemfile(MemFile *memfile, ReportList *reports)
1261 {
1262         if (!memfile) {
1263                 BKE_report(reports, RPT_WARNING, "Unable to open blend <memory>");
1264                 return NULL;
1265         }
1266         else {
1267                 FileData *fd = filedata_new();
1268                 fd->memfile = memfile;
1269                 
1270                 fd->read = fd_read_from_memfile;
1271                 fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
1272                 
1273                 return blo_decode_and_check(fd, reports);
1274         }
1275 }
1276
1277
1278 void blo_freefiledata(FileData *fd)
1279 {
1280         if (fd) {
1281                 if (fd->filedes != -1) {
1282                         close(fd->filedes);
1283                 }
1284                 
1285                 if (fd->gzfiledes != NULL) {
1286                         gzclose(fd->gzfiledes);
1287                 }
1288                 
1289                 if (fd->strm.next_in) {
1290                         if (inflateEnd(&fd->strm) != Z_OK) {
1291                                 printf("close gzip stream error\n");
1292                         }
1293                 }
1294                 
1295                 if (fd->buffer && !(fd->flags & FD_FLAGS_NOT_MY_BUFFER)) {
1296                         MEM_freeN((void *)fd->buffer);
1297                         fd->buffer = NULL;
1298                 }
1299                 
1300                 // Free all BHeadN data blocks
1301                 BLI_freelistN(&fd->listbase);
1302
1303                 if (fd->filesdna)
1304                         DNA_sdna_free(fd->filesdna);
1305                 if (fd->compflags)
1306                         MEM_freeN((void *)fd->compflags);
1307                 
1308                 if (fd->datamap)
1309                         oldnewmap_free(fd->datamap);
1310                 if (fd->globmap)
1311                         oldnewmap_free(fd->globmap);
1312                 if (fd->imamap)
1313                         oldnewmap_free(fd->imamap);
1314                 if (fd->movieclipmap)
1315                         oldnewmap_free(fd->movieclipmap);
1316                 if (fd->soundmap)
1317                         oldnewmap_free(fd->soundmap);
1318                 if (fd->packedmap)
1319                         oldnewmap_free(fd->packedmap);
1320                 if (fd->libmap && !(fd->flags & FD_FLAGS_NOT_MY_LIBMAP))
1321                         oldnewmap_free(fd->libmap);
1322                 if (fd->bheadmap)
1323                         MEM_freeN(fd->bheadmap);
1324                 
1325 #ifdef USE_GHASH_BHEAD
1326                 if (fd->bhead_idname_hash) {
1327                         BLI_ghash_free(fd->bhead_idname_hash, NULL, NULL);
1328                 }
1329 #endif
1330
1331                 MEM_freeN(fd);
1332         }
1333 }
1334
1335 /* ************ DIV ****************** */
1336
1337 /**
1338  * Check whether given path ends with a blend file compatible extension (.blend, .ble or .blend.gz).
1339  *
1340  * \param str The path to check.
1341  * \return true is this path ends with a blender file extension.
1342  */
1343 bool BLO_has_bfile_extension(const char *str)
1344 {
1345         const char *ext_test[4] = {".blend", ".ble", ".blend.gz", NULL};
1346         return BLI_testextensie_array(str, ext_test);
1347 }
1348
1349 /**
1350  * Try to explode given path into its 'library components' (i.e. a .blend file, id type/group, and datablock itself).
1351  *
1352  * \param path the full path to explode.
1353  * \param r_dir the string that'll contain path up to blend file itself ('library' path).
1354  *              WARNING! Must be FILE_MAX_LIBEXTRA long (it also stores group and name strings)!
1355  * \param r_group the string that'll contain 'group' part of the path, if any. May be NULL.
1356  * \param r_name the string that'll contain data's name part of the path, if any. May be NULL.
1357  * \return true if path contains a blend file.
1358  */
1359 bool BLO_library_path_explode(const char *path, char *r_dir, char **r_group, char **r_name)
1360 {
1361         /* We might get some data names with slashes, so we have to go up in path until we find blend file itself,
1362          * then we now next path item is group, and everything else is data name. */
1363         char *slash = NULL, *prev_slash = NULL, c = '\0';
1364
1365         r_dir[0] = '\0';
1366         if (r_group) {
1367                 *r_group = NULL;
1368         }
1369         if (r_name) {
1370                 *r_name = NULL;
1371         }
1372
1373         /* if path leads to an existing directory, we can be sure we're not (in) a library */
1374         if (BLI_is_dir(path)) {
1375                 return false;
1376         }
1377
1378         strcpy(r_dir, path);
1379
1380         while ((slash = (char *)BLI_last_slash(r_dir))) {
1381                 char tc = *slash;
1382                 *slash = '\0';
1383                 if (BLO_has_bfile_extension(r_dir) && BLI_is_file(r_dir)) {
1384                         break;
1385                 }
1386
1387                 if (prev_slash) {
1388                         *prev_slash = c;
1389                 }
1390                 prev_slash = slash;
1391                 c = tc;
1392         }
1393
1394         if (!slash) {
1395                 return false;
1396         }
1397
1398         if (slash[1] != '\0') {
1399                 BLI_assert(strlen(slash + 1) < BLO_GROUP_MAX);
1400                 if (r_group) {
1401                         *r_group = slash + 1;
1402                 }
1403         }
1404
1405         if (prev_slash && (prev_slash[1] != '\0')) {
1406                 BLI_assert(strlen(prev_slash + 1) < MAX_ID_NAME - 2);
1407                 if (r_name) {
1408                         *r_name = prev_slash + 1;
1409                 }
1410         }
1411
1412         return true;
1413 }
1414
1415 /**
1416  * Does a very light reading of given .blend file to extract its stored thumbnail.
1417  *
1418  * \param filepath The path of the file to extract thumbnail from.
1419  * \return The raw thumbnail
1420  *         (MEM-allocated, as stored in file, use BKE_main_thumbnail_to_imbuf() to convert it to ImBuf image).
1421  */
1422 BlendThumbnail *BLO_thumbnail_from_file(const char *filepath)
1423 {
1424         FileData *fd;
1425         BlendThumbnail *data;
1426         int *fd_data;
1427
1428         fd = blo_openblenderfile_minimal(filepath);
1429         fd_data = fd ? read_file_thumbnail(fd) : NULL;
1430
1431         if (fd_data) {
1432                 const size_t sz = BLEN_THUMB_MEMSIZE(fd_data[0], fd_data[1]);
1433                 data = MEM_mallocN(sz, __func__);
1434
1435                 BLI_assert((sz - sizeof(*data)) == (BLEN_THUMB_MEMSIZE_FILE(fd_data[0], fd_data[1]) - (sizeof(*fd_data) * 2)));
1436                 data->width = fd_data[0];
1437                 data->height = fd_data[1];
1438                 memcpy(data->rect, &fd_data[2], sz - sizeof(*data));
1439         }
1440         else {
1441                 data = NULL;
1442         }
1443
1444         blo_freefiledata(fd);
1445
1446         return data;
1447 }
1448
1449 /* ************** OLD POINTERS ******************* */
1450
1451 static void *newdataadr(FileData *fd, const void *adr)          /* only direct databocks */
1452 {
1453         return oldnewmap_lookup_and_inc(fd->datamap, adr, true);
1454 }
1455
1456 /* This is a special version of newdataadr() which allows us to keep lasthit of
1457  * map unchanged. In certain cases this makes file loading time significantly
1458  * faster.
1459  *
1460  * Use this function in cases like restoring pointer from one list element to
1461  * another list element, but keep lasthit value so we can continue restoring
1462  * pointers efficiently.
1463  *
1464  * Example of this could be found in direct_link_fcurves() which restores the
1465  * fcurve group pointer and keeps lasthit optimal for linking all further
1466  * fcurves.
1467  */
1468 static void *newdataadr_ex(FileData *fd, const void *adr, bool increase_lasthit)                /* only direct databocks */
1469 {
1470         if (increase_lasthit) {
1471                 return newdataadr(fd, adr);
1472         }
1473         else {
1474                 int lasthit = fd->datamap->lasthit;
1475                 void *newadr = newdataadr(fd, adr);
1476                 fd->datamap->lasthit = lasthit;
1477                 return newadr;
1478         }
1479 }
1480
1481 static void *newdataadr_no_us(FileData *fd, const void *adr)            /* only direct databocks */
1482 {
1483         return oldnewmap_lookup_and_inc(fd->datamap, adr, false);
1484 }
1485
1486 static void *newglobadr(FileData *fd, const void *adr)      /* direct datablocks with global linking */
1487 {
1488         return oldnewmap_lookup_and_inc(fd->globmap, adr, true);
1489 }
1490
1491 static void *newimaadr(FileData *fd, const void *adr)               /* used to restore image data after undo */
1492 {
1493         if (fd->imamap && adr)
1494                 return oldnewmap_lookup_and_inc(fd->imamap, adr, true);
1495         return NULL;
1496 }
1497
1498 static void *newmclipadr(FileData *fd, const void *adr)      /* used to restore movie clip data after undo */
1499 {
1500         if (fd->movieclipmap && adr)
1501                 return oldnewmap_lookup_and_inc(fd->movieclipmap, adr, true);
1502         return NULL;
1503 }
1504
1505 static void *newsoundadr(FileData *fd, const void *adr)      /* used to restore sound data after undo */
1506 {
1507         if (fd->soundmap && adr)
1508                 return oldnewmap_lookup_and_inc(fd->soundmap, adr, true);
1509         return NULL;
1510 }
1511
1512 static void *newpackedadr(FileData *fd, const void *adr)      /* used to restore packed data after undo */
1513 {
1514         if (fd->packedmap && adr)
1515                 return oldnewmap_lookup_and_inc(fd->packedmap, adr, true);
1516         
1517         return oldnewmap_lookup_and_inc(fd->datamap, adr, true);
1518 }
1519
1520
1521 static void *newlibadr(FileData *fd, const void *lib, const void *adr)          /* only lib data */
1522 {
1523         return oldnewmap_liblookup(fd->libmap, adr, lib);
1524 }
1525
1526 void *blo_do_versions_newlibadr(FileData *fd, const void *lib, const void *adr)         /* only lib data */
1527 {
1528         return newlibadr(fd, lib, adr);
1529 }
1530
1531 static void *newlibadr_us(FileData *fd, const void *lib, const void *adr)       /* increases user number */
1532 {
1533         ID *id = newlibadr(fd, lib, adr);
1534         
1535         id_us_plus_no_lib(id);
1536         
1537         return id;
1538 }
1539
1540 void *blo_do_versions_newlibadr_us(FileData *fd, const void *lib, const void *adr)      /* increases user number */
1541 {
1542         return newlibadr_us(fd, lib, adr);
1543 }
1544
1545 static void *newlibadr_real_us(FileData *fd, const void *lib, const void *adr)  /* ensures real user */
1546 {
1547         ID *id = newlibadr(fd, lib, adr);
1548
1549         id_us_ensure_real(id);
1550
1551         return id;
1552 }
1553
1554 static void change_idid_adr_fd(FileData *fd, const void *old, void *new)
1555 {
1556         int i;
1557         
1558         /* use a binary search if we have a sorted libmap, for now it's not needed. */
1559         BLI_assert(fd->libmap->sorted == false);
1560
1561         for (i = 0; i < fd->libmap->nentries; i++) {
1562                 OldNew *entry = &fd->libmap->entries[i];
1563                 
1564                 if (old==entry->newp && entry->nr==ID_ID) {
1565                         entry->newp = new;
1566                         if (new) entry->nr = GS( ((ID *)new)->name );
1567                 }
1568         }
1569 }
1570
1571 static void change_idid_adr(ListBase *mainlist, FileData *basefd, void *old, void *new)
1572 {
1573         Main *mainptr;
1574         
1575         for (mainptr = mainlist->first; mainptr; mainptr = mainptr->next) {
1576                 FileData *fd;
1577                 
1578                 if (mainptr->curlib)
1579                         fd = mainptr->curlib->filedata;
1580                 else
1581                         fd = basefd;
1582                 
1583                 if (fd) {
1584                         change_idid_adr_fd(fd, old, new);
1585                 }
1586         }
1587 }
1588
1589 /* lib linked proxy objects point to our local data, we need
1590  * to clear that pointer before reading the undo memfile since
1591  * the object might be removed, it is set again in reading
1592  * if the local object still exists */
1593 void blo_clear_proxy_pointers_from_lib(Main *oldmain)
1594 {
1595         Object *ob = oldmain->object.first;
1596         
1597         for (; ob; ob= ob->id.next) {
1598                 if (ob->id.lib)
1599                         ob->proxy_from = NULL;
1600         }
1601 }
1602
1603 void blo_make_image_pointer_map(FileData *fd, Main *oldmain)
1604 {
1605         Image *ima = oldmain->image.first;
1606         Scene *sce = oldmain->scene.first;
1607         int a;
1608         
1609         fd->imamap = oldnewmap_new();
1610         
1611         for (; ima; ima = ima->id.next) {
1612                 if (ima->cache)
1613                         oldnewmap_insert(fd->imamap, ima->cache, ima->cache, 0);
1614                 for (a = 0; a < TEXTARGET_COUNT; a++)
1615                         if (ima->gputexture[a])
1616                                 oldnewmap_insert(fd->imamap, ima->gputexture[a], ima->gputexture[a], 0);
1617                 if (ima->rr)
1618                         oldnewmap_insert(fd->imamap, ima->rr, ima->rr, 0);
1619                 for (a=0; a < IMA_MAX_RENDER_SLOT; a++)
1620                         if (ima->renders[a])
1621                                 oldnewmap_insert(fd->imamap, ima->renders[a], ima->renders[a], 0);
1622         }
1623         for (; sce; sce = sce->id.next) {
1624                 if (sce->nodetree && sce->nodetree->previews) {
1625                         bNodeInstanceHashIterator iter;
1626                         NODE_INSTANCE_HASH_ITER(iter, sce->nodetree->previews) {
1627                                 bNodePreview *preview = BKE_node_instance_hash_iterator_get_value(&iter);
1628                                 oldnewmap_insert(fd->imamap, preview, preview, 0);
1629                         }
1630                 }
1631         }
1632 }
1633
1634 /* set old main image ibufs to zero if it has been restored */
1635 /* this works because freeing old main only happens after this call */
1636 void blo_end_image_pointer_map(FileData *fd, Main *oldmain)
1637 {
1638         OldNew *entry = fd->imamap->entries;
1639         Image *ima = oldmain->image.first;
1640         Scene *sce = oldmain->scene.first;
1641         int i;
1642         
1643         /* used entries were restored, so we put them to zero */
1644         for (i = 0; i < fd->imamap->nentries; i++, entry++) {
1645                 if (entry->nr > 0)
1646                         entry->newp = NULL;
1647         }
1648         
1649         for (; ima; ima = ima->id.next) {
1650                 ima->cache = newimaadr(fd, ima->cache);
1651                 if (ima->cache == NULL) {
1652                         ima->tpageflag &= ~IMA_GLBIND_IS_DATA;
1653                         for (i = 0; i < TEXTARGET_COUNT; i++) {
1654                                 ima->bindcode[i] = 0;
1655                                 ima->gputexture[i] = NULL;
1656                         }
1657                         ima->rr = NULL;
1658                 }
1659                 for (i = 0; i < IMA_MAX_RENDER_SLOT; i++)
1660                         ima->renders[i] = newimaadr(fd, ima->renders[i]);
1661                 
1662                 for (i = 0; i < TEXTARGET_COUNT; i++)
1663                         ima->gputexture[i] = newimaadr(fd, ima->gputexture[i]);
1664                 ima->rr = newimaadr(fd, ima->rr);
1665         }
1666         for (; sce; sce = sce->id.next) {
1667                 if (sce->nodetree && sce->nodetree->previews) {
1668                         bNodeInstanceHash *new_previews = BKE_node_instance_hash_new("node previews");
1669                         bNodeInstanceHashIterator iter;
1670                         
1671                         /* reconstruct the preview hash, only using remaining pointers */
1672                         NODE_INSTANCE_HASH_ITER(iter, sce->nodetree->previews) {
1673                                 bNodePreview *preview = BKE_node_instance_hash_iterator_get_value(&iter);
1674                                 if (preview) {
1675                                         bNodePreview *new_preview = newimaadr(fd, preview);
1676                                         if (new_preview) {
1677                                                 bNodeInstanceKey key = BKE_node_instance_hash_iterator_get_key(&iter);
1678                                                 BKE_node_instance_hash_insert(new_previews, key, new_preview);
1679                                         }
1680                                 }
1681                         }
1682                         BKE_node_instance_hash_free(sce->nodetree->previews, NULL);
1683                         sce->nodetree->previews = new_previews;
1684                 }
1685         }
1686 }
1687
1688 void blo_make_movieclip_pointer_map(FileData *fd, Main *oldmain)
1689 {
1690         MovieClip *clip = oldmain->movieclip.first;
1691         Scene *sce = oldmain->scene.first;
1692         
1693         fd->movieclipmap = oldnewmap_new();
1694         
1695         for (; clip; clip = clip->id.next) {
1696                 if (clip->cache)
1697                         oldnewmap_insert(fd->movieclipmap, clip->cache, clip->cache, 0);
1698                 
1699                 if (clip->tracking.camera.intrinsics)
1700                         oldnewmap_insert(fd->movieclipmap, clip->tracking.camera.intrinsics, clip->tracking.camera.intrinsics, 0);
1701         }
1702         
1703         for (; sce; sce = sce->id.next) {
1704                 if (sce->nodetree) {
1705                         bNode *node;
1706                         for (node = sce->nodetree->nodes.first; node; node = node->next)
1707                                 if (node->type == CMP_NODE_MOVIEDISTORTION)
1708                                         oldnewmap_insert(fd->movieclipmap, node->storage, node->storage, 0);
1709                 }
1710         }
1711 }
1712
1713 /* set old main movie clips caches to zero if it has been restored */
1714 /* this works because freeing old main only happens after this call */
1715 void blo_end_movieclip_pointer_map(FileData *fd, Main *oldmain)
1716 {
1717         OldNew *entry = fd->movieclipmap->entries;
1718         MovieClip *clip = oldmain->movieclip.first;
1719         Scene *sce = oldmain->scene.first;
1720         int i;
1721         
1722         /* used entries were restored, so we put them to zero */
1723         for (i=0; i < fd->movieclipmap->nentries; i++, entry++) {
1724                 if (entry->nr > 0)
1725                         entry->newp = NULL;
1726         }
1727         
1728         for (; clip; clip = clip->id.next) {
1729                 clip->cache = newmclipadr(fd, clip->cache);
1730                 clip->tracking.camera.intrinsics = newmclipadr(fd, clip->tracking.camera.intrinsics);
1731         }
1732         
1733         for (; sce; sce = sce->id.next) {
1734                 if (sce->nodetree) {
1735                         bNode *node;
1736                         for (node = sce->nodetree->nodes.first; node; node = node->next)
1737                                 if (node->type == CMP_NODE_MOVIEDISTORTION)
1738                                         node->storage = newmclipadr(fd, node->storage);
1739                 }
1740         }
1741 }
1742
1743 void blo_make_sound_pointer_map(FileData *fd, Main *oldmain)
1744 {
1745         bSound *sound = oldmain->sound.first;
1746         
1747         fd->soundmap = oldnewmap_new();
1748         
1749         for (; sound; sound = sound->id.next) {
1750                 if (sound->waveform)
1751                         oldnewmap_insert(fd->soundmap, sound->waveform, sound->waveform, 0);                    
1752         }
1753 }
1754
1755 /* set old main sound caches to zero if it has been restored */
1756 /* this works because freeing old main only happens after this call */
1757 void blo_end_sound_pointer_map(FileData *fd, Main *oldmain)
1758 {
1759         OldNew *entry = fd->soundmap->entries;
1760         bSound *sound = oldmain->sound.first;
1761         int i;
1762         
1763         /* used entries were restored, so we put them to zero */
1764         for (i = 0; i < fd->soundmap->nentries; i++, entry++) {
1765                 if (entry->nr > 0)
1766                         entry->newp = NULL;
1767         }
1768         
1769         for (; sound; sound = sound->id.next) {
1770                 sound->waveform = newsoundadr(fd, sound->waveform);
1771         }
1772 }
1773
1774 /* XXX disabled this feature - packed files also belong in temp saves and quit.blend, to make restore work */
1775
1776 static void insert_packedmap(FileData *fd, PackedFile *pf)
1777 {
1778         oldnewmap_insert(fd->packedmap, pf, pf, 0);
1779         oldnewmap_insert(fd->packedmap, pf->data, pf->data, 0);
1780 }
1781
1782 void blo_make_packed_pointer_map(FileData *fd, Main *oldmain)
1783 {
1784         Image *ima;
1785         VFont *vfont;
1786         bSound *sound;
1787         Library *lib;
1788         
1789         fd->packedmap = oldnewmap_new();
1790         
1791         for (ima = oldmain->image.first; ima; ima = ima->id.next) {
1792                 ImagePackedFile *imapf;
1793
1794                 if (ima->packedfile)
1795                         insert_packedmap(fd, ima->packedfile);
1796
1797                 for (imapf = ima->packedfiles.first; imapf; imapf = imapf->next)
1798                         if (imapf->packedfile)
1799                                 insert_packedmap(fd, imapf->packedfile);
1800         }
1801                         
1802         for (vfont = oldmain->vfont.first; vfont; vfont = vfont->id.next)
1803                 if (vfont->packedfile)
1804                         insert_packedmap(fd, vfont->packedfile);
1805         
1806         for (sound = oldmain->sound.first; sound; sound = sound->id.next)
1807                 if (sound->packedfile)
1808                         insert_packedmap(fd, sound->packedfile);
1809         
1810         for (lib = oldmain->library.first; lib; lib = lib->id.next)
1811                 if (lib->packedfile)
1812                         insert_packedmap(fd, lib->packedfile);
1813
1814 }
1815
1816 /* set old main packed data to zero if it has been restored */
1817 /* this works because freeing old main only happens after this call */
1818 void blo_end_packed_pointer_map(FileData *fd, Main *oldmain)
1819 {
1820         Image *ima;
1821         VFont *vfont;
1822         bSound *sound;
1823         Library *lib;
1824         OldNew *entry = fd->packedmap->entries;
1825         int i;
1826         
1827         /* used entries were restored, so we put them to zero */
1828         for (i=0; i < fd->packedmap->nentries; i++, entry++) {
1829                 if (entry->nr > 0)
1830                         entry->newp = NULL;
1831         }
1832         
1833         for (ima = oldmain->image.first; ima; ima = ima->id.next) {
1834                 ImagePackedFile *imapf;
1835
1836                 ima->packedfile = newpackedadr(fd, ima->packedfile);
1837
1838                 for (imapf = ima->packedfiles.first; imapf; imapf = imapf->next)
1839                         imapf->packedfile = newpackedadr(fd, imapf->packedfile);
1840         }
1841         
1842         for (vfont = oldmain->vfont.first; vfont; vfont = vfont->id.next)
1843                 vfont->packedfile = newpackedadr(fd, vfont->packedfile);
1844
1845         for (sound = oldmain->sound.first; sound; sound = sound->id.next)
1846                 sound->packedfile = newpackedadr(fd, sound->packedfile);
1847                 
1848         for (lib = oldmain->library.first; lib; lib = lib->id.next)
1849                 lib->packedfile = newpackedadr(fd, lib->packedfile);
1850 }
1851
1852
1853 /* undo file support: add all library pointers in lookup */
1854 void blo_add_library_pointer_map(ListBase *old_mainlist, FileData *fd)
1855 {
1856         Main *ptr = old_mainlist->first;
1857         ListBase *lbarray[MAX_LIBARRAY];
1858         
1859         for (ptr = ptr->next; ptr; ptr = ptr->next) {
1860                 int i = set_listbasepointers(ptr, lbarray);
1861                 while (i--) {
1862                         ID *id;
1863                         for (id = lbarray[i]->first; id; id = id->next)
1864                                 oldnewmap_insert(fd->libmap, id, id, GS(id->name));
1865                 }
1866         }
1867
1868         fd->old_mainlist = old_mainlist;
1869 }
1870
1871
1872 /* ********** END OLD POINTERS ****************** */
1873 /* ********** READ FILE ****************** */
1874
1875 static void switch_endian_structs(const struct SDNA *filesdna, BHead *bhead)
1876 {
1877         int blocksize, nblocks;
1878         char *data;
1879         
1880         data = (char *)(bhead+1);
1881         blocksize = filesdna->typelens[ filesdna->structs[bhead->SDNAnr][0] ];
1882         
1883         nblocks = bhead->nr;
1884         while (nblocks--) {
1885                 DNA_struct_switch_endian(filesdna, bhead->SDNAnr, data);
1886                 
1887                 data += blocksize;
1888         }
1889 }
1890
1891 static void *read_struct(FileData *fd, BHead *bh, const char *blockname)
1892 {
1893         void *temp = NULL;
1894         
1895         if (bh->len) {
1896                 /* switch is based on file dna */
1897                 if (bh->SDNAnr && (fd->flags & FD_FLAGS_SWITCH_ENDIAN))
1898                         switch_endian_structs(fd->filesdna, bh);
1899                 
1900                 if (fd->compflags[bh->SDNAnr] != SDNA_CMP_REMOVED) {
1901                         if (fd->compflags[bh->SDNAnr] == SDNA_CMP_NOT_EQUAL) {
1902                                 temp = DNA_struct_reconstruct(fd->memsdna, fd->filesdna, fd->compflags, bh->SDNAnr, bh->nr, (bh+1));
1903                         }
1904                         else {
1905                                 /* SDNA_CMP_EQUAL */
1906                                 temp = MEM_mallocN(bh->len, blockname);
1907                                 memcpy(temp, (bh+1), bh->len);
1908                         }
1909                 }
1910         }
1911
1912         return temp;
1913 }
1914
1915 typedef void (*link_list_cb)(FileData *fd, void *data);
1916
1917 static void link_list_ex(FileData *fd, ListBase *lb, link_list_cb callback)             /* only direct data */
1918 {
1919         Link *ln, *prev;
1920         
1921         if (BLI_listbase_is_empty(lb)) return;
1922         
1923         lb->first = newdataadr(fd, lb->first);
1924         if (callback != NULL) {
1925                 callback(fd, lb->first);
1926         }
1927         ln = lb->first;
1928         prev = NULL;
1929         while (ln) {
1930                 ln->next = newdataadr(fd, ln->next);
1931                 if (ln->next != NULL && callback != NULL) {
1932                         callback(fd, ln->next);
1933                 }
1934                 ln->prev = prev;
1935                 prev = ln;
1936                 ln = ln->next;
1937         }
1938         lb->last = prev;
1939 }
1940
1941 static void link_list(FileData *fd, ListBase *lb)               /* only direct data */
1942 {
1943         link_list_ex(fd, lb, NULL);
1944 }
1945
1946 static void link_glob_list(FileData *fd, ListBase *lb)          /* for glob data */
1947 {
1948         Link *ln, *prev;
1949         void *poin;
1950
1951         if (BLI_listbase_is_empty(lb)) return;
1952         poin = newdataadr(fd, lb->first);
1953         if (lb->first) {
1954                 oldnewmap_insert(fd->globmap, lb->first, poin, 0);
1955         }
1956         lb->first = poin;
1957         
1958         ln = lb->first;
1959         prev = NULL;
1960         while (ln) {
1961                 poin = newdataadr(fd, ln->next);
1962                 if (ln->next) {
1963                         oldnewmap_insert(fd->globmap, ln->next, poin, 0);
1964                 }
1965                 ln->next = poin;
1966                 ln->prev = prev;
1967                 prev = ln;
1968                 ln = ln->next;
1969         }
1970         lb->last = prev;
1971 }
1972
1973 static void test_pointer_array(FileData *fd, void **mat)
1974 {
1975         int64_t *lpoin, *lmat;
1976         int *ipoin, *imat;
1977         size_t len;
1978
1979                 /* manually convert the pointer array in
1980                  * the old dna format to a pointer array in
1981                  * the new dna format.
1982                  */
1983         if (*mat) {
1984                 len = MEM_allocN_len(*mat)/fd->filesdna->pointerlen;
1985                         
1986                 if (fd->filesdna->pointerlen==8 && fd->memsdna->pointerlen==4) {
1987                         ipoin=imat= MEM_mallocN(len * 4, "newmatar");
1988                         lpoin= *mat;
1989                         
1990                         while (len-- > 0) {
1991                                 if ((fd->flags & FD_FLAGS_SWITCH_ENDIAN))
1992                                         BLI_endian_switch_int64(lpoin);
1993                                 *ipoin = (int)((*lpoin) >> 3);
1994                                 ipoin++;
1995                                 lpoin++;
1996                         }
1997                         MEM_freeN(*mat);
1998                         *mat = imat;
1999                 }
2000                 
2001                 if (fd->filesdna->pointerlen==4 && fd->memsdna->pointerlen==8) {
2002                         lpoin = lmat = MEM_mallocN(len * 8, "newmatar");
2003                         ipoin = *mat;
2004                         
2005                         while (len-- > 0) {
2006                                 *lpoin = *ipoin;
2007                                 ipoin++;
2008                                 lpoin++;
2009                         }
2010                         MEM_freeN(*mat);
2011                         *mat= lmat;
2012                 }
2013         }
2014 }
2015
2016 /* ************ READ ID Properties *************** */
2017
2018 static void IDP_DirectLinkProperty(IDProperty *prop, int switch_endian, FileData *fd);
2019 static void IDP_LibLinkProperty(IDProperty *prop, FileData *fd);
2020
2021 static void IDP_DirectLinkIDPArray(IDProperty *prop, int switch_endian, FileData *fd)
2022 {
2023         IDProperty *array;
2024         int i;
2025         
2026         /* since we didn't save the extra buffer, set totallen to len */
2027         prop->totallen = prop->len;
2028         prop->data.pointer = newdataadr(fd, prop->data.pointer);
2029
2030         array = (IDProperty *)prop->data.pointer;
2031         
2032         /* note!, idp-arrays didn't exist in 2.4x, so the pointer will be cleared
2033          * theres not really anything we can do to correct this, at least don't crash */
2034         if (array == NULL) {
2035                 prop->len = 0;
2036                 prop->totallen = 0;
2037         }
2038         
2039         
2040         for (i = 0; i < prop->len; i++)
2041                 IDP_DirectLinkProperty(&array[i], switch_endian, fd);
2042 }
2043
2044 static void IDP_DirectLinkArray(IDProperty *prop, int switch_endian, FileData *fd)
2045 {
2046         IDProperty **array;
2047         int i;
2048         
2049         /* since we didn't save the extra buffer, set totallen to len */
2050         prop->totallen = prop->len;
2051         prop->data.pointer = newdataadr(fd, prop->data.pointer);
2052         
2053         if (prop->subtype == IDP_GROUP) {
2054                 test_pointer_array(fd, prop->data.pointer);
2055                 array = prop->data.pointer;
2056                 
2057                 for (i = 0; i < prop->len; i++)
2058                         IDP_DirectLinkProperty(array[i], switch_endian, fd);
2059         }
2060         else if (prop->subtype == IDP_DOUBLE) {
2061                 if (switch_endian) {
2062                         BLI_endian_switch_double_array(prop->data.pointer, prop->len);
2063                 }
2064         }
2065         else {
2066                 if (switch_endian) {
2067                         /* also used for floats */
2068                         BLI_endian_switch_int32_array(prop->data.pointer, prop->len);
2069                 }
2070         }
2071 }
2072
2073 static void IDP_DirectLinkString(IDProperty *prop, FileData *fd)
2074 {
2075         /*since we didn't save the extra string buffer, set totallen to len.*/
2076         prop->totallen = prop->len;
2077         prop->data.pointer = newdataadr(fd, prop->data.pointer);
2078 }
2079
2080 static void IDP_DirectLinkGroup(IDProperty *prop, int switch_endian, FileData *fd)
2081 {
2082         ListBase *lb = &prop->data.group;
2083         IDProperty *loop;
2084         
2085         link_list(fd, lb);
2086         
2087         /*Link child id properties now*/
2088         for (loop=prop->data.group.first; loop; loop=loop->next) {
2089                 IDP_DirectLinkProperty(loop, switch_endian, fd);
2090         }
2091 }
2092
2093 static void IDP_DirectLinkProperty(IDProperty *prop, int switch_endian, FileData *fd)
2094 {
2095         switch (prop->type) {
2096                 case IDP_GROUP:
2097                         IDP_DirectLinkGroup(prop, switch_endian, fd);
2098                         break;
2099                 case IDP_STRING:
2100                         IDP_DirectLinkString(prop, fd);
2101                         break;
2102                 case IDP_ARRAY:
2103                         IDP_DirectLinkArray(prop, switch_endian, fd);
2104                         break;
2105                 case IDP_IDPARRAY:
2106                         IDP_DirectLinkIDPArray(prop, switch_endian, fd);
2107                         break;
2108                 case IDP_DOUBLE:
2109                         /* erg, stupid doubles.  since I'm storing them
2110                          * in the same field as int val; val2 in the
2111                          * IDPropertyData struct, they have to deal with
2112                          * endianness specifically
2113                          *
2114                          * in theory, val and val2 would've already been swapped
2115                          * if switch_endian is true, so we have to first unswap
2116                          * them then reswap them as a single 64-bit entity.
2117                          */
2118                         
2119                         if (switch_endian) {
2120                                 BLI_endian_switch_int32(&prop->data.val);
2121                                 BLI_endian_switch_int32(&prop->data.val2);
2122                                 BLI_endian_switch_int64((int64_t *)&prop->data.val);
2123                         }
2124                         break;
2125                 case IDP_INT:
2126                 case IDP_FLOAT:
2127                 case IDP_ID:
2128                         break;  /* Nothing special to do here. */
2129                 default:
2130                         /* Unknown IDP type, nuke it (we cannot handle unknown types everywhere in code,
2131                          * IDP are way too polymorphic to do it safely. */
2132                         printf("%s: found unknown IDProperty type %d, reset to Integer one !\n", __func__, prop->type);
2133                         /* Note: we do not attempt to free unknown prop, we have no way to know how to do that! */
2134                         prop->type = IDP_INT;
2135                         prop->subtype = 0;
2136                         IDP_Int(prop) = 0;
2137         }
2138 }
2139
2140 #define IDP_DirectLinkGroup_OrFree(prop, switch_endian, fd) \
2141        _IDP_DirectLinkGroup_OrFree(prop, switch_endian, fd, __func__)
2142
2143 static void _IDP_DirectLinkGroup_OrFree(IDProperty **prop, int switch_endian, FileData *fd,
2144                                         const char *caller_func_id)
2145 {
2146         if (*prop) {
2147                 if ((*prop)->type == IDP_GROUP) {
2148                         IDP_DirectLinkGroup(*prop, switch_endian, fd);
2149                 }
2150                 else {
2151                         /* corrupt file! */
2152                         printf("%s: found non group data, freeing type %d!\n",
2153                                caller_func_id, (*prop)->type);
2154                         /* don't risk id, data's likely corrupt. */
2155                         // IDP_FreeProperty(*prop);
2156                         *prop = NULL;
2157                 }
2158         }
2159 }
2160
2161 static void IDP_LibLinkProperty(IDProperty *prop, FileData *fd)
2162 {
2163         if (!prop)
2164                 return;
2165
2166         switch (prop->type) {
2167                 case IDP_ID: /* PointerProperty */
2168                 {
2169                         void *newaddr = newlibadr_us(fd, NULL, IDP_Id(prop));
2170                         if (IDP_Id(prop) && !newaddr && G.debug) {
2171                                 printf("Error while loading \"%s\". Data not found in file!\n", prop->name);
2172                         }
2173                         prop->data.pointer = newaddr;
2174                         break;
2175                 }
2176                 case IDP_IDPARRAY: /* CollectionProperty */
2177                 {
2178                         IDProperty *idp_array = IDP_IDPArray(prop);
2179                         for (int i = 0; i < prop->len; i++) {
2180                                 IDP_LibLinkProperty(&(idp_array[i]), fd);
2181                         }
2182                         break;
2183                 }
2184                 case IDP_GROUP: /* PointerProperty */
2185                 {
2186                         for (IDProperty *loop = prop->data.group.first; loop; loop = loop->next) {
2187                                 IDP_LibLinkProperty(loop, fd);
2188                         }
2189                         break;
2190                 }
2191                 default:
2192                         break;  /* Nothing to do for other IDProps. */
2193         }
2194 }
2195
2196 /* ************ READ IMAGE PREVIEW *************** */
2197
2198 static PreviewImage *direct_link_preview_image(FileData *fd, PreviewImage *old_prv)
2199 {
2200         PreviewImage *prv = newdataadr(fd, old_prv);
2201         
2202         if (prv) {
2203                 int i;
2204                 for (i = 0; i < NUM_ICON_SIZES; ++i) {
2205                         if (prv->rect[i]) {
2206                                 prv->rect[i] = newdataadr(fd, prv->rect[i]);
2207                         }
2208                         prv->gputexture[i] = NULL;
2209                 }
2210                 prv->icon_id = 0;
2211                 prv->tag = 0;
2212         }
2213         
2214         return prv;
2215 }
2216
2217 /* ************ READ ID *************** */
2218
2219 static void lib_link_id(FileData *fd, Main *main)
2220 {
2221         ListBase *lbarray[MAX_LIBARRAY];
2222         int base_count, i;
2223
2224         base_count = set_listbasepointers(main, lbarray);
2225
2226         for (i = 0; i < base_count; i++) {
2227                 ListBase *lb = lbarray[i];
2228                 ID *id;
2229
2230                 for (id = lb->first; id; id = id->next) {
2231                         if (id->override_static) {
2232                                 id->override_static->reference = newlibadr_us(fd, id->lib, id->override_static->reference);
2233                                 id->override_static->storage = newlibadr_us(fd, id->lib, id->override_static->storage);
2234                         }
2235                 }
2236         }
2237 }
2238
2239 static void direct_link_id_override_property_operation_cb(FileData *fd, void *data)
2240 {
2241         IDOverrideStaticPropertyOperation *opop = data;
2242
2243         opop->subitem_reference_name = newdataadr(fd, opop->subitem_reference_name);
2244         opop->subitem_local_name = newdataadr(fd, opop->subitem_local_name);
2245 }
2246
2247 static void direct_link_id_override_property_cb(FileData *fd, void *data)
2248 {
2249         IDOverrideStaticProperty *op = data;
2250
2251         op->rna_path = newdataadr(fd, op->rna_path);
2252         link_list_ex(fd, &op->operations, direct_link_id_override_property_operation_cb);
2253 }
2254
2255 static void direct_link_id(FileData *fd, ID *id)
2256 {
2257         /*link direct data of ID properties*/
2258         if (id->properties) {
2259                 id->properties = newdataadr(fd, id->properties);
2260                 /* this case means the data was written incorrectly, it should not happen */
2261                 IDP_DirectLinkGroup_OrFree(&id->properties, (fd->flags & FD_FLAGS_SWITCH_ENDIAN), fd);
2262         }
2263         id->py_instance = NULL;
2264
2265         /* Link direct data of overrides. */
2266         if (id->override_static) {
2267                 id->override_static = newdataadr(fd, id->override_static);
2268                 link_list_ex(fd, &id->override_static->properties, direct_link_id_override_property_cb);
2269         }
2270 }
2271
2272 /* ************ READ CurveMapping *************** */
2273
2274 /* cuma itself has been read! */
2275 static void direct_link_curvemapping(FileData *fd, CurveMapping *cumap)
2276 {
2277         int a;
2278         
2279         /* flag seems to be able to hang? Maybe old files... not bad to clear anyway */
2280         cumap->flag &= ~CUMA_PREMULLED;
2281         
2282         for (a = 0; a < CM_TOT; a++) {
2283                 cumap->cm[a].curve = newdataadr(fd, cumap->cm[a].curve);
2284                 cumap->cm[a].table = NULL;
2285                 cumap->cm[a].premultable = NULL;
2286         }
2287 }
2288
2289 /* ************ READ Brush *************** */
2290 /* library brush linking after fileread */
2291 static void lib_link_brush(FileData *fd, Main *main)
2292 {
2293         /* only link ID pointers */
2294         for (Brush *brush = main->brush.first; brush; brush = brush->id.next) {
2295                 if (brush->id.tag & LIB_TAG_NEED_LINK) {
2296                         IDP_LibLinkProperty(brush->id.properties, fd);
2297
2298                         /* brush->(mask_)mtex.obj is ignored on purpose? */
2299                         brush->mtex.tex = newlibadr_us(fd, brush->id.lib, brush->mtex.tex);
2300                         brush->mask_mtex.tex = newlibadr_us(fd, brush->id.lib, brush->mask_mtex.tex);
2301                         brush->clone.image = newlibadr(fd, brush->id.lib, brush->clone.image);
2302                         brush->toggle_brush = newlibadr(fd, brush->id.lib, brush->toggle_brush);
2303                         brush->paint_curve = newlibadr_us(fd, brush->id.lib, brush->paint_curve);
2304
2305                         brush->id.tag &= ~LIB_TAG_NEED_LINK;
2306                 }
2307         }
2308 }
2309
2310 static void direct_link_brush(FileData *fd, Brush *brush)
2311 {
2312         /* brush itself has been read */
2313
2314         /* fallof curve */
2315         brush->curve = newdataadr(fd, brush->curve);
2316         brush->gradient = newdataadr(fd, brush->gradient);
2317
2318         if (brush->curve)
2319                 direct_link_curvemapping(fd, brush->curve);
2320         else
2321                 BKE_brush_curve_preset(brush, CURVE_PRESET_SHARP);
2322
2323         brush->preview = NULL;
2324         brush->icon_imbuf = NULL;
2325 }
2326
2327 /* ************ READ Palette *************** */
2328 static void lib_link_palette(FileData *fd, Main *main)
2329 {
2330         /* only link ID pointers */
2331         for (Palette *palette = main->palettes.first; palette; palette = palette->id.next) {
2332                 if (palette->id.tag & LIB_TAG_NEED_LINK) {
2333                         IDP_LibLinkProperty(palette->id.properties, fd);
2334
2335                         palette->id.tag &= ~LIB_TAG_NEED_LINK;
2336                 }
2337         }
2338 }
2339
2340 static void direct_link_palette(FileData *fd, Palette *palette)
2341 {
2342         /* palette itself has been read */
2343         link_list(fd, &palette->colors);
2344 }
2345
2346 static void lib_link_paint_curve(FileData *fd, Main *main)
2347 {
2348         /* only link ID pointers */
2349         for (PaintCurve *pc = main->paintcurves.first; pc; pc = pc->id.next) {
2350                 if (pc->id.tag & LIB_TAG_NEED_LINK) {
2351                         IDP_LibLinkProperty(pc->id.properties, fd);
2352
2353                         pc->id.tag &= ~LIB_TAG_NEED_LINK;
2354                 }
2355         }
2356 }
2357
2358 static void direct_link_paint_curve(FileData *fd, PaintCurve *pc)
2359 {
2360         pc->points = newdataadr(fd, pc->points);
2361 }
2362
2363 /* ************ READ PACKEDFILE *************** */
2364
2365 static PackedFile *direct_link_packedfile(FileData *fd, PackedFile *oldpf)
2366 {
2367         PackedFile *pf = newpackedadr(fd, oldpf);
2368
2369         if (pf) {
2370                 pf->data = newpackedadr(fd, pf->data);
2371         }
2372         
2373         return pf;
2374 }
2375
2376 /* ************ READ ANIMATION STUFF ***************** */
2377
2378 /* Legacy Data Support (for Version Patching) ----------------------------- */
2379
2380 // XXX deprecated - old animation system
2381 static void lib_link_ipo(FileData *fd, Main *main)
2382 {
2383         Ipo *ipo;
2384         
2385         for (ipo = main->ipo.first; ipo; ipo = ipo->id.next) {
2386                 if (ipo->id.tag & LIB_TAG_NEED_LINK) {
2387                         IpoCurve *icu;
2388                         for (icu = ipo->curve.first; icu; icu = icu->next) {
2389                                 if (icu->driver)
2390                                         icu->driver->ob = newlibadr(fd, ipo->id.lib, icu->driver->ob);
2391                         }
2392                         ipo->id.tag &= ~LIB_TAG_NEED_LINK;
2393                 }
2394         }
2395 }
2396
2397 // XXX deprecated - old animation system
2398 static void direct_link_ipo(FileData *fd, Ipo *ipo)
2399 {
2400         IpoCurve *icu;
2401
2402         link_list(fd, &(ipo->curve));
2403         
2404         for (icu = ipo->curve.first; icu; icu = icu->next) {
2405                 icu->bezt = newdataadr(fd, icu->bezt);
2406                 icu->bp = newdataadr(fd, icu->bp);
2407                 icu->driver = newdataadr(fd, icu->driver);
2408         }
2409 }
2410
2411 // XXX deprecated - old animation system
2412 static void lib_link_nlastrips(FileData *fd, ID *id, ListBase *striplist)
2413 {
2414         bActionStrip *strip;
2415         bActionModifier *amod;
2416         
2417         for (strip=striplist->first; strip; strip=strip->next) {
2418                 strip->object = newlibadr(fd, id->lib, strip->object);
2419                 strip->act = newlibadr_us(fd, id->lib, strip->act);
2420                 strip->ipo = newlibadr(fd, id->lib, strip->ipo);
2421                 for (amod = strip->modifiers.first; amod; amod = amod->next)
2422                         amod->ob = newlibadr(fd, id->lib, amod->ob);
2423         }
2424 }
2425
2426 // XXX deprecated - old animation system
2427 static void direct_link_nlastrips(FileData *fd, ListBase *strips)
2428 {
2429         bActionStrip *strip;
2430         
2431         link_list(fd, strips);
2432         
2433         for (strip = strips->first; strip; strip = strip->next)
2434                 link_list(fd, &strip->modifiers);
2435 }
2436
2437 // XXX deprecated - old animation system
2438 static void lib_link_constraint_channels(FileData *fd, ID *id, ListBase *chanbase)
2439 {
2440         bConstraintChannel *chan;
2441
2442         for (chan=chanbase->first; chan; chan=chan->next) {
2443                 chan->ipo = newlibadr_us(fd, id->lib, chan->ipo);
2444         }
2445 }
2446
2447 /* Data Linking ----------------------------- */
2448
2449 static void lib_link_fmodifiers(FileData *fd, ID *id, ListBase *list)
2450 {
2451         FModifier *fcm;
2452         
2453         for (fcm = list->first; fcm; fcm = fcm->next) {
2454                 /* data for specific modifiers */
2455                 switch (fcm->type) {
2456                         case FMODIFIER_TYPE_PYTHON:
2457                         {
2458                                 FMod_Python *data = (FMod_Python *)fcm->data;
2459                                 data->script = newlibadr(fd, id->lib, data->script);
2460
2461                                 break;
2462                         }
2463                 }
2464         }
2465 }
2466
2467 static void lib_link_fcurves(FileData *fd, ID *id, ListBase *list) 
2468 {
2469         FCurve *fcu;
2470         
2471         if (list == NULL)
2472                 return;
2473         
2474         /* relink ID-block references... */
2475         for (fcu = list->first; fcu; fcu = fcu->next) {
2476                 /* driver data */
2477                 if (fcu->driver) {
2478                         ChannelDriver *driver = fcu->driver;
2479                         DriverVar *dvar;
2480                         
2481                         for (dvar= driver->variables.first; dvar; dvar= dvar->next) {
2482                                 DRIVER_TARGETS_LOOPER(dvar)
2483                                 {
2484                                         /* only relink if still used */
2485                                         if (tarIndex < dvar->num_targets)
2486                                                 dtar->id = newlibadr(fd, id->lib, dtar->id); 
2487                                         else
2488                                                 dtar->id = NULL;
2489                                 }
2490                                 DRIVER_TARGETS_LOOPER_END
2491                         }
2492                 }
2493                 
2494                 /* modifiers */
2495                 lib_link_fmodifiers(fd, id, &fcu->modifiers);
2496         }
2497 }
2498
2499
2500 /* NOTE: this assumes that link_list has already been called on the list */
2501 static void direct_link_fmodifiers(FileData *fd, ListBase *list, FCurve *curve)
2502 {
2503         FModifier *fcm;
2504         
2505         for (fcm = list->first; fcm; fcm = fcm->next) {
2506                 /* relink general data */
2507                 fcm->data  = newdataadr(fd, fcm->data);
2508                 fcm->curve = curve;
2509                 
2510                 /* do relinking of data for specific types */
2511                 switch (fcm->type) {
2512                         case FMODIFIER_TYPE_GENERATOR:
2513                         {
2514                                 FMod_Generator *data = (FMod_Generator *)fcm->data;
2515                                 
2516                                 data->coefficients = newdataadr(fd, data->coefficients);
2517                                 
2518                                 if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
2519                                         BLI_endian_switch_float_array(data->coefficients, data->arraysize);
2520                                 }
2521
2522                                 break;
2523                         }
2524                         case FMODIFIER_TYPE_ENVELOPE:
2525                         {
2526                                 FMod_Envelope *data=  (FMod_Envelope *)fcm->data;
2527                                 
2528                                 data->data= newdataadr(fd, data->data);
2529
2530                                 break;
2531                         }
2532                         case FMODIFIER_TYPE_PYTHON:
2533                         {
2534                                 FMod_Python *data = (FMod_Python *)fcm->data;
2535                                 
2536                                 data->prop = newdataadr(fd, data->prop);
2537                                 IDP_DirectLinkGroup_OrFree(&data->prop, (fd->flags & FD_FLAGS_SWITCH_ENDIAN), fd);
2538
2539                                 break;
2540                         }
2541                 }
2542         }
2543 }
2544
2545 /* NOTE: this assumes that link_list has already been called on the list */
2546 static void direct_link_fcurves(FileData *fd, ListBase *list)
2547 {
2548         FCurve *fcu;
2549         
2550         /* link F-Curve data to F-Curve again (non ID-libs) */
2551         for (fcu = list->first; fcu; fcu = fcu->next) {
2552                 /* curve data */
2553                 fcu->bezt = newdataadr(fd, fcu->bezt);
2554                 fcu->fpt = newdataadr(fd, fcu->fpt);
2555                 
2556                 /* rna path */
2557                 fcu->rna_path = newdataadr(fd, fcu->rna_path);
2558                 
2559                 /* group */
2560                 fcu->grp = newdataadr_ex(fd, fcu->grp, false);
2561                 
2562                 /* clear disabled flag - allows disabled drivers to be tried again ([#32155]),
2563                  * but also means that another method for "reviving disabled F-Curves" exists
2564                  */
2565                 fcu->flag &= ~FCURVE_DISABLED;
2566                 
2567                 /* driver */
2568                 fcu->driver= newdataadr(fd, fcu->driver);
2569                 if (fcu->driver) {
2570                         ChannelDriver *driver= fcu->driver;
2571                         DriverVar *dvar;
2572                         
2573                         /* compiled expression data will need to be regenerated (old pointer may still be set here) */
2574                         driver->expr_comp = NULL;
2575                         
2576                         /* give the driver a fresh chance - the operating environment may be different now 
2577                          * (addons, etc. may be different) so the driver namespace may be sane now [#32155]
2578                          */
2579                         driver->flag &= ~DRIVER_FLAG_INVALID;
2580                         
2581                         /* relink variables, targets and their paths */
2582                         link_list(fd, &driver->variables);
2583                         for (dvar= driver->variables.first; dvar; dvar= dvar->next) {
2584                                 DRIVER_TARGETS_LOOPER(dvar)
2585                                 {
2586                                         /* only relink the targets being used */
2587                                         if (tarIndex < dvar->num_targets)
2588                                                 dtar->rna_path = newdataadr(fd, dtar->rna_path);
2589                                         else
2590                                                 dtar->rna_path = NULL;
2591                                 }
2592                                 DRIVER_TARGETS_LOOPER_END
2593                         }
2594                 }
2595                 
2596                 /* modifiers */
2597                 link_list(fd, &fcu->modifiers);
2598                 direct_link_fmodifiers(fd, &fcu->modifiers, fcu);
2599         }
2600 }
2601
2602
2603 static void lib_link_action(FileData *fd, Main *main)
2604 {
2605         for (bAction *act = main->action.first; act; act = act->id.next) {
2606                 if (act->id.tag & LIB_TAG_NEED_LINK) {
2607                         IDP_LibLinkProperty(act->id.properties, fd);
2608                         
2609 // XXX deprecated - old animation system <<<
2610                         for (bActionChannel *chan = act->chanbase.first; chan; chan = chan->next) {
2611                                 chan->ipo = newlibadr_us(fd, act->id.lib, chan->ipo);
2612                                 lib_link_constraint_channels(fd, &act->id, &chan->constraintChannels);
2613                         }
2614 // >>> XXX deprecated - old animation system
2615                         
2616                         lib_link_fcurves(fd, &act->id, &act->curves);
2617
2618                         for (TimeMarker *marker = act->markers.first; marker; marker = marker->next) {
2619                                 if (marker->camera) {
2620                                         marker->camera = newlibadr(fd, act->id.lib, marker->camera);
2621                                 }
2622                         }
2623
2624                         act->id.tag &= ~LIB_TAG_NEED_LINK;
2625                 }
2626         }
2627 }
2628
2629 static void direct_link_action(FileData *fd, bAction *act)
2630 {
2631         bActionChannel *achan; // XXX deprecated - old animation system
2632         bActionGroup *agrp;
2633
2634         link_list(fd, &act->curves);
2635         link_list(fd, &act->chanbase); // XXX deprecated - old animation system
2636         link_list(fd, &act->groups);
2637         link_list(fd, &act->markers);
2638
2639 // XXX deprecated - old animation system <<<
2640         for (achan = act->chanbase.first; achan; achan=achan->next) {
2641                 achan->grp = newdataadr(fd, achan->grp);
2642                 
2643                 link_list(fd, &achan->constraintChannels);
2644         }
2645 // >>> XXX deprecated - old animation system
2646
2647         direct_link_fcurves(fd, &act->curves);
2648         
2649         for (agrp = act->groups.first; agrp; agrp= agrp->next) {
2650                 agrp->channels.first= newdataadr(fd, agrp->channels.first);
2651                 agrp->channels.last= newdataadr(fd, agrp->channels.last);
2652         }
2653 }
2654
2655 static void lib_link_nladata_strips(FileData *fd, ID *id, ListBase *list)
2656 {
2657         NlaStrip *strip;
2658         
2659         for (strip = list->first; strip; strip = strip->next) {
2660                 /* check strip's children */
2661                 lib_link_nladata_strips(fd, id, &strip->strips);
2662                 
2663                 /* check strip's F-Curves */
2664                 lib_link_fcurves(fd, id, &strip->fcurves);
2665                 
2666                 /* reassign the counted-reference to action */
2667                 strip->act = newlibadr_us(fd, id->lib, strip->act);
2668                 
2669                 /* fix action id-root (i.e. if it comes from a pre 2.57 .blend file) */
2670                 if ((strip->act) && (strip->act->idroot == 0))
2671                         strip->act->idroot = GS(id->name);
2672         }
2673 }
2674
2675 static void lib_link_nladata(FileData *fd, ID *id, ListBase *list)
2676 {
2677         NlaTrack *nlt;
2678         
2679         /* we only care about the NLA strips inside the tracks */
2680         for (nlt = list->first; nlt; nlt = nlt->next) {
2681                 lib_link_nladata_strips(fd, id, &nlt->strips);
2682         }
2683 }
2684
2685 /* This handles Animato NLA-Strips linking 
2686  * NOTE: this assumes that link_list has already been called on the list 
2687  */
2688 static void direct_link_nladata_strips(FileData *fd, ListBase *list)
2689 {
2690         NlaStrip *strip;
2691         
2692         for (strip = list->first; strip; strip = strip->next) {
2693                 /* strip's child strips */
2694                 link_list(fd, &strip->strips);
2695                 direct_link_nladata_strips(fd, &strip->strips);
2696                 
2697                 /* strip's F-Curves */
2698                 link_list(fd, &strip->fcurves);
2699                 direct_link_fcurves(fd, &strip->fcurves);
2700                 
2701                 /* strip's F-Modifiers */
2702                 link_list(fd, &strip->modifiers);
2703                 direct_link_fmodifiers(fd, &strip->modifiers, NULL);
2704         }
2705 }
2706
2707 /* NOTE: this assumes that link_list has already been called on the list */
2708 static void direct_link_nladata(FileData *fd, ListBase *list)
2709 {
2710         NlaTrack *nlt;
2711         
2712         for (nlt = list->first; nlt; nlt = nlt->next) {
2713                 /* relink list of strips */
2714                 link_list(fd, &nlt->strips);
2715                 
2716                 /* relink strip data */
2717                 direct_link_nladata_strips(fd, &nlt->strips);
2718         }
2719 }
2720
2721 /* ------- */
2722
2723 static void lib_link_keyingsets(FileData *fd, ID *id, ListBase *list)
2724 {
2725         KeyingSet *ks;
2726         KS_Path *ksp;
2727         
2728         /* here, we're only interested in the ID pointer stored in some of the paths */
2729         for (ks = list->first; ks; ks = ks->next) {
2730                 for (ksp = ks->paths.first; ksp; ksp = ksp->next) {
2731                         ksp->id= newlibadr(fd, id->lib, ksp->id); 
2732                 }
2733         }
2734 }
2735
2736 /* NOTE: this assumes that link_list has already been called on the list */
2737 static void direct_link_keyingsets(FileData *fd, ListBase *list)
2738 {
2739         KeyingSet *ks;
2740         KS_Path *ksp;
2741         
2742         /* link KeyingSet data to KeyingSet again (non ID-libs) */
2743         for (ks = list->first; ks; ks = ks->next) {
2744                 /* paths */
2745                 link_list(fd, &ks->paths);
2746                 
2747                 for (ksp = ks->paths.first; ksp; ksp = ksp->next) {
2748                         /* rna path */
2749                         ksp->rna_path= newdataadr(fd, ksp->rna_path);
2750                 }
2751         }
2752 }
2753
2754 /* ------- */
2755
2756 static void lib_link_animdata(FileData *fd, ID *id, AnimData *adt)
2757 {
2758         if (adt == NULL)
2759                 return;
2760         
2761         /* link action data */
2762         adt->action= newlibadr_us(fd, id->lib, adt->action);
2763         adt->tmpact= newlibadr_us(fd, id->lib, adt->tmpact);
2764         
2765         /* fix action id-roots (i.e. if they come from a pre 2.57 .blend file) */
2766         if ((adt->action) && (adt->action->idroot == 0))
2767                 adt->action->idroot = GS(id->name);
2768         if ((adt->tmpact) && (adt->tmpact->idroot == 0))
2769                 adt->tmpact->idroot = GS(id->name);
2770         
2771         /* link drivers */
2772         lib_link_fcurves(fd, id, &adt->drivers);
2773         
2774         /* overrides don't have lib-link for now, so no need to do anything */
2775         
2776         /* link NLA-data */
2777         lib_link_nladata(fd, id, &adt->nla_tracks);
2778 }
2779
2780 static void direct_link_animdata(FileData *fd, AnimData *adt)
2781 {
2782         /* NOTE: must have called newdataadr already before doing this... */
2783         if (adt == NULL)
2784                 return;
2785         
2786         /* link drivers */
2787         link_list(fd, &adt->drivers);
2788         direct_link_fcurves(fd, &adt->drivers);
2789         
2790         /* link overrides */
2791         // TODO...
2792         
2793         /* link NLA-data */
2794         link_list(fd, &adt->nla_tracks);
2795         direct_link_nladata(fd, &adt->nla_tracks);
2796         
2797         /* relink active track/strip - even though strictly speaking this should only be used
2798          * if we're in 'tweaking mode', we need to be able to have this loaded back for
2799          * undo, but also since users may not exit tweakmode before saving (#24535)
2800          */
2801         // TODO: it's not really nice that anyone should be able to save the file in this
2802         //              state, but it's going to be too hard to enforce this single case...
2803         adt->act_track = newdataadr(fd, adt->act_track);
2804         adt->actstrip = newdataadr(fd, adt->actstrip);
2805 }       
2806
2807 /* ************ READ CACHEFILES *************** */
2808
2809 static void lib_link_cachefiles(FileData *fd, Main *bmain)
2810 {
2811         /* only link ID pointers */
2812         for (CacheFile *cache_file = bmain->cachefiles.first; cache_file; cache_file = cache_file->id.next) {
2813                 if (cache_file->id.tag & LIB_TAG_NEED_LINK) {
2814                         IDP_LibLinkProperty(cache_file->id.properties, fd);
2815                         lib_link_animdata(fd, &cache_file->id, cache_file->adt);
2816
2817                         cache_file->id.tag &= ~LIB_TAG_NEED_LINK;
2818                 }
2819         }
2820 }
2821
2822 static void direct_link_cachefile(FileData *fd, CacheFile *cache_file)
2823 {
2824         BLI_listbase_clear(&cache_file->object_paths);
2825         cache_file->handle = NULL;
2826         cache_file->handle_mutex = NULL;
2827
2828         /* relink animdata */
2829         cache_file->adt = newdataadr(fd, cache_file->adt);
2830         direct_link_animdata(fd, cache_file->adt);
2831 }
2832
2833 /* ************ READ WORKSPACES *************** */
2834
2835 static void lib_link_workspaces(FileData *fd, Main *bmain)
2836 {
2837         for (WorkSpace *workspace = bmain->workspaces.first; workspace; workspace = workspace->id.next) {
2838                 ListBase *layouts = BKE_workspace_layouts_get(workspace);
2839                 ID *id = (ID *)workspace;
2840
2841                 if ((id->tag & LIB_TAG_NEED_LINK) == 0) {
2842                         continue;
2843                 }
2844                 IDP_LibLinkProperty(id->properties, fd);
2845                 id_us_ensure_real(id);
2846
2847                 for (WorkSpaceDataRelation *relation = workspace->scene_viewlayer_relations.first;
2848                      relation != NULL;
2849                      relation = relation->next)
2850                 {
2851                         relation->parent = newlibadr(fd, id->lib, relation->parent);
2852                         /* relation->value is set in direct_link_workspace_link_scene_data */
2853                 }
2854
2855                 for (WorkSpaceLayout *layout = layouts->first, *layout_next; layout; layout = layout_next) {
2856                         bScreen *screen = newlibadr(fd, id->lib, BKE_workspace_layout_screen_get(layout));
2857
2858                         layout_next = layout->next;
2859                         if (screen) {
2860                                 BKE_workspace_layout_screen_set(layout, screen);
2861
2862                                 if (ID_IS_LINKED(id)) {
2863                                         screen->winid = 0;
2864                                         if (screen->temp) {
2865                                                 /* delete temp layouts when appending */
2866                                                 BKE_workspace_layout_remove(bmain, workspace, layout);
2867                                         }
2868                                 }
2869                         }
2870                 }
2871
2872                 id->tag &= ~LIB_TAG_NEED_LINK;
2873         }
2874 }
2875
2876 static void direct_link_workspace(FileData *fd, WorkSpace *workspace, const Main *main)
2877 {
2878         link_list(fd, BKE_workspace_layouts_get(workspace));
2879         link_list(fd, &workspace->hook_layout_relations);
2880         link_list(fd, &workspace->scene_viewlayer_relations);
2881         link_list(fd, BKE_workspace_transform_orientations_get(workspace));
2882
2883         for (WorkSpaceDataRelation *relation = workspace->hook_layout_relations.first;
2884              relation;
2885              relation = relation->next)
2886         {
2887                 relation->parent = newglobadr(fd, relation->parent); /* data from window - need to access through global oldnew-map */
2888                 relation->value = newdataadr(fd, relation->value);
2889         }
2890
2891         if (ID_IS_LINKED(&workspace->id)) {
2892                 /* Appending workspace so render layer is likely from a different scene. Unset
2893                  * now, when activating workspace later we set a valid one from current scene. */
2894                 BKE_workspace_relations_free(&workspace->scene_viewlayer_relations);
2895         }
2896
2897         /* Same issue/fix as in direct_link_workspace_link_scene_data: Can't read workspace data
2898          * when reading windows, so have to update windows after/when reading workspaces. */
2899         for (wmWindowManager *wm = main->wm.first; wm; wm = wm->id.next) {
2900                 for (wmWindow *win = wm->windows.first; win; win = win->next) {
2901                         WorkSpaceLayout *act_layout = newdataadr(fd, BKE_workspace_active_layout_get(win->workspace_hook));
2902                         if (act_layout) {
2903                                 BKE_workspace_active_layout_set(win->workspace_hook, act_layout);
2904                         }
2905                 }
2906         }
2907 }
2908
2909 static void lib_link_workspace_instance_hook(FileData *fd, WorkSpaceInstanceHook *hook, ID *id)
2910 {
2911         WorkSpace *workspace = BKE_workspace_active_get(hook);
2912         BKE_workspace_active_set(hook, newlibadr(fd, id->lib, workspace));
2913 }
2914
2915
2916 /* ************ READ MOTION PATHS *************** */
2917
2918 /* direct data for cache */
2919 static void direct_link_motionpath(FileData *fd, bMotionPath *mpath)
2920 {
2921         /* sanity check */
2922         if (mpath == NULL)
2923                 return;
2924         
2925         /* relink points cache */
2926         mpath->points = newdataadr(fd, mpath->points);
2927 }
2928
2929 /* ************ READ NODE TREE *************** */
2930
2931 /* Single node tree (also used for material/scene trees), ntree is not NULL */
2932 static void lib_link_ntree(FileData *fd, ID *id, bNodeTree *ntree)
2933 {
2934         bNode *node;
2935         bNodeSocket *sock;
2936         
2937         IDP_LibLinkProperty(ntree->id.properties, fd);
2938         lib_link_animdata(fd, &ntree->id, ntree->adt);
2939         
2940         ntree->gpd = newlibadr_us(fd, id->lib, ntree->gpd);
2941         
2942         for (node = ntree->nodes.first; node; node = node->next) {
2943                 /* Link ID Properties -- and copy this comment EXACTLY for easy finding
2944                  * of library blocks that implement this.*/
2945                 IDP_LibLinkProperty(node->prop, fd);
2946                 
2947                 node->id = newlibadr_us(fd, id->lib, node->id);
2948
2949                 for (sock = node->inputs.first; sock; sock = sock->next) {
2950                         IDP_LibLinkProperty(sock->prop, fd);
2951                 }
2952                 for (sock = node->outputs.first; sock; sock = sock->next) {
2953                         IDP_LibLinkProperty(sock->prop, fd);
2954                 }
2955         }
2956         
2957         for (sock = ntree->inputs.first; sock; sock = sock->next) {
2958                 IDP_LibLinkProperty(sock->prop, fd);
2959         }
2960         for (sock = ntree->outputs.first; sock; sock = sock->next) {
2961                 IDP_LibLinkProperty(sock->prop, fd);
2962         }
2963 }
2964
2965 /* library ntree linking after fileread */
2966 static void lib_link_nodetree(FileData *fd, Main *main)
2967 {
2968         /* only link ID pointers */
2969         for (bNodeTree *ntree = main->nodetree.first; ntree; ntree = ntree->id.next) {
2970                 if (ntree->id.tag & LIB_TAG_NEED_LINK) {
2971                         lib_link_ntree(fd, &ntree->id, ntree);
2972
2973                         ntree->id.tag &= ~LIB_TAG_NEED_LINK;
2974                 }
2975         }
2976 }
2977
2978 /* updates group node socket identifier so that
2979  * external links to/from the group node are preserved.
2980  */
2981 static void lib_node_do_versions_group_indices(bNode *gnode)
2982 {
2983         bNodeTree *ngroup = (bNodeTree*)gnode->id;
2984         bNodeSocket *sock;
2985         bNodeLink *link;
2986         
2987         for (sock=gnode->outputs.first; sock; sock = sock->next) {
2988                 int old_index = sock->to_index;
2989                 
2990                 for (link = ngroup->links.first; link; link = link->next) {
2991                         if (link->tonode == NULL && link->fromsock->own_index == old_index) {
2992                                 strcpy(sock->identifier, link->fromsock->identifier);
2993                                 /* deprecated */
2994                                 sock->own_index = link->fromsock->own_index;
2995                                 sock->to_index = 0;
2996                                 sock->groupsock = NULL;
2997                         }
2998                 }
2999         }
3000         for (sock=gnode->inputs.first; sock; sock = sock->next) {
3001                 int old_index = sock->to_index;
3002                 
3003                 for (link = ngroup->links.first; link; link = link->next) {
3004                         if (link->fromnode == NULL && link->tosock->own_index == old_index) {
3005                                 strcpy(sock->identifier, link->tosock->identifier);
3006                                 /* deprecated */
3007                                 sock->own_index = link->tosock->own_index;
3008                                 sock->to_index = 0;
3009                                 sock->groupsock = NULL;
3010                         }
3011                 }
3012         }
3013 }
3014
3015 /* verify types for nodes and groups, all data has to be read */
3016 /* open = 0: appending/linking, open = 1: open new file (need to clean out dynamic
3017  * typedefs */
3018 static void lib_verify_nodetree(Main *main, int UNUSED(open))
3019 {
3020         /* this crashes blender on undo/redo */
3021 #if 0
3022                 if (open == 1) {
3023                         reinit_nodesystem();
3024                 }
3025 #endif
3026         
3027         /* set node->typeinfo pointers */
3028         FOREACH_NODETREE(main, ntree, id) {
3029                 ntreeSetTypes(NULL, ntree);
3030         } FOREACH_NODETREE_END
3031         
3032         /* verify static socket templates */
3033         FOREACH_NODETREE(main, ntree, id) {
3034                 bNode *node;
3035                 for (node=ntree->nodes.first; node; node=node->next)
3036                         node_verify_socket_templates(ntree, node);
3037         } FOREACH_NODETREE_END
3038         
3039         {
3040                 bool has_old_groups = false;
3041                 /* XXX this should actually be part of do_versions, but since we need
3042                  * finished library linking, it is not possible there. Instead in do_versions
3043                  * we have set the NTREE_DO_VERSIONS_GROUP_EXPOSE_2_56_2 flag, so at this point we can do the
3044                  * actual group node updates.
3045                  */
3046                 for (bNodeTree *ntree = main->nodetree.first; ntree; ntree = ntree->id.next) {
3047                         if (ntree->flag & NTREE_DO_VERSIONS_GROUP_EXPOSE_2_56_2) {
3048                                 has_old_groups = 1;
3049                         }
3050                 }
3051                 
3052                 if (has_old_groups) {
3053                         FOREACH_NODETREE(main, ntree, id) {
3054                                 /* updates external links for all group nodes in a tree */
3055                                 bNode *node;
3056                                 for (node = ntree->nodes.first; node; node = node->next) {
3057                                         if (node->type == NODE_GROUP) {
3058                                                 bNodeTree *ngroup = (bNodeTree*)node->id;
3059                                                 if (ngroup && (ngroup->flag & NTREE_DO_VERSIONS_GROUP_EXPOSE_2_56_2))
3060                                                         lib_node_do_versions_group_indices(node);
3061                                         }
3062                                 }
3063                         } FOREACH_NODETREE_END
3064                 }
3065                 
3066                 for (bNodeTree *ntree = main->nodetree.first; ntree; ntree = ntree->id.next) {
3067                         ntree->flag &= ~NTREE_DO_VERSIONS_GROUP_EXPOSE_2_56_2;
3068                 }
3069         }
3070         
3071         {
3072                 /* Convert the previously used ntree->inputs/ntree->outputs lists to interface nodes.
3073                  * Pre 2.56.2 node trees automatically have all unlinked sockets exposed already
3074                  * (see NTREE_DO_VERSIONS_GROUP_EXPOSE_2_56_2).
3075                  *
3076                  * XXX this should actually be part of do_versions,
3077                  * but needs valid typeinfo pointers to create interface nodes.
3078                  *
3079                  * Note: theoretically only needed in node groups (main->nodetree),
3080                  * but due to a temporary bug such links could have been added in all trees,
3081                  * so have to clean up all of them ...
3082                  */
3083                 
3084                 FOREACH_NODETREE(main, ntree, id) {
3085                         if (ntree->flag & NTREE_DO_VERSIONS_CUSTOMNODES_GROUP) {
3086                                 bNode *input_node = NULL, *output_node = NULL;
3087                                 int num_inputs = 0, num_outputs = 0;
3088                                 bNodeLink *link, *next_link;
3089                                 /* Only create new interface nodes for actual older files.
3090                                  * New file versions already have input/output nodes with duplicate links,
3091                                  * in that case just remove the invalid links.
3092                                  */
3093                                 const bool create_io_nodes = (ntree->flag & NTREE_DO_VERSIONS_CUSTOMNODES_GROUP_CREATE_INTERFACE) != 0;
3094                                 
3095                                 float input_locx = 1000000.0f, input_locy = 0.0f;
3096                                 float output_locx = -1000000.0f, output_locy = 0.0f;
3097                                 /* rough guess, not nice but we don't have access to UI constants here ... */
3098                                 static const float offsetx = 42 + 3*20 + 20;
3099                                 /*static const float offsety = 0.0f;*/
3100                                 
3101                                 if (create_io_nodes) {
3102                                         if (ntree->inputs.first)
3103                                                 input_node = nodeAddStaticNode(NULL, ntree, NODE_GROUP_INPUT);
3104                                         
3105                                         if (ntree->outputs.first)
3106                                                 output_node = nodeAddStaticNode(NULL, ntree, NODE_GROUP_OUTPUT);
3107                                 }
3108                                 
3109                                 /* Redirect links from/to the node tree interface to input/output node.
3110                                  * If the fromnode/tonode pointers are NULL, this means a link from/to
3111                                  * the ntree interface sockets, which need to be redirected to new interface nodes.
3112                                  */
3113                                 for (link = ntree->links.first; link; link = next_link) {
3114                                         bool free_link = false;
3115                                         next_link = link->next;
3116                                         
3117                                         if (link->fromnode == NULL) {
3118                                                 if (input_node) {
3119                                                         link->fromnode = input_node;
3120                                                         link->fromsock = node_group_input_find_socket(input_node, link->fromsock->identifier);
3121                                                         ++num_inputs;
3122                                                         
3123                                                         if (link->tonode) {
3124                                                                 if (input_locx > link->tonode->locx - offsetx)
3125                                                                         input_locx = link->tonode->locx - offsetx;
3126                                                                 input_locy += link->tonode->locy;
3127                                                         }
3128                                                 }
3129                                                 else {
3130                                                         free_link = true;
3131                                                 }
3132                                         }
3133                                         
3134                                         if (link->tonode == NULL) {
3135                                                 if (output_node) {
3136                                                         link->tonode = output_node;
3137                                                         link->tosock = node_group_output_find_socket(output_node, link->tosock->identifier);
3138                                                         ++num_outputs;
3139                                                         
3140                                                         if (link->fromnode) {
3141                                                                 if (output_locx < link->fromnode->locx + offsetx)
3142                                                                         output_locx = link->fromnode->locx + offsetx;
3143                                                                 output_locy += link->fromnode->locy;
3144                                                         }
3145                                                 }
3146                                                 else {
3147                                                         free_link = true;
3148                                                 }
3149                                         }
3150                                         
3151                                         if (free_link)
3152                                                 nodeRemLink(ntree, link);
3153                                 }
3154                                 
3155                                 if (num_inputs > 0) {
3156                                         input_locy /= num_inputs;
3157                                         input_node->locx = input_locx;
3158                                         input_node->locy = input_locy;
3159                                 }
3160                                 if (num_outputs > 0) {
3161                                         output_locy /= num_outputs;
3162                                         output_node->locx = output_locx;
3163                                         output_node->locy = output_locy;
3164                                 }
3165                                 
3166                                 /* clear do_versions flags */
3167                                 ntree->flag &= ~(NTREE_DO_VERSIONS_CUSTOMNODES_GROUP | NTREE_DO_VERSIONS_CUSTOMNODES_GROUP_CREATE_INTERFACE);
3168                         }
3169                 }
3170                 FOREACH_NODETREE_END
3171         }
3172         
3173         /* verify all group user nodes */
3174         for (bNodeTree *ntree = main->nodetree.first; ntree; ntree = ntree->id.next) {
3175                 ntreeVerifyNodes(main, &ntree->id);
3176         }
3177         
3178         /* make update calls where necessary */
3179         {
3180                 FOREACH_NODETREE(main, ntree, id) {
3181                         /* make an update call for the tree */
3182                         ntreeUpdateTree(main, ntree);
3183                 } FOREACH_NODETREE_END
3184         }
3185 }
3186
3187 static void direct_link_node_socket(FileData *fd, bNodeSocket *sock)
3188 {
3189    &nb