doxygen: add newline after \file
[blender.git] / source / blender / blenloader / intern / readfile.c
index 229d0d044233fac1fdabe873a996317ac5c1e5e1..8fcae12a4fdffbf00986b8b130a0220838bf7d37 100644 (file)
@@ -1,6 +1,4 @@
 /*
- * ***** BEGIN GPL LICENSE BLOCK *****
- *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version 2
  *
  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
  * All rights reserved.
- *
- *
- * Contributor(s): Blender Foundation
- *
- * ***** END GPL LICENSE BLOCK *****
- *
  */
 
-/** \file blender/blenloader/intern/readfile.c
- *  \ingroup blenloader
+/** \file
+ * \ingroup blenloader
  */
 
 
 #include "BLI_math.h"
 #include "BLI_threads.h"
 #include "BLI_mempool.h"
+#include "BLI_ghash.h"
 
 #include "BLT_translation.h"
 
 #include "BKE_collection.h"
 #include "BKE_colortools.h"
 #include "BKE_constraint.h"
-#include "BKE_context.h"
 #include "BKE_curve.h"
 #include "BKE_effect.h"
 #include "BKE_fcurve.h"
 #include "NOD_common.h"
 #include "NOD_socket.h"
 
+#include "BLO_blend_defs.h"
+#include "BLO_blend_validate.h"
 #include "BLO_readfile.h"
 #include "BLO_undofile.h"
-#include "BLO_blend_defs.h"
 
 #include "RE_engine.h"
 
 #include "readfile.h"
 
-
 #include <errno.h>
 
 /**
 #  define DEBUG_PRINTF(...)
 #endif
 
-/***/
-
-typedef struct OldNew {
-       const void *old;
-       void *newp;
-       int nr;
-} OldNew;
-
-typedef struct OldNewMap {
-       OldNew *entries;
-       int nentries, entriessize;
-       bool sorted;
-       int lasthit;
-} OldNewMap;
-
 
 /* local prototypes */
 static void *read_struct(FileData *fd, BHead *bh, const char *blockname);
@@ -306,175 +283,155 @@ static const char *library_parent_filepath(Library *lib)
        return lib->parent ? lib->parent->filepath : "<direct>";
 }
 
-static OldNewMap *oldnewmap_new(void)
-{
-       OldNewMap *onm = MEM_callocN(sizeof(*onm), "OldNewMap");
 
-       onm->entriessize = 1024;
-       onm->entries = MEM_malloc_arrayN(onm->entriessize, sizeof(*onm->entries), "OldNewMap.entries");
+/* ************** OldNewMap ******************* */
 
-       return onm;
-}
+typedef struct OldNew {
+       const void *oldp;
+       void *newp;
+       /* `nr` is "user count" for data, and ID code for libdata. */
+       int nr;
+} OldNew;
 
-static int verg_oldnewmap(const void *v1, const void *v2)
-{
-       const struct OldNew *x1 = v1, *x2 = v2;
+typedef struct OldNewMap {
+       /* Array that stores the actual entries. */
+       OldNew *entries;
+       int nentries;
+       /* Hashmap that stores indices into the `entries` array. */
+       int32_t *map;
 
-       if (x1->old > x2->old) return 1;
-       else if (x1->old < x2->old) return -1;
-       return 0;
-}
+       int capacity_exp;
+} OldNewMap;
 
+#define ENTRIES_CAPACITY(onm) (1 << (onm)->capacity_exp)
+#define MAP_CAPACITY(onm) (1 << ((onm)->capacity_exp + 1))
+#define SLOT_MASK(onm) (MAP_CAPACITY(onm) - 1)
+#define DEFAULT_SIZE_EXP 6
+#define PERTURB_SHIFT 5
+
+/* based on the probing algorithm used in Python dicts. */
+#define ITER_SLOTS(onm, KEY, SLOT_NAME, INDEX_NAME) \
+       uint32_t hash = BLI_ghashutil_ptrhash(KEY); \
+       uint32_t mask = SLOT_MASK(onm); \
+       uint perturb = hash; \
+       int SLOT_NAME = mask & hash; \
+       int INDEX_NAME = onm->map[SLOT_NAME]; \
+       for (;;SLOT_NAME = mask & ((5 * SLOT_NAME) + 1 + perturb), perturb >>= PERTURB_SHIFT, INDEX_NAME = onm->map[SLOT_NAME])
+
+static void oldnewmap_insert_index_in_map(OldNewMap *onm, const void *ptr, int index)
+{
+       ITER_SLOTS(onm, ptr, slot, stored_index) {
+               if (stored_index == -1) {
+                       onm->map[slot] = index;
+                       break;
+               }
+       }
+}
 
-static void oldnewmap_sort(FileData *fd)
+static void oldnewmap_insert_or_replace(OldNewMap *onm, OldNew entry)
 {
-       BLI_assert(fd->libmap->sorted == false);
-       qsort(fd->libmap->entries, fd->libmap->nentries, sizeof(OldNew), verg_oldnewmap);
-       fd->libmap->sorted = 1;
+       ITER_SLOTS(onm, entry.oldp, slot, index) {
+               if (index == -1) {
+                       onm->entries[onm->nentries] = entry;
+                       onm->map[slot] = onm->nentries;
+                       onm->nentries++;
+                       break;
+               }
+               else if (onm->entries[index].oldp == entry.oldp) {
+                       onm->entries[index] = entry;
+                       break;
+               }
+       }
 }
 
-/* nr is zero for data, and ID code for libdata */
-static void oldnewmap_insert(OldNewMap *onm, const void *oldaddr, void *newaddr, int nr)
+static OldNew *oldnewmap_lookup_entry(const OldNewMap *onm, const void *addr)
 {
-       OldNew *entry;
-
-       if (oldaddr == NULL || newaddr == NULL) return;
-
-       if (UNLIKELY(onm->nentries == onm->entriessize)) {
-               onm->entriessize *= 2;
-               onm->entries = MEM_reallocN(onm->entries, sizeof(*onm->entries) * onm->entriessize);
+       ITER_SLOTS(onm, addr, slot, index) {
+               if (index >= 0) {
+                       OldNew *entry = &onm->entries[index];
+                       if (entry->oldp == addr) {
+                               return entry;
+                       }
+               }
+               else {
+                       return NULL;
+               }
        }
-
-       entry = &onm->entries[onm->nentries++];
-       entry->old = oldaddr;
-       entry->newp = newaddr;
-       entry->nr = nr;
 }
 
-void blo_do_versions_oldnewmap_insert(OldNewMap *onm, const void *oldaddr, void *newaddr, int nr)
+static void oldnewmap_clear_map(OldNewMap *onm)
 {
-       oldnewmap_insert(onm, oldaddr, newaddr, nr);
+       memset(onm->map, 0xFF, MAP_CAPACITY(onm) * sizeof(*onm->map));
 }
 
-/**
- * Do a full search (no state).
- *
- * \param lasthit: Use as a reference position to avoid a full search
- * from either end of the array, giving more efficient lookups.
- *
- * \note This would seem an ideal case for hash or btree lookups.
- * However the data is written in-order, using the \a lasthit will normally avoid calling this function.
- * Creating a btree/hash structure adds overhead for the common-case to optimize the corner-case
- * (since most entries will never be retrieved).
- * So just keep full lookups as a fall-back.
- */
-static int oldnewmap_lookup_entry_full(const OldNewMap *onm, const void *addr, int lasthit)
+static void oldnewmap_increase_size(OldNewMap *onm)
 {
-       const int nentries = onm->nentries;
-       const OldNew *entries = onm->entries;
-       int i;
-
-       /* search relative to lasthit where possible */
-       if (lasthit >= 0 && lasthit < nentries) {
-
-               /* search forwards */
-               i = lasthit;
-               while (++i != nentries) {
-                       if (entries[i].old == addr) {
-                               return i;
-                       }
-               }
-
-               /* search backwards */
-               i = lasthit + 1;
-               while (i--) {
-                       if (entries[i].old == addr) {
-                               return i;
-                       }
-               }
-       }
-       else {
-               /* search backwards (full) */
-               i = nentries;
-               while (i--) {
-                       if (entries[i].old == addr) {
-                               return i;
-                       }
-               }
+       onm->capacity_exp++;
+       onm->entries = MEM_reallocN(onm->entries, sizeof(*onm->entries) * ENTRIES_CAPACITY(onm));
+       onm->map = MEM_reallocN(onm->map, sizeof(*onm->map) * MAP_CAPACITY(onm));
+       oldnewmap_clear_map(onm);
+       for (int i = 0; i < onm->nentries; i++) {
+               oldnewmap_insert_index_in_map(onm, onm->entries[i].oldp, i);
        }
-
-       return -1;
 }
 
-static void *oldnewmap_lookup_and_inc(OldNewMap *onm, const void *addr, bool increase_users)
+
+/* Public OldNewMap API */
+
+static OldNewMap *oldnewmap_new(void)
 {
-       int i;
+       OldNewMap *onm = MEM_callocN(sizeof(*onm), "OldNewMap");
 
-       if (addr == NULL) return NULL;
+       onm->capacity_exp = DEFAULT_SIZE_EXP;
+       onm->entries = MEM_malloc_arrayN(ENTRIES_CAPACITY(onm), sizeof(*onm->entries), "OldNewMap.entries");
+       onm->map = MEM_malloc_arrayN(MAP_CAPACITY(onm), sizeof(*onm->map), "OldNewMap.map");
+       oldnewmap_clear_map(onm);
 
-       if (onm->lasthit < onm->nentries - 1) {
-               OldNew *entry = &onm->entries[++onm->lasthit];
+       return onm;
+}
 
-               if (entry->old == addr) {
-                       if (increase_users)
-                               entry->nr++;
-                       return entry->newp;
-               }
-       }
+static void oldnewmap_insert(OldNewMap *onm, const void *oldaddr, void *newaddr, int nr)
+{
+       if (oldaddr == NULL || newaddr == NULL) return;
 
-       i = oldnewmap_lookup_entry_full(onm, addr, onm->lasthit);
-       if (i != -1) {
-               OldNew *entry = &onm->entries[i];
-               BLI_assert(entry->old == addr);
-               onm->lasthit = i;
-               if (increase_users)
-                       entry->nr++;
-               return entry->newp;
+       if (UNLIKELY(onm->nentries == ENTRIES_CAPACITY(onm))) {
+               oldnewmap_increase_size(onm);
        }
 
-       return NULL;
+       OldNew entry;
+       entry.oldp = oldaddr;
+       entry.newp = newaddr;
+       entry.nr = nr;
+       oldnewmap_insert_or_replace(onm, entry);
 }
 
-/* for libdata, nr has ID code, no increment */
-static void *oldnewmap_liblookup(OldNewMap *onm, const void *addr, const void *lib)
+void blo_do_versions_oldnewmap_insert(OldNewMap *onm, const void *oldaddr, void *newaddr, int nr)
 {
-       if (addr == NULL) {
-               return NULL;
-       }
+       oldnewmap_insert(onm, oldaddr, newaddr, nr);
+}
 
-       /* lasthit works fine for non-libdata, linking there is done in same sequence as writing */
-       if (onm->sorted) {
-               const OldNew entry_s = {.old = addr};
-               OldNew *entry = bsearch(&entry_s, onm->entries, onm->nentries, sizeof(OldNew), verg_oldnewmap);
-               if (entry) {
-                       ID *id = entry->newp;
+static void *oldnewmap_lookup_and_inc(OldNewMap *onm, const void *addr, bool increase_users)
+{
+       OldNew *entry = oldnewmap_lookup_entry(onm, addr);
+       if (entry == NULL) return NULL;
+       if (increase_users) entry->nr++;
+       return entry->newp;
+}
 
-                       if (id && (!lib || id->lib)) {
-                               return id;
-                       }
-               }
-       }
-       else {
-               /* note, this can be a bottle neck when loading some files */
-               const int i = oldnewmap_lookup_entry_full(onm, addr, -1);
-               if (i != -1) {
-                       OldNew *entry = &onm->entries[i];
-                       ID *id = entry->newp;
-                       BLI_assert(entry->old == addr);
-                       if (id && (!lib || id->lib)) {
-                               return id;
-                       }
-               }
-       }
+/* for libdata, OldNew.nr has ID code, no increment */
+static void *oldnewmap_liblookup(OldNewMap *onm, const void *addr, const void *lib)
+{
+       if (addr == NULL) return NULL;
 
+       ID *id = oldnewmap_lookup_and_inc(onm, addr, false);
+       if (id == NULL) return NULL;
+       if (!lib || id->lib) return id;
        return NULL;
 }
 
 static void oldnewmap_free_unused(OldNewMap *onm)
 {
-       int i;
-
-       for (i = 0; i < onm->nentries; i++) {
+       for (int i = 0; i < onm->nentries; i++) {
                OldNew *entry = &onm->entries[i];
                if (entry->nr == 0) {
                        MEM_freeN(entry->newp);
@@ -485,16 +442,25 @@ static void oldnewmap_free_unused(OldNewMap *onm)
 
 static void oldnewmap_clear(OldNewMap *onm)
 {
+       onm->capacity_exp = DEFAULT_SIZE_EXP;
+       oldnewmap_clear_map(onm);
        onm->nentries = 0;
-       onm->lasthit = 0;
 }
 
 static void oldnewmap_free(OldNewMap *onm)
 {
        MEM_freeN(onm->entries);
+       MEM_freeN(onm->map);
        MEM_freeN(onm);
 }
 
+#undef ENTRIES_CAPACITY
+#undef MAP_CAPACITY
+#undef SLOT_MASK
+#undef DEFAULT_SIZE_EXP
+#undef PERTURB_SHIFT
+#undef ITER_SLOTS
+
 /***/
 
 static void read_libraries(FileData *basefd, ListBase *mainlist);
@@ -967,13 +933,30 @@ static void decode_blender_header(FileData *fd)
 static bool read_file_dna(FileData *fd, const char **r_error_message)
 {
        BHead *bhead;
+       int subversion = 0;
 
        for (bhead = blo_firstbhead(fd); bhead; bhead = blo_nextbhead(fd, bhead)) {
-               if (bhead->code == DNA1) {
+               if (bhead->code == GLOB) {
+                       /* Before this, the subversion didn't exist in 'FileGlobal' so the subversion
+                        * value isn't accessible for the purpose of DNA versioning in this case. */
+                       if (fd->fileversion <= 242) {
+                               continue;
+                       }
+                       /* We can't use read_global because this needs 'DNA1' to be decoded,
+                        * however the first 4 chars are _always_ the subversion. */
+                       FileGlobal *fg = (void *)&bhead[1];
+                       BLI_STATIC_ASSERT(offsetof(FileGlobal, subvstr) == 0, "Must be first: subvstr");
+                       char num[5];
+                       memcpy(num, fg->subvstr, 4);
+                       num[4] = 0;
+                       subversion = atoi(num);
+               }
+               else if (bhead->code == DNA1) {
                        const bool do_endian_swap = (fd->flags & FD_FLAGS_SWITCH_ENDIAN) != 0;
 
                        fd->filesdna = DNA_sdna_from_data(&bhead[1], bhead->len, do_endian_swap, true, r_error_message);
                        if (fd->filesdna) {
+                               blo_do_versions_dna(fd->filesdna, fd->fileversion, subversion);
                                fd->compflags = DNA_struct_get_compareflags(fd->filesdna, fd->memsdna);
                                /* used to retrieve ID names from (bhead+1) */
                                fd->id_name_offs = DNA_elem_offset(fd->filesdna, "ID", "char", "name[]");
@@ -1012,10 +995,9 @@ static int *read_file_thumbnail(FileData *fd)
                                BLI_endian_switch_int32(&data[1]);
                        }
 
-                       int width = data[0];
-                       int height = data[1];
-
-                       if (!BLEN_THUMB_SAFE_MEMSIZE(width, height)) {
+                       const int width = data[0];
+                       const int height = data[1];
+                       if (!BLEN_THUMB_MEMSIZE_IS_VALID(width, height)) {
                                break;
                        }
                        if (bhead->len < BLEN_THUMB_MEMSIZE_FILE(width, height)) {
@@ -1362,7 +1344,7 @@ void blo_freefiledata(FileData *fd)
 /**
  * Check whether given path ends with a blend file compatible extension (.blend, .ble or .blend.gz).
  *
- * \param str The path to check.
+ * \param str: The path to check.
  * \return true is this path ends with a blender file extension.
  */
 bool BLO_has_bfile_extension(const char *str)
@@ -1374,11 +1356,11 @@ bool BLO_has_bfile_extension(const char *str)
 /**
  * Try to explode given path into its 'library components' (i.e. a .blend file, id type/group, and datablock itself).
  *
- * \param path the full path to explode.
- * \param r_dir the string that'll contain path up to blend file itself ('library' path).
+ * \param path: the full path to explode.
+ * \param r_dir: the string that'll contain path up to blend file itself ('library' path).
  *              WARNING! Must be FILE_MAX_LIBEXTRA long (it also stores group and name strings)!
- * \param r_group the string that'll contain 'group' part of the path, if any. May be NULL.
- * \param r_name the string that'll contain data's name part of the path, if any. May be NULL.
+ * \param r_group: the string that'll contain 'group' part of the path, if any. May be NULL.
+ * \param r_name: the string that'll contain data's name part of the path, if any. May be NULL.
  * \return true if path contains a blend file.
  */
 bool BLO_library_path_explode(const char *path, char *r_dir, char **r_group, char **r_name)
@@ -1443,7 +1425,7 @@ bool BLO_library_path_explode(const char *path, char *r_dir, char **r_group, cha
 /**
  * Does a very light reading of given .blend file to extract its stored thumbnail.
  *
- * \param filepath The path of the file to extract thumbnail from.
+ * \param filepath: The path of the file to extract thumbnail from.
  * \return The raw thumbnail
  *         (MEM-allocated, as stored in file, use BKE_main_thumbnail_to_imbuf() to convert it to ImBuf image).
  */
@@ -1457,14 +1439,11 @@ BlendThumbnail *BLO_thumbnail_from_file(const char *filepath)
        fd_data = fd ? read_file_thumbnail(fd) : NULL;
 
        if (fd_data) {
-               int width = fd_data[0];
-               int height = fd_data[1];
-
-               /* Protect against buffer overflow vulnerability. */
-               if (BLEN_THUMB_SAFE_MEMSIZE(width, height)) {
+               const int width = fd_data[0];
+               const int height = fd_data[1];
+               if (BLEN_THUMB_MEMSIZE_IS_VALID(width, height)) {
                        const size_t sz = BLEN_THUMB_MEMSIZE(width, height);
                        data = MEM_mallocN(sz, __func__);
-
                        if (data) {
                                BLI_assert((sz - sizeof(*data)) == (BLEN_THUMB_MEMSIZE_FILE(width, height) - (sizeof(*fd_data) * 2)));
                                data->width = width;
@@ -1486,31 +1465,6 @@ static void *newdataadr(FileData *fd, const void *adr)      /* only direct datab
        return oldnewmap_lookup_and_inc(fd->datamap, adr, true);
 }
 
-/* This is a special version of newdataadr() which allows us to keep lasthit of
- * map unchanged. In certain cases this makes file loading time significantly
- * faster.
- *
- * Use this function in cases like restoring pointer from one list element to
- * another list element, but keep lasthit value so we can continue restoring
- * pointers efficiently.
- *
- * Example of this could be found in direct_link_fcurves() which restores the
- * fcurve group pointer and keeps lasthit optimal for linking all further
- * fcurves.
- */
-static void *newdataadr_ex(FileData *fd, const void *adr, bool increase_lasthit)        /* only direct databocks */
-{
-       if (increase_lasthit) {
-               return newdataadr(fd, adr);
-       }
-       else {
-               int lasthit = fd->datamap->lasthit;
-               void *newadr = newdataadr(fd, adr);
-               fd->datamap->lasthit = lasthit;
-               return newadr;
-       }
-}
-
 static void *newdataadr_no_us(FileData *fd, const void *adr)        /* only direct databocks */
 {
        return oldnewmap_lookup_and_inc(fd->datamap, adr, false);
@@ -1593,12 +1547,7 @@ static void *newlibadr_real_us(FileData *fd, const void *lib, const void *adr)
 
 static void change_idid_adr_fd(FileData *fd, const void *old, void *new)
 {
-       int i;
-
-       /* use a binary search if we have a sorted libmap, for now it's not needed. */
-       BLI_assert(fd->libmap->sorted == false);
-
-       for (i = 0; i < fd->libmap->nentries; i++) {
+       for (int i = 0; i < fd->libmap->nentries; i++) {
                OldNew *entry = &fd->libmap->entries[i];
 
                if (old == entry->newp && entry->nr == ID_ID) {
@@ -1629,14 +1578,17 @@ static void change_idid_adr(ListBase *mainlist, FileData *basefd, void *old, voi
 /* lib linked proxy objects point to our local data, we need
  * to clear that pointer before reading the undo memfile since
  * the object might be removed, it is set again in reading
- * if the local object still exists */
+ * if the local object still exists.
+ * This is only valid for local proxy objects though, linked ones should not be affected here.
+ */
 void blo_clear_proxy_pointers_from_lib(Main *oldmain)
 {
        Object *ob = oldmain->object.first;
 
        for (; ob; ob = ob->id.next) {
-               if (ob->id.lib)
+               if (ob->id.lib != NULL && ob->proxy_from != NULL && ob->proxy_from->id.lib == NULL) {
                        ob->proxy_from = NULL;
+               }
        }
 }
 
@@ -2669,7 +2621,7 @@ static void direct_link_fcurves(FileData *fd, ListBase *list)
                fcu->rna_path = newdataadr(fd, fcu->rna_path);
 
                /* group */
-               fcu->grp = newdataadr_ex(fd, fcu->grp, false);
+               fcu->grp = newdataadr(fd, fcu->grp);
 
                /* clear disabled flag - allows disabled drivers to be tried again ([#32155]),
                 * but also means that another method for "reviving disabled F-Curves" exists
@@ -3607,7 +3559,7 @@ static void lib_link_pose(FileData *fd, Main *bmain, Object *ob, bPose *pose)
 
 
        if (rebuild) {
-               DEG_id_tag_update_ex(bmain, &ob->id, OB_RECALC_OB | OB_RECALC_DATA | OB_RECALC_TIME);
+               DEG_id_tag_update_ex(bmain, &ob->id, ID_RECALC_TRANSFORM | ID_RECALC_GEOMETRY | ID_RECALC_ANIMATION);
                BKE_pose_tag_recalc(bmain, pose);
        }
 }
@@ -3999,6 +3951,12 @@ static void direct_link_image(FileData *fd, Image *ima)
                }
                ima->rr = NULL;
        }
+       else {
+               for (int i = 0; i < TEXTARGET_COUNT; i++) {
+                       ima->gputexture[i] = newimaadr(fd, ima->gputexture[i]);
+               }
+               ima->rr = newimaadr(fd, ima->rr);
+       }
 
        /* undo system, try to restore render buffers */
        link_list(fd, &(ima->renderslots));
@@ -4233,7 +4191,7 @@ static const char *ptcache_data_struct[] = {
        "", // BPHYS_DATA_AVELOCITY / BPHYS_DATA_XCONST */
        "", // BPHYS_DATA_SIZE:
        "", // BPHYS_DATA_TIMES:
-       "BoidData" // case BPHYS_DATA_BOIDS:
+       "BoidData", // case BPHYS_DATA_BOIDS:
 };
 
 static void direct_link_pointcache_cb(FileData *fd, void *data)
@@ -4319,8 +4277,8 @@ static void lib_link_particlesettings(FileData *fd, Main *main)
 
                        part->ipo = newlibadr_us(fd, part->id.lib, part->ipo); // XXX deprecated - old animation system
 
-                       part->dup_ob = newlibadr(fd, part->id.lib, part->dup_ob);
-                       part->dup_group = newlibadr(fd, part->id.lib, part->dup_group);
+                       part->instance_object = newlibadr(fd, part->id.lib, part->instance_object);
+                       part->instance_collection = newlibadr_us(fd, part->id.lib, part->instance_collection);
                        part->eff_group = newlibadr(fd, part->id.lib, part->eff_group);
                        part->bb_ob = newlibadr(fd, part->id.lib, part->bb_ob);
                        part->collision_group = newlibadr(fd, part->id.lib, part->collision_group);
@@ -4332,16 +4290,16 @@ static void lib_link_particlesettings(FileData *fd, Main *main)
                                part->effector_weights->group = newlibadr(fd, part->id.lib, part->effector_weights->group);
                        }
                        else {
-                               part->effector_weights = BKE_add_effector_weights(part->eff_group);
+                               part->effector_weights = BKE_effector_add_weights(part->eff_group);
                        }
 
-                       if (part->dupliweights.first && part->dup_group) {
-                               for (ParticleDupliWeight *dw = part->dupliweights.first; dw; dw = dw->next) {
+                       if (part->instance_weights.first && part->instance_collection) {
+                               for (ParticleDupliWeight *dw = part->instance_weights.first; dw; dw = dw->next) {
                                        dw->ob = newlibadr(fd, part->id.lib, dw->ob);
                                }
                        }
                        else {
-                               BLI_listbase_clear(&part->dupliweights);
+                               BLI_listbase_clear(&part->instance_weights);
                        }
 
                        if (part->boids) {
@@ -4411,9 +4369,9 @@ static void direct_link_particlesettings(FileData *fd, ParticleSettings *part)
 
        part->effector_weights = newdataadr(fd, part->effector_weights);
        if (!part->effector_weights)
-               part->effector_weights = BKE_add_effector_weights(part->eff_group);
+               part->effector_weights = BKE_effector_add_weights(part->eff_group);
 
-       link_list(fd, &part->dupliweights);
+       link_list(fd, &part->instance_weights);
 
        part->boids = newdataadr(fd, part->boids);
        part->fluid = newdataadr(fd, part->fluid);
@@ -4760,7 +4718,7 @@ static void direct_link_mesh(FileData *fd, Mesh *mesh)
        direct_link_customdata(fd, &mesh->pdata, mesh->totpoly);
 
        mesh->bb = NULL;
-       mesh->edit_btmesh = NULL;
+       mesh->edit_mesh = NULL;
        BKE_mesh_runtime_reset(mesh);
 
        /* happens with old files */
@@ -4926,10 +4884,10 @@ static void lib_link_object(FileData *fd, Main *main)
 
                        /* 2.8x drops support for non-empty dupli instances. */
                        if (ob->type == OB_EMPTY) {
-                               ob->dup_group = newlibadr_us(fd, ob->id.lib, ob->dup_group);
+                               ob->instance_collection = newlibadr_us(fd, ob->id.lib, ob->instance_collection);
                        }
                        else {
-                               ob->dup_group = NULL;
+                               ob->instance_collection = NULL;
                                ob->transflag &= ~OB_DUPLICOLLECTION;
                        }
 
@@ -5129,6 +5087,7 @@ static void direct_link_modifiers(FileData *fd, ListBase *lb)
                        SubsurfModifierData *smd = (SubsurfModifierData *)md;
 
                        smd->emCache = smd->mCache = NULL;
+                       smd->subdiv = NULL;
                }
                else if (md->type == eModifierType_Armature) {
                        ArmatureModifierData *amd = (ArmatureModifierData *)md;
@@ -5155,7 +5114,7 @@ static void direct_link_modifiers(FileData *fd, ListBase *lb)
                                clmd->sim_parms->effector_weights = newdataadr(fd, clmd->sim_parms->effector_weights);
 
                                if (!clmd->sim_parms->effector_weights) {
-                                       clmd->sim_parms->effector_weights = BKE_add_effector_weights(NULL);
+                                       clmd->sim_parms->effector_weights = BKE_effector_add_weights(NULL);
                                }
                        }
 
@@ -5197,7 +5156,7 @@ static void direct_link_modifiers(FileData *fd, ListBase *lb)
 
                                smd->domain->effector_weights = newdataadr(fd, smd->domain->effector_weights);
                                if (!smd->domain->effector_weights)
-                                       smd->domain->effector_weights = BKE_add_effector_weights(NULL);
+                                       smd->domain->effector_weights = BKE_effector_add_weights(NULL);
 
                                direct_link_pointcache_list(fd, &(smd->domain->ptcaches[0]), &(smd->domain->point_cache[0]), 1);
 
@@ -5264,7 +5223,7 @@ static void direct_link_modifiers(FileData *fd, ListBase *lb)
                                                direct_link_pointcache_list(fd, &(surface->ptcaches), &(surface->pointcache), 1);
 
                                                if (!(surface->effector_weights = newdataadr(fd, surface->effector_weights)))
-                                                       surface->effector_weights = BKE_add_effector_weights(NULL);
+                                                       surface->effector_weights = BKE_effector_add_weights(NULL);
                                        }
                                }
                        }
@@ -5448,6 +5407,10 @@ static void direct_link_modifiers(FileData *fd, ListBase *lb)
                        BevelModifierData *bmd = (BevelModifierData *)md;
                        bmd->clnordata.faceHash = NULL;
                }
+               else if (md->type == eModifierType_Multires) {
+                       MultiresModifierData *mmd = (MultiresModifierData *)md;
+                       mmd->subdiv = NULL;
+               }
        }
 }
 
@@ -5625,7 +5588,7 @@ static void direct_link_object(FileData *fd, Object *ob)
 
                sb->effector_weights = newdataadr(fd, sb->effector_weights);
                if (!sb->effector_weights)
-                       sb->effector_weights = BKE_add_effector_weights(NULL);
+                       sb->effector_weights = BKE_effector_add_weights(NULL);
 
                sb->shared = newdataadr(fd, sb->shared);
                if (sb->shared == NULL) {
@@ -5692,8 +5655,7 @@ static void direct_link_object(FileData *fd, Object *ob)
                BKE_object_empty_draw_type_set(ob, ob->empty_drawtype);
        }
 
-       ob->customdata_mask = 0;
-       ob->bb = NULL;
+       ob->runtime.bb = NULL;
        ob->derivedDeform = NULL;
        ob->derivedFinal = NULL;
        BKE_object_runtime_reset(ob);
@@ -5703,12 +5665,10 @@ static void direct_link_object(FileData *fd, Object *ob)
        CLAMP(ob->rotmode, ROT_MODE_MIN, ROT_MODE_MAX);
 
        if (ob->sculpt) {
-               if (ob->mode & OB_MODE_ALL_SCULPT) {
-                       ob->sculpt = MEM_callocN(sizeof(SculptSession), "reload sculpt session");
-                       ob->sculpt->mode_type = ob->mode;
-               }
-               else {
-                       ob->sculpt = NULL;
+               ob->sculpt = NULL;
+               /* Only create data on undo, otherwise rely on editor mode switching. */
+               if (fd->memfile && (ob->mode & OB_MODE_ALL_SCULPT)) {
+                       BKE_object_sculpt_data_create(ob);
                }
        }
 
@@ -5813,6 +5773,8 @@ static void lib_link_view_layer(FileData *fd, Library *lib, ViewLayer *view_laye
                lib_link_layer_collection(fd, lib, layer_collection, true);
        }
 
+       view_layer->mat_override = newlibadr_us(fd, lib, view_layer->mat_override);
+
        IDP_LibLinkProperty(view_layer->id_properties, fd);
 }
 
@@ -6067,6 +6029,8 @@ static void lib_link_scene(FileData *fd, Main *main)
 
                        sce->toolsettings->particle.shape_object = newlibadr(fd, sce->id.lib, sce->toolsettings->particle.shape_object);
 
+                       sce->toolsettings->gp_sculpt.guide.reference_object = newlibadr(fd, sce->id.lib, sce->toolsettings->gp_sculpt.guide.reference_object);
+
                        for (Base *base_legacy_next, *base_legacy = sce->base.first; base_legacy; base_legacy = base_legacy_next) {
                                base_legacy_next = base_legacy->next;
 
@@ -6116,6 +6080,10 @@ static void lib_link_scene(FileData *fd, Main *main)
                                                seq->scene_sound = BKE_sound_add_scene_sound_defaults(sce, seq);
                                        }
                                }
+                               if (seq->type == SEQ_TYPE_TEXT) {
+                                       TextVars *t = seq->effectdata;
+                                       t->text_font = newlibadr_us(fd, sce->id.lib, t->text_font);
+                               }
                                BLI_listbase_clear(&seq->anims);
 
                                lib_link_sequence_modifiers(fd, sce, &seq->modifiers);
@@ -6175,6 +6143,10 @@ static void lib_link_scene(FileData *fd, Main *main)
                                lib_link_view_layer(fd, sce->id.lib, view_layer);
                        }
 
+                       if (sce->r.bake.cage_object) {
+                               sce->r.bake.cage_object = newlibadr(fd, sce->id.lib, sce->r.bake.cage_object);
+                       }
+
 #ifdef USE_SETSCENE_CHECK
                        if (sce->set != NULL) {
                                /* link flag for scenes with set would be reset later,
@@ -6328,6 +6300,11 @@ static void direct_link_scene(FileData *fd, Scene *sce)
                if (sce->toolsettings->gp_sculpt.cur_falloff) {
                        direct_link_curvemapping(fd, sce->toolsettings->gp_sculpt.cur_falloff);
                }
+               /* relink grease pencil primitive curve */
+               sce->toolsettings->gp_sculpt.cur_primitive = newdataadr(fd, sce->toolsettings->gp_sculpt.cur_primitive);
+               if (sce->toolsettings->gp_sculpt.cur_primitive) {
+                       direct_link_curvemapping(fd, sce->toolsettings->gp_sculpt.cur_primitive);
+               }
        }
 
        if (sce->ed) {
@@ -6360,6 +6337,11 @@ static void direct_link_scene(FileData *fd, Scene *sce)
                                s->frameMap = NULL;
                        }
 
+                       if (seq->type == SEQ_TYPE_TEXT) {
+                               TextVars *t = seq->effectdata;
+                               t->text_blf_id = SEQ_FONT_NOT_LOADED;
+                       }
+
                        seq->prop = newdataadr(fd, seq->prop);
                        IDP_DirectLinkGroup_OrFree(&seq->prop, (fd->flags & FD_FLAGS_SWITCH_ENDIAN), fd);
 
@@ -6516,7 +6498,7 @@ static void direct_link_scene(FileData *fd, Scene *sce)
                /* set effector weights */
                rbw->effector_weights = newdataadr(fd, rbw->effector_weights);
                if (!rbw->effector_weights)
-                       rbw->effector_weights = BKE_add_effector_weights(NULL);
+                       rbw->effector_weights = BKE_effector_add_weights(NULL);
        }
 
        sce->preview = direct_link_preview_image(fd, sce->preview);
@@ -6533,9 +6515,9 @@ static void direct_link_scene(FileData *fd, Scene *sce)
 
        if (sce->master_collection) {
                sce->master_collection = newdataadr(fd, sce->master_collection);
-               direct_link_collection(fd, sce->master_collection);
                /* Needed because this is an ID outside of Main. */
-               sce->master_collection->id.py_instance = NULL;
+               direct_link_id(fd, &sce->master_collection->id);
+               direct_link_collection(fd, sce->master_collection);
        }
 
        /* insert into global old-new map for reading without UI (link_global accesses it again) */
@@ -6550,9 +6532,9 @@ static void direct_link_scene(FileData *fd, Scene *sce)
                else sce->eevee.light_cache = NULL;
        }
        else {
-               /* else read the cache from file. */
+               /* else try to read the cache from file. */
+               sce->eevee.light_cache = newdataadr(fd, sce->eevee.light_cache);
                if (sce->eevee.light_cache) {
-                       sce->eevee.light_cache = newdataadr(fd, sce->eevee.light_cache);
                        direct_link_lightcache(fd, sce->eevee.light_cache);
                }
        }
@@ -6605,6 +6587,11 @@ static void direct_link_gpencil(FileData *fd, bGPdata *gpd)
        gpd->adt = newdataadr(fd, gpd->adt);
        direct_link_animdata(fd, gpd->adt);
 
+       /* init stroke buffer */
+       gpd->runtime.sbuffer = NULL;
+       gpd->runtime.sbuffer_size = 0;
+       gpd->runtime.tot_cp_points = 0;
+
        /* relink palettes (old palettes deprecated, only to convert old files) */
        link_list(fd, &gpd->palettes);
        if (gpd->palettes.first != NULL) {
@@ -6645,7 +6632,7 @@ static void direct_link_gpencil(FileData *fd, bGPdata *gpd)
                                /* the triangulation is not saved, so need to be recalculated */
                                gps->triangles = NULL;
                                gps->tot_triangles = 0;
-                               gps->flag |= GP_STROKE_RECALC_CACHES;
+                               gps->flag |= GP_STROKE_RECALC_GEOMETRY;
                        }
                }
        }
@@ -6805,11 +6792,11 @@ static void direct_link_area(FileData *fd, ScrArea *area)
 
                        blo_do_versions_view3d_split_250(v3d, &sl->regionbase);
                }
-               else if (sl->spacetype == SPACE_IPO) {
-                       SpaceIpo *sipo = (SpaceIpo *)sl;
+               else if (sl->spacetype == SPACE_GRAPH) {
+                       SpaceGraph *sipo = (SpaceGraph *)sl;
 
                        sipo->ads = newdataadr(fd, sipo->ads);
-                       BLI_listbase_clear(&sipo->ghostCurves);
+                       BLI_listbase_clear(&sipo->runtime.ghost_curves);
                }
                else if (sl->spacetype == SPACE_NLA) {
                        SpaceNla *snla = (SpaceNla *)sl;
@@ -6817,7 +6804,7 @@ static void direct_link_area(FileData *fd, ScrArea *area)
                        snla->ads = newdataadr(fd, snla->ads);
                }
                else if (sl->spacetype == SPACE_OUTLINER) {
-                       SpaceOops *soops = (SpaceOops *)sl;
+                       SpaceOutliner *soops = (SpaceOutliner *)sl;
 
                        /* use newdataadr_no_us and do not free old memory avoiding double
                         * frees and use of freed memory. this could happen because of a
@@ -6907,8 +6894,8 @@ static void direct_link_area(FileData *fd, ScrArea *area)
                        sseq->scopes.histogram_ibuf = NULL;
                        sseq->compositor = NULL;
                }
-               else if (sl->spacetype == SPACE_BUTS) {
-                       SpaceButs *sbuts = (SpaceButs *)sl;
+               else if (sl->spacetype == SPACE_PROPERTIES) {
+                       SpaceProperties *sbuts = (SpaceProperties *)sl;
 
                        sbuts->path = NULL;
                        sbuts->texuser = NULL;
@@ -6991,9 +6978,9 @@ static void lib_link_area(FileData *fd, ID *parent_id, ScrArea *area)
                                }
                                break;
                        }
-                       case SPACE_IPO:
+                       case SPACE_GRAPH:
                        {
-                               SpaceIpo *sipo = (SpaceIpo *)sl;
+                               SpaceGraph *sipo = (SpaceGraph *)sl;
                                bDopeSheet *ads = sipo->ads;
 
                                if (ads) {
@@ -7002,9 +6989,9 @@ static void lib_link_area(FileData *fd, ID *parent_id, ScrArea *area)
                                }
                                break;
                        }
-                       case SPACE_BUTS:
+                       case SPACE_PROPERTIES:
                        {
-                               SpaceButs *sbuts = (SpaceButs *)sl;
+                               SpaceProperties *sbuts = (SpaceProperties *)sl;
                                sbuts->pinid = newlibadr(fd, parent_id->lib, sbuts->pinid);
                                if (sbuts->pinid == NULL) {
                                        sbuts->flag &= ~SB_PIN_CONTEXT;
@@ -7081,7 +7068,7 @@ static void lib_link_area(FileData *fd, ID *parent_id, ScrArea *area)
                        }
                        case SPACE_OUTLINER:
                        {
-                               SpaceOops *so = (SpaceOops *)sl;
+                               SpaceOutliner *so = (SpaceOutliner *)sl;
                                so->search_tse.id = newlibadr(fd, NULL, so->search_tse.id);
 
                                if (so->treestore) {
@@ -7478,8 +7465,8 @@ static void lib_link_workspace_layout_restore(struct IDNameLib_Map *id_map, Main
                                                }
                                        }
                                }
-                               else if (sl->spacetype == SPACE_IPO) {
-                                       SpaceIpo *sipo = (SpaceIpo *)sl;
+                               else if (sl->spacetype == SPACE_GRAPH) {
+                                       SpaceGraph *sipo = (SpaceGraph *)sl;
                                        bDopeSheet *ads = sipo->ads;
 
                                        if (ads) {
@@ -7492,10 +7479,10 @@ static void lib_link_workspace_layout_restore(struct IDNameLib_Map *id_map, Main
                                        /* force recalc of list of channels (i.e. includes calculating F-Curve colors)
                                         * thus preventing the "black curves" problem post-undo
                                         */
-                                       sipo->flag |= SIPO_TEMP_NEEDCHANSYNC;
+                                       sipo->runtime.flag |= SIPO_RUNTIME_FLAG_NEED_CHAN_SYNC_COLOR;
                                }
-                               else if (sl->spacetype == SPACE_BUTS) {
-                                       SpaceButs *sbuts = (SpaceButs *)sl;
+                               else if (sl->spacetype == SPACE_PROPERTIES) {
+                                       SpaceProperties *sbuts = (SpaceProperties *)sl;
                                        sbuts->pinid = restore_pointer_by_name(id_map, sbuts->pinid, USER_IGNORE);
                                        if (sbuts->pinid == NULL) {
                                                sbuts->flag &= ~SB_PIN_CONTEXT;
@@ -7523,7 +7510,7 @@ static void lib_link_workspace_layout_restore(struct IDNameLib_Map *id_map, Main
                                        /* force recalc of list of channels, potentially updating the active action
                                         * while we're at it (as it can only be updated that way) [#28962]
                                         */
-                                       saction->flag |= SACTION_TEMP_NEEDCHANSYNC;
+                                       saction->runtime.flag |= SACTION_RUNTIME_FLAG_NEED_CHAN_SYNC;
                                }
                                else if (sl->spacetype == SPACE_IMAGE) {
                                        SpaceImage *sima = (SpaceImage *)sl;
@@ -7585,7 +7572,7 @@ static void lib_link_workspace_layout_restore(struct IDNameLib_Map *id_map, Main
                                        }
                                }
                                else if (sl->spacetype == SPACE_OUTLINER) {
-                                       SpaceOops *so = (SpaceOops *)sl;
+                                       SpaceOutliner *so = (SpaceOutliner *)sl;
 
                                        so->search_tse.id = restore_pointer_by_name(id_map, so->search_tse.id, USER_IGNORE);
 
@@ -7667,9 +7654,9 @@ static void lib_link_workspace_layout_restore(struct IDNameLib_Map *id_map, Main
  * Used to link a file (without UI) to the current UI.
  * Note that it assumes the old pointers in UI are still valid, so old Main is not freed.
  */
-void blo_lib_link_restore(Main *newmain, wmWindowManager *curwm, Scene *curscene, ViewLayer *cur_view_layer)
+void blo_lib_link_restore(Main *oldmain, Main *newmain, wmWindowManager *curwm, Scene *curscene, ViewLayer *cur_view_layer)
 {
-       struct IDNameLib_Map *id_map = BKE_main_idmap_create(newmain);
+       struct IDNameLib_Map *id_map = BKE_main_idmap_create(newmain, true, oldmain);
 
        for (WorkSpace *workspace = newmain->workspaces.first; workspace; workspace = workspace->id.next) {
                ListBase *layouts = BKE_workspace_layouts_get(workspace);
@@ -8719,7 +8706,7 @@ static BHead *read_libblock(FileData *fd, Main *main, BHead *bhead, const short
        oldnewmap_clear(fd->datamap);
 
        if (wrong_id) {
-               BKE_libblock_free(main, id);
+               BKE_id_free(main, id);
        }
 
        return (bhead);
@@ -8794,8 +8781,8 @@ static void do_versions_userdef(FileData *fd, BlendFileData *bfd)
 
                /* themes for Node and Sequence editor were not using grid color, but back. we copy this over then */
                for (btheme = user->themes.first; btheme; btheme = btheme->next) {
-                       copy_v4_v4_char(btheme->tnode.grid, btheme->tnode.back);
-                       copy_v4_v4_char(btheme->tseq.grid, btheme->tseq.back);
+                       copy_v4_v4_char(btheme->space_node.grid, btheme->space_node.back);
+                       copy_v4_v4_char(btheme->space_sequencer.grid, btheme->space_sequencer.back);
                }
        }
 
@@ -8862,8 +8849,6 @@ static void do_versions_after_linking(Main *main)
 
 static void lib_link_all(FileData *fd, Main *main)
 {
-       oldnewmap_sort(fd);
-
        lib_link_id(fd, main);
 
        /* No load UI for undo memfiles */
@@ -8907,6 +8892,10 @@ static void lib_link_all(FileData *fd, Main *main)
        lib_link_workspaces(fd, main);
 
        lib_link_library(fd, main);    /* only init users */
+
+       /* We could integrate that to mesh/curve/lattice lib_link, but this is really cheap process,
+        * so simpler to just use it directly in this single call. */
+       BLO_main_validate_shapekeys(main, NULL);
 }
 
 static void direct_link_keymapitem(FileData *fd, wmKeyMapItem *kmi)
@@ -9022,11 +9011,9 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
                const int *data = read_file_thumbnail(fd);
 
                if (data) {
-                       int width = data[0];
-                       int height = data[1];
-
-                       /* Protect against buffer overflow vulnerability. */
-                       if (BLEN_THUMB_SAFE_MEMSIZE(width, height)) {
+                       const int width = data[0];
+                       const int height = data[1];
+                       if (BLEN_THUMB_MEMSIZE_IS_VALID(width, height)) {
                                const size_t sz = BLEN_THUMB_MEMSIZE(width, height);
                                bfd->main->blen_thumb = MEM_mallocN(sz, __func__);
 
@@ -9112,15 +9099,17 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
 
        BKE_main_id_tag_all(bfd->main, LIB_TAG_NEW, false);
 
+       /* Before static overrides, which needs typeinfo. */
+       lib_verify_nodetree(bfd->main, true);
+
        /* Now that all our data-blocks are loaded, we can re-generate overrides from their references. */
        if (fd->memfile == NULL) {
                /* Do not apply in undo case! */
-               lib_verify_nodetree(bfd->main, true);  /* Needed to ensure we have typeinfo in nodes... */
                BKE_main_override_static_update(bfd->main);
-               BKE_collections_after_lib_link(bfd->main);
        }
 
-       lib_verify_nodetree(bfd->main, true);
+       BKE_collections_after_lib_link(bfd->main);
+
        fix_relpaths_library(fd->relabase, bfd->main); /* make all relative paths, relative to the open blend file */
 
        link_global(fd, bfd);   /* as last */
@@ -9511,8 +9500,8 @@ static void expand_particlesettings(FileData *fd, Main *mainvar, ParticleSetting
 {
        int a;
 
-       expand_doit(fd, mainvar, part->dup_ob);
-       expand_doit(fd, mainvar, part->dup_group);
+       expand_doit(fd, mainvar, part->instance_object);
+       expand_doit(fd, mainvar, part->instance_collection);
        expand_doit(fd, mainvar, part->eff_group);
        expand_doit(fd, mainvar, part->bb_ob);
        expand_doit(fd, mainvar, part->collision_group);
@@ -9558,7 +9547,7 @@ static void expand_particlesettings(FileData *fd, Main *mainvar, ParticleSetting
                }
        }
 
-       for (ParticleDupliWeight *dw = part->dupliweights.first; dw; dw = dw->next) {
+       for (ParticleDupliWeight *dw = part->instance_weights.first; dw; dw = dw->next) {
                expand_doit(fd, mainvar, dw->ob);
        }
 }
@@ -9880,8 +9869,8 @@ static void expand_object(FileData *fd, Main *mainvar, Object *ob)
        if (paf && paf->group)
                expand_doit(fd, mainvar, paf->group);
 
-       if (ob->dup_group)
-               expand_doit(fd, mainvar, ob->dup_group);
+       if (ob->instance_collection)
+               expand_doit(fd, mainvar, ob->instance_collection);
 
        if (ob->proxy)
                expand_doit(fd, mainvar, ob->proxy);
@@ -9997,6 +9986,11 @@ static void expand_scene(FileData *fd, Main *mainvar, Scene *sce)
                        if (seq->clip) expand_doit(fd, mainvar, seq->clip);
                        if (seq->mask) expand_doit(fd, mainvar, seq->mask);
                        if (seq->sound) expand_doit(fd, mainvar, seq->sound);
+
+                       if (seq->type == SEQ_TYPE_TEXT && seq->effectdata) {
+                               TextVars *data = seq->effectdata;
+                               expand_doit(fd, mainvar, data->text_font);
+                       }
                } SEQ_END;
        }
 
@@ -10022,6 +10016,10 @@ static void expand_scene(FileData *fd, Main *mainvar, Scene *sce)
        if (sce->master_collection) {
                expand_collection(fd, mainvar, sce->master_collection);
        }
+
+       if (sce->r.bake.cage_object) {
+               expand_doit(fd, mainvar, sce->r.bake.cage_object);
+       }
 }
 
 static void expand_camera(FileData *fd, Main *mainvar, Camera *ca)
@@ -10153,7 +10151,7 @@ static void expand_workspace(FileData *fd, Main *mainvar, WorkSpace *workspace)
 /**
  * Set the callback func used over all ID data found by \a BLO_expand_main func.
  *
- * \param expand_doit_func Called for each ID block it finds.
+ * \param expand_doit_func: Called for each ID block it finds.
  */
 void BLO_main_expander(BLOExpandDoitCallback expand_doit_func)
 {
@@ -10164,8 +10162,8 @@ void BLO_main_expander(BLOExpandDoitCallback expand_doit_func)
  * Loop over all ID data in Main to mark relations.
  * Set (id->tag & LIB_TAG_NEED_EXPAND) to mark expanding. Flags get cleared after expanding.
  *
- * \param fdhandle usually filedata, or own handle.
- * \param mainvar the Main database to expand.
+ * \param fdhandle: usually filedata, or own handle.
+ * \param mainvar: the Main database to expand.
  */
 void BLO_expand_main(void *fdhandle, Main *mainvar)
 {
@@ -10314,7 +10312,8 @@ static Collection *get_collection_active(
 }
 
 static void add_loose_objects_to_scene(
-        Main *mainvar, Main *bmain, Scene *scene, ViewLayer *view_layer, Library *lib, const short flag)
+        Main *mainvar, Main *bmain,
+        Scene *scene, ViewLayer *view_layer, const View3D *v3d, Library *lib, const short flag)
 {
        const bool is_link = (flag & FILE_LINK) != 0;
 
@@ -10342,6 +10341,11 @@ static void add_loose_objects_to_scene(
                                Collection *active_collection = get_collection_active(bmain, scene, view_layer, FILE_ACTIVE_COLLECTION);
                                BKE_collection_object_add(bmain, active_collection, ob);
                                Base *base = BKE_view_layer_base_find(view_layer, ob);
+
+                               if (v3d != NULL) {
+                                       base->local_view_bits |= v3d->local_view_uuid;
+                               }
+
                                BKE_scene_object_base_flag_sync_from_base(base);
 
                                if (flag & FILE_AUTOSELECT) {
@@ -10362,39 +10366,62 @@ static void add_loose_objects_to_scene(
 }
 
 static void add_collections_to_scene(
-        Main *mainvar, Main *bmain, Scene *scene, ViewLayer *view_layer, Library *UNUSED(lib), const short flag)
+        Main *mainvar, Main *bmain,
+        Scene *scene, ViewLayer *view_layer, const View3D *v3d, Library *lib, const short flag)
 {
        Collection *active_collection = get_collection_active(bmain, scene, view_layer, FILE_ACTIVE_COLLECTION);
 
        /* Give all objects which are tagged a base. */
        for (Collection *collection = mainvar->collection.first; collection; collection = collection->id.next) {
-               if (collection->id.tag & LIB_TAG_DOIT) {
-                       if (flag & FILE_GROUP_INSTANCE) {
-                               /* Any indirect collection should not have been tagged. */
-                               BLI_assert((collection->id.tag & LIB_TAG_INDIRECT) == 0);
+               if ((flag & FILE_GROUP_INSTANCE) && (collection->id.tag & LIB_TAG_DOIT)) {
+                       /* Any indirect collection should not have been tagged. */
+                       BLI_assert((collection->id.tag & LIB_TAG_INDIRECT) == 0);
 
-                               /* BKE_object_add(...) messes with the selection. */
-                               Object *ob = BKE_object_add_only_object(bmain, OB_EMPTY, collection->id.name + 2);
-                               ob->type = OB_EMPTY;
+                       /* BKE_object_add(...) messes with the selection. */
+                       Object *ob = BKE_object_add_only_object(bmain, OB_EMPTY, collection->id.name + 2);
+                       ob->type = OB_EMPTY;
 
-                               BKE_collection_object_add(bmain, active_collection, ob);
-                               Base *base = BKE_view_layer_base_find(view_layer, ob);
+                       BKE_collection_object_add(bmain, active_collection, ob);
+                       Base *base = BKE_view_layer_base_find(view_layer, ob);
 
-                               if (base->flag & BASE_SELECTABLE) {
-                                       base->flag |= BASE_SELECTED;
-                               }
+                       if (v3d != NULL) {
+                               base->local_view_bits |= v3d->local_view_uuid;
+                       }
 
-                               BKE_scene_object_base_flag_sync_from_base(base);
-                               DEG_id_tag_update(&ob->id, OB_RECALC_OB | OB_RECALC_DATA | OB_RECALC_TIME);
-                               view_layer->basact = base;
+                       if (base->flag & BASE_SELECTABLE) {
+                               base->flag |= BASE_SELECTED;
+                       }
 
-                               /* Assign the collection. */
-                               ob->dup_group = collection;
-                               id_us_plus(&collection->id);
-                               ob->transflag |= OB_DUPLICOLLECTION;
-                               copy_v3_v3(ob->loc, scene->cursor.location);
+                       BKE_scene_object_base_flag_sync_from_base(base);
+                       DEG_id_tag_update(&ob->id, ID_RECALC_TRANSFORM | ID_RECALC_GEOMETRY | ID_RECALC_ANIMATION);
+                       view_layer->basact = base;
+
+                       /* Assign the collection. */
+                       ob->instance_collection = collection;
+                       id_us_plus(&collection->id);
+                       ob->transflag |= OB_DUPLICOLLECTION;
+                       copy_v3_v3(ob->loc, scene->cursor.location);
+               }
+               else {
+                       bool do_add_collection = (collection->id.tag & LIB_TAG_DOIT) != 0;
+                       if (!do_add_collection) {
+                               /* We need to check that objects in that collections are already instantiated in a scene.
+                                * Otherwise, it's better to add the collection to the scene's active collection, than to
+                                * instantiate its objects in active scene's collection directly. See T61141.
+                                * Note that we only check object directly into that collection, not recursively into its children.
+                                */
+                               for (CollectionObject *coll_ob = collection->gobject.first; coll_ob != NULL; coll_ob = coll_ob->next) {
+                                       Object *ob = coll_ob->ob;
+                                       if ((ob->id.tag & LIB_TAG_PRE_EXISTING) == 0 &&
+                                           (ob->id.lib == lib) &&
+                                           (object_in_any_scene(bmain, ob) == 0))
+                                       {
+                                               do_add_collection = true;
+                                               break;
+                                       }
+                               }
                        }
-                       else {
+                       if (do_add_collection) {
                                /* Add collection as child of active collection. */
                                BKE_collection_child_add(bmain, active_collection, collection);
 
@@ -10474,7 +10501,8 @@ static ID *link_named_part(
        return id;
 }
 
-static void link_object_postprocess(ID *id, Main *bmain, Scene *scene, ViewLayer *view_layer, const int flag)
+static void link_object_postprocess(
+        ID *id, Main *bmain, Scene *scene, ViewLayer *view_layer, const View3D *v3d, const int flag)
 {
        if (scene) {
                /* link to scene */
@@ -10490,6 +10518,11 @@ static void link_object_postprocess(ID *id, Main *bmain, Scene *scene, ViewLayer
                base = BKE_view_layer_base_find(view_layer, ob);
                BKE_scene_object_base_flag_sync_from_base(base);
 
+               /* Link at active local view (view3d if available in context. */
+               if (v3d != NULL) {
+                       base->local_view_bits |= v3d->local_view_uuid;
+               }
+
                if (flag & FILE_AUTOSELECT) {
                        if (base->flag & BASE_SELECTABLE) {
                                base->flag |= BASE_SELECTED;
@@ -10537,12 +10570,12 @@ void BLO_library_link_copypaste(Main *mainl, BlendHandle *bh)
 
 static ID *link_named_part_ex(
         Main *mainl, FileData *fd, const short idcode, const char *name, const int flag,
-        Main *bmain, Scene *scene, ViewLayer *view_layer)
+        Main *bmain, Scene *scene, ViewLayer *view_layer, const View3D *v3d)
 {
        ID *id = link_named_part(mainl, fd, idcode, name, flag);
 
        if (id && (GS(id->name) == ID_OB)) {    /* loose object: give a base */
-               link_object_postprocess(id, bmain, scene, view_layer, flag);
+               link_object_postprocess(id, bmain, scene, view_layer, v3d, flag);
        }
        else if (id && (GS(id->name) == ID_GR)) {
                /* tag as needing to be instantiated or linked */
@@ -10555,10 +10588,10 @@ static ID *link_named_part_ex(
 /**
  * Link a named datablock from an external blend file.
  *
- * \param mainl The main database to link from (not the active one).
- * \param bh The blender file handle.
- * \param idcode The kind of datablock to link.
- * \param name The name of the datablock (without the 2 char ID prefix).
+ * \param mainl: The main database to link from (not the active one).
+ * \param bh: The blender file handle.
+ * \param idcode: The kind of datablock to link.
+ * \param name: The name of the datablock (without the 2 char ID prefix).
  * \return the linked ID when found.
  */
 ID *BLO_library_link_named_part(Main *mainl, BlendHandle **bh, const short idcode, const char *name)
@@ -10571,22 +10604,22 @@ ID *BLO_library_link_named_part(Main *mainl, BlendHandle **bh, const short idcod
  * Link a named datablock from an external blend file.
  * Optionally instantiate the object/collection in the scene when the flags are set.
  *
- * \param mainl The main database to link from (not the active one).
- * \param bh The blender file handle.
- * \param idcode The kind of datablock to link.
- * \param name The name of the datablock (without the 2 char ID prefix).
- * \param flag Options for linking, used for instantiating.
- * \param scene The scene in which to instantiate objects/collections (if NULL, no instantiation is done).
- * \param v3d The active View3D (only to define active layers for instantiated objects & collections, can be NULL).
+ * \param mainl: The main database to link from (not the active one).
+ * \param bh: The blender file handle.
+ * \param idcode: The kind of datablock to link.
+ * \param name: The name of the datablock (without the 2 char ID prefix).
+ * \param flag: Options for linking, used for instantiating.
+ * \param scene: The scene in which to instantiate objects/collections (if NULL, no instantiation is done).
+ * \param v3d: The active View3D (only to define active layers for instantiated objects & collections, can be NULL).
  * \return the linked ID when found.
  */
 ID *BLO_library_link_named_part_ex(
         Main *mainl, BlendHandle **bh,
         const short idcode, const char *name, const int flag,
-        Main *bmain, Scene *scene, ViewLayer *view_layer)
+        Main *bmain, Scene *scene, ViewLayer *view_layer, const View3D *v3d)
 {
        FileData *fd = (FileData *)(*bh);
-       return link_named_part_ex(mainl, fd, idcode, name, flag, bmain, scene, view_layer);
+       return link_named_part_ex(mainl, fd, idcode, name, flag, bmain, scene, view_layer, v3d);
 }
 
 static void link_id_part(ReportList *reports, FileData *fd, Main *mainvar, ID *id, ID **r_id)
@@ -10661,9 +10694,9 @@ static Main *library_link_begin(Main *mainvar, FileData **fd, const char *filepa
 /**
  * Initialize the BlendHandle for linking library data.
  *
- * \param mainvar The current main database, e.g. G_MAIN or CTX_data_main(C).
- * \param bh A blender file handle as returned by \a BLO_blendhandle_from_file or \a BLO_blendhandle_from_memory.
- * \param filepath Used for relative linking, copied to the \a lib->name.
+ * \param mainvar: The current main database, e.g. G_MAIN or CTX_data_main(C).
+ * \param bh: A blender file handle as returned by \a BLO_blendhandle_from_file or \a BLO_blendhandle_from_memory.
+ * \param filepath: Used for relative linking, copied to the \a lib->name.
  * \return the library Main, to be passed to \a BLO_library_append_named_part as \a mainl.
  */
 Main *BLO_library_link_begin(Main *mainvar, BlendHandle **bh, const char *filepath)
@@ -10699,7 +10732,9 @@ static void split_main_newid(Main *mainptr, Main *main_newid)
 }
 
 /* scene and v3d may be NULL. */
-static void library_link_end(Main *mainl, FileData **fd, const short flag, Main *bmain, Scene *scene, ViewLayer *view_layer)
+static void library_link_end(
+        Main *mainl, FileData **fd, const short flag, Main *bmain,
+        Scene *scene, ViewLayer *view_layer, const View3D *v3d)
 {
        Main *mainvar;
        Library *curlib;
@@ -10757,8 +10792,8 @@ static void library_link_end(Main *mainl, FileData **fd, const short flag, Main
         * Only directly linked objects & collections are instantiated by `BLO_library_link_named_part_ex()` & co,
         * here we handle indirect ones and other possible edge-cases. */
        if (scene) {
-               add_collections_to_scene(mainvar, bmain, scene, view_layer, curlib, flag);
-               add_loose_objects_to_scene(mainvar, bmain, scene, view_layer, curlib, flag);
+               add_collections_to_scene(mainvar, bmain, scene, view_layer, v3d, curlib, flag);
+               add_loose_objects_to_scene(mainvar, bmain, scene, view_layer, v3d, curlib, flag);
        }
        else {
                /* printf("library_append_end, scene is NULL (objects wont get bases)\n"); */
@@ -10779,17 +10814,20 @@ static void library_link_end(Main *mainl, FileData **fd, const short flag, Main
  * Optionally instance the indirect object/collection in the scene when the flags are set.
  * \note Do not use \a bh after calling this function, it may frees it.
  *
- * \param mainl The main database to link from (not the active one).
- * \param bh The blender file handle (WARNING! may be freed by this function!).
- * \param flag Options for linking, used for instantiating.
- * \param bmain The main database in which to instantiate objects/collections
- * \param scene The scene in which to instantiate objects/collections (if NULL, no instantiation is done).
- * \param view_layer The scene layer in which to instantiate objects/collections (if NULL, no instantiation is done).
+ * \param mainl: The main database to link from (not the active one).
+ * \param bh: The blender file handle (WARNING! may be freed by this function!).
+ * \param flag: Options for linking, used for instantiating.
+ * \param bmain: The main database in which to instantiate objects/collections
+ * \param scene: The scene in which to instantiate objects/collections (if NULL, no instantiation is done).
+ * \param view_layer: The scene layer in which to instantiate objects/collections (if NULL, no instantiation is done).
+ * \param v3d: The active View3D (only to define local-view for instantiated objects & groups, can be NULL).
  */
-void BLO_library_link_end(Main *mainl, BlendHandle **bh, int flag, Main *bmain, Scene *scene, ViewLayer *view_layer)
+void BLO_library_link_end(
+        Main *mainl, BlendHandle **bh, int flag, Main *bmain,
+        Scene *scene, ViewLayer *view_layer, const View3D *v3d)
 {
        FileData *fd = (FileData *)(*bh);
-       library_link_end(mainl, &fd, flag, bmain, scene, view_layer);
+       library_link_end(mainl, &fd, flag, bmain, scene, view_layer, v3d);
        *bh = (BlendHandle *)fd;
 }
 
@@ -10842,7 +10880,6 @@ static void read_libraries(FileData *basefd, ListBase *mainlist)
                                FileData *fd = mainptr->curlib->filedata;
 
                                if (fd == NULL) {
-
                                        /* printf and reports for now... its important users know this */
 
                                        /* if packed file... */
@@ -10867,30 +10904,6 @@ static void read_libraries(FileData *basefd, ListBase *mainlist)
                                                        library_parent_filepath(mainptr->curlib));
                                                fd = blo_openblenderfile(mainptr->curlib->filepath, basefd->reports);
                                        }
-                                       /* allow typing in a new lib path */
-                                       if (G.debug_value == -666) {
-                                               while (fd == NULL) {
-                                                       char newlib_path[FILE_MAX] = {0};
-                                                       printf("Missing library...'\n");
-                                                       printf("        current file: %s\n", BKE_main_blendfile_path_from_global());
-                                                       printf("        absolute lib: %s\n", mainptr->curlib->filepath);
-                                                       printf("        relative lib: %s\n", mainptr->curlib->name);
-                                                       printf("  enter a new path:\n");
-
-                                                       if (scanf("%1023s", newlib_path) > 0) {  /* Warning, keep length in sync with FILE_MAX! */
-                                                               BLI_strncpy(mainptr->curlib->name, newlib_path, sizeof(mainptr->curlib->name));
-                                                               BLI_strncpy(mainptr->curlib->filepath, newlib_path, sizeof(mainptr->curlib->filepath));
-                                                               BLI_cleanup_path(BKE_main_blendfile_path_from_global(), mainptr->curlib->filepath);
-
-                                                               fd = blo_openblenderfile(mainptr->curlib->filepath, basefd->reports);
-
-                                                               if (fd) {
-                                                                       fd->mainlist = mainlist;
-                                                                       printf("found: '%s', party on macuno!\n", mainptr->curlib->filepath);
-                                                               }
-                                                       }
-                                               }
-                                       }
 
                                        if (fd) {
                                                /* share the mainlist, so all libraries are added immediately in a
@@ -10914,7 +10927,6 @@ static void read_libraries(FileData *basefd, ListBase *mainlist)
 #ifdef USE_GHASH_BHEAD
                                                read_file_bhead_idname_map_create(fd);
 #endif
-
                                        }
                                        else {
                                                mainptr->curlib->filedata = NULL;
@@ -10958,7 +10970,7 @@ static void read_libraries(FileData *basefd, ListBase *mainlist)
                                                        change_idid_adr(mainlist, basefd, id, *realid);
 
                                                        /* We cannot free old lib-ref placeholder ID here anymore, since we use its name
-                                                        * as key in loaded_ids hass. */
+                                                        * as key in loaded_ids has. */
                                                        BLI_addtail(&pending_free_ids, id);
                                                }
                                                id = idn;
@@ -10978,34 +10990,6 @@ static void read_libraries(FileData *basefd, ListBase *mainlist)
        BLI_ghash_free(loaded_ids, NULL, NULL);
        loaded_ids = NULL;
 
-       /* test if there are unread libblocks */
-       /* XXX This code block is kept for 2.77, until we are sure it never gets reached anymore. Can be removed later. */
-       for (mainptr = mainl->next; mainptr; mainptr = mainptr->next) {
-               a = set_listbasepointers(mainptr, lbarray);
-               while (a--) {
-                       ID *id, *idn = NULL;
-
-                       for (id = lbarray[a]->first; id; id = idn) {
-                               idn = id->next;
-                               if (id->tag & LIB_TAG_READ) {
-                                       BLI_assert(0);
-                                       BLI_remlink(lbarray[a], id);
-                                       blo_reportf_wrap(
-                                               basefd->reports, RPT_ERROR,
-                                               TIP_("LIB: %s: '%s' unread lib block missing from '%s', parent '%s' - "
-                                                    "Please file a bug report if you see this message"),
-                                               BKE_idcode_to_name(GS(id->name)),
-                                               id->name + 2,
-                                               mainptr->curlib->filepath,
-                                               library_parent_filepath(mainptr->curlib));
-                                       change_idid_adr(mainlist, basefd, id, NULL);
-
-                                       MEM_freeN(id);
-                               }
-                       }
-               }
-       }
-
        /* do versions, link, and free */
        Main *main_newid = BKE_main_new();
        for (mainptr = mainl->next; mainptr; mainptr = mainptr->next) {