DNA: support versioning structs & struct members
authorCampbell Barton <ideasman42@gmail.com>
Mon, 11 Feb 2019 22:29:18 +0000 (09:29 +1100)
committerCampbell Barton <ideasman42@gmail.com>
Mon, 11 Feb 2019 22:38:46 +0000 (09:38 +1100)
This is only to be used rarely because it's not forwards compatible.

Replace version patching of old 2.80 DNA with a more generic API.

source/blender/blenloader/CMakeLists.txt
source/blender/blenloader/intern/readfile.c
source/blender/blenloader/intern/readfile.h
source/blender/blenloader/intern/versioning_dna.c [new file with mode: 0644]
source/blender/makesdna/DNA_fileglobal_types.h
source/blender/makesdna/DNA_genfile.h
source/blender/makesdna/DNA_sdna_types.h
source/blender/makesdna/intern/CMakeLists.txt
source/blender/makesdna/intern/dna_genfile.c

index efd64b1..b9bec1b 100644 (file)
@@ -54,6 +54,7 @@ set(SRC
        intern/versioning_270.c
        intern/versioning_280.c
        intern/versioning_defaults.c
+       intern/versioning_dna.c
        intern/versioning_legacy.c
        intern/versioning_userdef.c
        intern/writefile.c
index c4fdb87..55a72e3 100644 (file)
@@ -932,13 +932,30 @@ static void decode_blender_header(FileData *fd)
 static bool read_file_dna(FileData *fd, const char **r_error_message)
 {
        BHead *bhead;
+       int subversion = 0;
 
        for (bhead = blo_firstbhead(fd); bhead; bhead = blo_nextbhead(fd, bhead)) {
-               if (bhead->code == DNA1) {
+               if (bhead->code == GLOB) {
+                       /* Before this, the subversion didn't exist in 'FileGlobal' so the subversion
+                        * value isn't accessible for the purpose of DNA versioning in this case. */
+                       if (fd->fileversion <= 242) {
+                               continue;
+                       }
+                       /* We can't use read_global because this needs 'DNA1' to be decoded,
+                        * however the first 4 chars are _always_ the subversion. */
+                       FileGlobal *fg = (void *)&bhead[1];
+                       BLI_STATIC_ASSERT(offsetof(FileGlobal, subvstr) == 0, "Must be first: subvstr");
+                       char num[5];
+                       memcpy(num, fg->subvstr, 4);
+                       num[4] = 0;
+                       subversion = atoi(num);
+               }
+               else if (bhead->code == DNA1) {
                        const bool do_endian_swap = (fd->flags & FD_FLAGS_SWITCH_ENDIAN) != 0;
 
                        fd->filesdna = DNA_sdna_from_data(&bhead[1], bhead->len, do_endian_swap, true, r_error_message);
                        if (fd->filesdna) {
+                               blo_do_versions_dna(fd->filesdna, fd->fileversion, subversion);
                                fd->compflags = DNA_struct_get_compareflags(fd->filesdna, fd->memsdna);
                                /* used to retrieve ID names from (bhead+1) */
                                fd->id_name_offs = DNA_elem_offset(fd->filesdna, "ID", "char", "name[]");
index 8541689..39a474d 100644 (file)
@@ -150,6 +150,8 @@ const char *bhead_id_name(const FileData *fd, const BHead *bhead);
 
 void blo_reportf_wrap(struct ReportList *reports, ReportType type, const char *format, ...) ATTR_PRINTF_FORMAT(3, 4);
 
+void blo_do_versions_dna(struct SDNA *sdna, const int versionfile, const int subversionfile);
+
 void blo_do_versions_oldnewmap_insert(struct OldNewMap *onm, const void *oldaddr, void *newaddr, int nr);
 void *blo_do_versions_newlibadr(struct FileData *fd, const void *lib, const void *adr);
 void *blo_do_versions_newlibadr_us(struct FileData *fd, const void *lib, const void *adr);
diff --git a/source/blender/blenloader/intern/versioning_dna.c b/source/blender/blenloader/intern/versioning_dna.c
new file mode 100644 (file)
index 0000000..d379edd
--- /dev/null
@@ -0,0 +1,59 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file \ingroup blenloader
+ *
+ * Apply edits to DNA at load time
+ * to behave as if old files were written new names.
+ */
+
+#include "BLI_compiler_attrs.h"
+#include "BLI_utildefines.h"
+
+#include "DNA_genfile.h"
+#include "DNA_listBase.h"
+
+#include "BLO_readfile.h"
+#include "readfile.h"
+
+/**
+ * Manipulates SDNA before calling #DNA_struct_get_compareflags,
+ * allowing us to rename structs and struct members.
+ *
+ * \attention Changes here will cause breakages in forward compatbility,
+ * Use this only in the _rare_ cases when migrating to new naming is needed.
+ */
+void blo_do_versions_dna(SDNA *sdna, const int versionfile, const int subversionfile)
+{
+#define DNA_VERSION_ATLEAST(ver, subver) \
+       (versionfile > (ver) || (versionfile == (ver) && (subversionfile >= (subver))))
+
+       if (!DNA_VERSION_ATLEAST(280, 2)) {
+               /* Version files created in the 'blender2.8' branch
+                * between October 2016, and November 2017 (>=280.0 and < 280.2). */
+               if (versionfile >= 280) {
+                       DNA_sdna_patch_struct(sdna, "SceneLayer", "ViewLayer");
+                       DNA_sdna_patch_struct(sdna, "SceneLayerEngineData", "ViewLayerEngineData");
+                       DNA_sdna_patch_struct_member(sdna, "FileGlobal", "cur_render_layer", "cur_view_layer");
+                       DNA_sdna_patch_struct_member(sdna, "ParticleEditSettings", "scene_layer", "view_layer");
+                       DNA_sdna_patch_struct_member(sdna, "Scene", "active_layer", "active_view_layer");
+                       DNA_sdna_patch_struct_member(sdna, "Scene", "render_layers", "view_layers");
+                       DNA_sdna_patch_struct_member(sdna, "WorkSpace", "render_layer", "view_layer");
+               }
+       }
+
+#undef DNA_VERSION_ATLEAST
+}
index c5833fc..52fbdbc 100644 (file)
@@ -28,8 +28,9 @@
  * the moment of saving, and the file-specific settings.
  */
 typedef struct FileGlobal {
-       /** Needs to be here, for human fileformat recognition. */
+       /** Needs to be here, for human fileformat recognition (keep first!). */
        char subvstr[4];
+
        short subversion;
        short minversion, minsubversion;
        char pad[6];
index 774b903..4f67246 100644 (file)
@@ -103,4 +103,9 @@ bool DNA_struct_elem_find(const struct SDNA *sdna, const char *stype, const char
 
 int DNA_elem_type_size(const eSDNA_Type elem_nr);
 
+bool DNA_sdna_patch_struct(
+        struct SDNA *sdna, const char *struct_name_old, const char *struct_name_new);
+bool DNA_sdna_patch_struct_member(
+        struct SDNA *sdna, const char *struct_name, const char *member_old, const char *member_new);
+
 #endif /* __DNA_GENFILE_H__ */
index 1adeb30..3ca2ce2 100644 (file)
@@ -22,6 +22,8 @@
 #ifndef __DNA_SDNA_TYPES_H__
 #define __DNA_SDNA_TYPES_H__
 
+struct MemArena;
+
 #
 #
 typedef struct SDNA {
@@ -59,6 +61,9 @@ typedef struct SDNA {
 
        /** #GHash for faster lookups, requires WITH_DNA_GHASH to be used for now. */
        struct GHash *structs_map;
+
+       /** Temporary memory currently only used for version patching DNA. */
+       struct MemArena *mem_arena;
 } SDNA;
 
 #
index 4a53b74..8ad1d18 100644 (file)
@@ -104,6 +104,8 @@ set(INC_SYS
 set(SRC
        ../../blenlib/intern/BLI_ghash.c
        ../../blenlib/intern/BLI_ghash_utils.c
+       ../../blenlib/intern/BLI_linklist.c
+       ../../blenlib/intern/BLI_memarena.c
        ../../blenlib/intern/BLI_mempool.c
        ../../blenlib/intern/endian_switch.c
        ../../blenlib/intern/hash_mm2a.c
index 650ee29..3d0f4fa 100644 (file)
@@ -36,6 +36,7 @@
 
 #include "BLI_utildefines.h"
 #include "BLI_endian_switch.h"
+#include "BLI_memarena.h"
 
 #ifdef WITH_DNA_GHASH
 #  include "BLI_ghash.h"
@@ -193,6 +194,10 @@ void DNA_sdna_free(SDNA *sdna)
        }
 #endif
 
+       if (sdna->mem_arena) {
+               BLI_memarena_free(sdna->mem_arena);
+       }
+
        MEM_freeN(sdna);
 }
 
@@ -327,53 +332,11 @@ BLI_INLINE const char *pad_up_4(const char *ptr)
        return (const char *)((((uintptr_t)ptr) + 3) & ~3);
 }
 
-/**
- * Temporary DNA doversion for files that were created with Blender 2.80
- * between October 2016, and November 2017 (>=280.0 and < 280.2).
- *
- * /note This would be way more efficient if we can get the version from SDNA
- * So we could return true if version == 280 && subversion < 2.
- *
- * Returns true if we need to do the DNA renaming.
- */
-static bool need_doversion_280(SDNA *sdna, int *data, const bool data_alloc)
-{
-       if (data_alloc == false) {
-               return false;
-       }
-
-       bool active_layer = false, render_layers = false;
-
-       const char *cp = (char *)data;
-       for (int nr = 0; nr < sdna->nr_names; nr++) {
-               if (strcmp(cp, "active_layer") == 0) {
-                       active_layer = true;
-                       if (active_layer && render_layers) {
-                               return true;
-                       }
-               }
-               else if (strcmp(cp, "render_layers") == 0) {
-                       render_layers = true;
-                       if (active_layer && render_layers) {
-                               return true;
-                       }
-               }
-
-               while (*cp) cp++;
-               cp++;
-       }
-
-       /* If someone adds only one of them to the DNA, don't! */
-       BLI_assert(!(active_layer || render_layers));
-       return false;
-}
-
 /**
  * In sdna->data the data, now we convert that to something understandable
  */
 static bool init_structDNA(
         SDNA *sdna, bool do_endian_swap,
-        bool data_alloc,
         const char **r_error_message)
 {
        int *data, gravity_fix = -1;
@@ -388,6 +351,7 @@ static bool init_structDNA(
 #ifdef WITH_DNA_GHASH
        sdna->structs_map = NULL;
 #endif
+       sdna->mem_arena = NULL;
 
        /* Struct DNA ('SDNA') */
        if (*data != MAKE_ID('S', 'D', 'N', 'A')) {
@@ -415,10 +379,6 @@ static bool init_structDNA(
                        return false;
                }
 
-               /* Temporary DNA doversion for files that were created with Blender 2.80
-                * between 280.0 and 280.2. */
-               const bool doversion_280 = need_doversion_280(sdna, data, data_alloc);
-
                cp = (char *)data;
                for (int nr = 0; nr < sdna->nr_names; nr++) {
                        sdna->names[nr] = cp;
@@ -432,29 +392,6 @@ static bool init_structDNA(
                                        gravity_fix = nr;
                                }
                        }
-                       else if (doversion_280) {
-                               if (strcmp(cp, "*render_layer") == 0) {
-                                       /* WorkSpace. */
-                                       sdna->names[nr] = "*view_layer";
-                               }
-                               else if (strcmp(cp, "*scene_layer") == 0) {
-                                       /* ParticleEditSettings. */
-                                       sdna->names[nr] = "*view_layer";
-                               }
-                               else if (strcmp(cp, "render_layers") == 0) {
-                                       /* Scene. */
-                                       sdna->names[nr] = "view_layers";
-                               }
-                               else if (strcmp(cp, "active_layer") == 0) {
-                                       /* Scene. */
-                                       sdna->names[nr] = "active_view_layer";
-                               }
-                               else if (strcmp(cp, "*cur_render_layer") == 0) {
-                                       /* FileGlobal. */
-                                       sdna->names[nr] = "*cur_view_layer";
-                               }
-                       }
-
                        while (*cp) cp++;
                        cp++;
                }
@@ -498,14 +435,6 @@ static bool init_structDNA(
                        else if (strcmp("CollectionObject", cp) == 0) {
                                sdna->types[nr] = "GroupObject";
                        }
-                       else if (doversion_280) {
-                               if (strcmp(cp, "SceneLayer") == 0) {
-                                       sdna->types[nr] = "ViewLayer";
-                               }
-                               else if (strcmp(cp, "SceneLayerEngineData") == 0) {
-                                       sdna->types[nr] = "ViewLayerEngineData";
-                               }
-                       }
 
                        while (*cp) cp++;
                        cp++;
@@ -646,7 +575,7 @@ SDNA *DNA_sdna_from_data(
        sdna->data_alloc = data_alloc;
 
 
-       if (init_structDNA(sdna, do_endian_swap, data_alloc, &error_message)) {
+       if (init_structDNA(sdna, do_endian_swap, &error_message)) {
                return sdna;
        }
        else {
@@ -1475,3 +1404,122 @@ int DNA_elem_type_size(const eSDNA_Type elem_nr)
        /* weak */
        return 8;
 }
+
+/* -------------------------------------------------------------------- */
+/** \name Version Patch DNA
+ * \{ */
+
+static bool is_identifier(const char c)
+{
+       return ((c >= 'a' && c <= 'z') ||
+               (c >= 'A' && c <= 'Z') ||
+               (c >= '0' && c <= '9') ||
+               (c == '_'));
+}
+
+/**
+ * Check if 'var' matches '*var[3]' for eg,
+ * return true if it does, with start/end offsets.
+ */
+static char str_member_match(
+        const char *member_search, const int member_search_len,
+        const char *member_dna,
+        uint *r_member_dna_offset)
+{
+       BLI_assert(strlen(member_search) == member_search_len);
+       int member_dna_offset = 0;
+       while (!is_identifier(member_dna[member_dna_offset])) {
+               member_dna_offset++;
+       }
+       const char *member_dna_trim = member_dna + member_dna_offset;
+       if (strncmp(member_search, member_dna_trim, member_search_len) == 0) {
+               const char c = member_dna_trim[member_search_len];
+               if (c == '\0' || !is_identifier(c)) {
+                       *r_member_dna_offset = member_dna_offset;
+                       return true;
+               }
+       }
+       return false;
+}
+
+static bool DNA_sdna_patch_struct_nr(
+        SDNA *sdna, const int struct_name_old_nr, const char *struct_name_new)
+{
+       BLI_assert(DNA_struct_find_nr(DNA_sdna_current_get(), struct_name_new) != -1);
+       const short *sp = sdna->structs[struct_name_old_nr];
+#ifdef WITH_DNA_GHASH
+       BLI_ghash_remove(sdna->structs_map, (void *)sdna->types[sp[0]], NULL, NULL);
+       BLI_ghash_insert(sdna->structs_map, (void *)struct_name_new, POINTER_FROM_INT(struct_name_old_nr));
+#endif
+       // printf("Struct rename: %s -> %s\n", sdna->types[struct_name_old_nr], struct_name_new);
+       sdna->types[sp[0]] = struct_name_new;
+       return true;
+}
+/**
+ * Rename a struct
+ */
+bool DNA_sdna_patch_struct(
+        SDNA *sdna, const char *struct_name_old, const char *struct_name_new)
+{
+       const int struct_name_old_nr = DNA_struct_find_nr(sdna, struct_name_old);
+       if (struct_name_old_nr != -1) {
+               return DNA_sdna_patch_struct_nr(sdna, struct_name_old_nr, struct_name_new);
+       }
+       return false;
+}
+
+/* Make public if called often with same struct (avoid duplicate look-ups). */
+static bool DNA_sdna_patch_struct_member_nr(
+        SDNA *sdna, const int struct_name_nr, const char *member_old, const char *member_new)
+{
+       const int member_old_len = strlen(member_old);
+       const int member_new_len = strlen(member_new);
+       BLI_assert(member_new != NULL);
+       const short *sp = sdna->structs[struct_name_nr];
+       for (int member_iter = sp[1]; member_iter > 0; member_iter--, sp += 2) {
+               const char *member_dna_old = sdna->names[sp[1]];
+               /* Start & end offsets in 'member_dna_old'. */
+               uint member_dna_offset_start;
+               if (str_member_match(member_old, member_old_len, member_dna_old, &member_dna_offset_start)) {
+                       if (sdna->mem_arena == NULL) {
+                               sdna->mem_arena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__);
+                       }
+                       const int member_dna_old_len = strlen(member_dna_old);
+                       const int member_final_len = (member_dna_old_len - member_old_len) + member_new_len;
+                       char *member_dna_new = BLI_memarena_alloc(sdna->mem_arena, member_final_len + 1);
+                       uint i = 0;
+                       if (member_dna_offset_start != 0) {
+                               memcpy(member_dna_new, member_dna_old, member_dna_offset_start);
+                               i = member_dna_offset_start;
+                       }
+                       memcpy(&member_dna_new[i], member_new, member_new_len + 1);
+                       i += member_new_len;
+                       uint member_dna_offset_end = member_dna_offset_start + member_old_len;
+                       if (member_dna_old[member_dna_offset_end] != '\0') {
+                               const int member_dna_tail_len = (member_dna_old_len - member_dna_offset_end);
+                               memcpy(&member_dna_new[i], &member_dna_old[member_dna_offset_end], member_dna_tail_len + 1);
+                               i += member_dna_tail_len;
+                       }
+                       BLI_assert((strlen(member_dna_new) == member_final_len) && (i == member_final_len));
+                       // printf("Struct member rename: '%s.%s' -> '%s'\n", member_old, member_dna_old, member_dna_new);
+                       sdna->names[sp[1]] = member_dna_new;
+                       return true;
+               }
+       }
+       return false;
+}
+/**
+ * Replace \a member_old with \a member_new for struct \a struct_name
+ * handles search & replace, maintaining surrounding non-identifier characters such as pointer & array size.
+ */
+bool DNA_sdna_patch_struct_member(
+        SDNA *sdna, const char *struct_name, const char *member_old, const char *member_new)
+{
+       const int struct_name_nr = DNA_struct_find_nr(sdna, struct_name);
+       if (struct_name_nr != -1) {
+               return DNA_sdna_patch_struct_member_nr(sdna, struct_name_nr, member_old, member_new);
+       }
+       return false;
+}
+
+/** \} */