Merge branch 'master' into blender2.8
[blender.git] / source / blender / editors / space_file / filelist.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2007 Blender Foundation.
19  * All rights reserved.
20  *
21  * The Original Code is: all of this file.
22  *
23  * Contributor(s): none yet.
24  *
25  * ***** END GPL LICENSE BLOCK *****
26  */
27
28 /** \file blender/editors/space_file/filelist.c
29  *  \ingroup spfile
30  */
31
32
33 /* global includes */
34
35 #include <stdlib.h>
36 #include <math.h>
37 #include <string.h>
38 #include <sys/stat.h>
39 #include <time.h>
40
41 #ifndef WIN32
42 #  include <unistd.h>
43 #else
44 #  include <io.h>
45 #  include <direct.h>
46 #endif
47 #include "MEM_guardedalloc.h"
48
49 #include "BLI_blenlib.h"
50 #include "BLI_fileops.h"
51 #include "BLI_fileops_types.h"
52 #include "BLI_fnmatch.h"
53 #include "BLI_ghash.h"
54 #include "BLI_hash_md5.h"
55 #include "BLI_linklist.h"
56 #include "BLI_math.h"
57 #include "BLI_stack.h"
58 #include "BLI_task.h"
59 #include "BLI_threads.h"
60 #include "BLI_utildefines.h"
61
62 #ifdef WIN32
63 #  include "BLI_winstuff.h"
64 #endif
65
66 #include "BKE_context.h"
67 #include "BKE_global.h"
68 #include "BKE_library.h"
69 #include "BKE_icons.h"
70 #include "BKE_idcode.h"
71 #include "BKE_main.h"
72 #include "BLO_readfile.h"
73
74 #include "DNA_space_types.h"
75
76 #include "ED_datafiles.h"
77 #include "ED_fileselect.h"
78 #include "ED_screen.h"
79
80 #include "IMB_imbuf.h"
81 #include "IMB_imbuf_types.h"
82 #include "IMB_thumbs.h"
83
84 #include "PIL_time.h"
85
86 #include "WM_api.h"
87 #include "WM_types.h"
88
89 #include "UI_resources.h"
90 #include "UI_interface_icons.h"
91
92 #include "atomic_ops.h"
93
94 #include "filelist.h"
95
96
97 /* ----------------- FOLDERLIST (previous/next) -------------- */
98
99 typedef struct FolderList {
100         struct FolderList *next, *prev;
101         char *foldername;
102 } FolderList;
103
104 ListBase *folderlist_new(void)
105 {
106         ListBase *p = MEM_callocN(sizeof(*p), __func__);
107         return p;
108 }
109
110 void folderlist_popdir(struct ListBase *folderlist, char *dir)
111 {
112         const char *prev_dir;
113         struct FolderList *folder;
114         folder = folderlist->last;
115
116         if (folder) {
117                 /* remove the current directory */
118                 MEM_freeN(folder->foldername);
119                 BLI_freelinkN(folderlist, folder);
120
121                 folder = folderlist->last;
122                 if (folder) {
123                         prev_dir = folder->foldername;
124                         BLI_strncpy(dir, prev_dir, FILE_MAXDIR);
125                 }
126         }
127         /* delete the folder next or use setdir directly before PREVIOUS OP */
128 }
129
130 void folderlist_pushdir(ListBase *folderlist, const char *dir)
131 {
132         struct FolderList *folder, *previous_folder;
133         previous_folder = folderlist->last;
134
135         /* check if already exists */
136         if (previous_folder && previous_folder->foldername) {
137                 if (BLI_path_cmp(previous_folder->foldername, dir) == 0) {
138                         return;
139                 }
140         }
141
142         /* create next folder element */
143         folder = MEM_mallocN(sizeof(*folder), __func__);
144         folder->foldername = BLI_strdup(dir);
145
146         /* add it to the end of the list */
147         BLI_addtail(folderlist, folder);
148 }
149
150 const char *folderlist_peeklastdir(ListBase *folderlist)
151 {
152         struct FolderList *folder;
153
154         if (!folderlist->last)
155                 return NULL;
156
157         folder = folderlist->last;
158         return folder->foldername;
159 }
160
161 int folderlist_clear_next(struct SpaceFile *sfile)
162 {
163         struct FolderList *folder;
164
165         /* if there is no folder_next there is nothing we can clear */
166         if (!sfile->folders_next)
167                 return 0;
168
169         /* if previous_folder, next_folder or refresh_folder operators are executed it doesn't clear folder_next */
170         folder = sfile->folders_prev->last;
171         if ((!folder) || (BLI_path_cmp(folder->foldername, sfile->params->dir) == 0))
172                 return 0;
173
174         /* eventually clear flist->folders_next */
175         return 1;
176 }
177
178 /* not listbase itself */
179 void folderlist_free(ListBase *folderlist)
180 {
181         if (folderlist) {
182                 FolderList *folder;
183                 for (folder = folderlist->first; folder; folder = folder->next)
184                         MEM_freeN(folder->foldername);
185                 BLI_freelistN(folderlist);
186         }
187 }
188
189 ListBase *folderlist_duplicate(ListBase *folderlist)
190 {
191
192         if (folderlist) {
193                 ListBase *folderlistn = MEM_callocN(sizeof(*folderlistn), __func__);
194                 FolderList *folder;
195
196                 BLI_duplicatelist(folderlistn, folderlist);
197
198                 for (folder = folderlistn->first; folder; folder = folder->next) {
199                         folder->foldername = MEM_dupallocN(folder->foldername);
200                 }
201                 return folderlistn;
202         }
203         return NULL;
204 }
205
206
207 /* ------------------FILELIST------------------------ */
208
209 typedef struct FileListInternEntry {
210         struct FileListInternEntry *next, *prev;
211
212         char uuid[16];  /* ASSET_UUID_LENGTH */
213
214         int typeflag;  /* eFileSel_File_Types */
215         int blentype;  /* ID type, in case typeflag has FILE_TYPE_BLENDERLIB set. */
216
217         char *relpath;
218         char *name;  /* not striclty needed, but used during sorting, avoids to have to recompute it there... */
219
220         BLI_stat_t st;
221 } FileListInternEntry;
222
223 typedef struct FileListIntern {
224         ListBase entries;  /* FileListInternEntry items. */
225         FileListInternEntry **filtered;
226
227         char curr_uuid[16];  /* Used to generate uuid during internal listing. */
228 } FileListIntern;
229
230 #define FILELIST_ENTRYCACHESIZE_DEFAULT 1024  /* Keep it a power of two! */
231 typedef struct FileListEntryCache {
232         size_t size;  /* The size of the cache... */
233
234         int flags;
235
236         /* This one gathers all entries from both block and misc caches. Used for easy bulk-freing. */
237         ListBase cached_entries;
238
239         /* Block cache: all entries between start and end index. used for part of the list on diplay. */
240         FileDirEntry **block_entries;
241         int block_start_index, block_end_index, block_center_index, block_cursor;
242
243         /* Misc cache: random indices, FIFO behavior.
244          * Note: Not 100% sure we actually need that, time will say. */
245         int misc_cursor;
246         int *misc_entries_indices;
247         GHash *misc_entries;
248
249         /* Allows to quickly get a cached entry from its UUID. */
250         GHash *uuids;
251
252         /* Previews handling. */
253         TaskPool *previews_pool;
254         ThreadQueue *previews_done;
255 } FileListEntryCache;
256
257 /* FileListCache.flags */
258 enum {
259         FLC_IS_INIT              = 1 << 0,
260         FLC_PREVIEWS_ACTIVE      = 1 << 1,
261 };
262
263 typedef struct FileListEntryPreview {
264         char path[FILE_MAX];
265         unsigned int flags;
266         int index;
267         ImBuf *img;
268 } FileListEntryPreview;
269
270
271 typedef struct FileListFilter {
272         unsigned int filter;
273         unsigned int filter_id;
274         char filter_glob[FILE_MAXFILE];
275         char filter_search[66];  /* + 2 for heading/trailing implicit '*' wildcards. */
276         short flags;
277 } FileListFilter;
278
279 /* FileListFilter.flags */
280 enum {
281         FLF_DO_FILTER    = 1 << 0,
282         FLF_HIDE_DOT     = 1 << 1,
283         FLF_HIDE_PARENT  = 1 << 2,
284         FLF_HIDE_LIB_DIR = 1 << 3,
285 };
286
287 typedef struct FileList {
288         FileDirEntryArr filelist;
289
290         short prv_w;
291         short prv_h;
292
293         short flags;
294
295         short sort;
296
297         FileListFilter filter_data;
298
299         struct FileListIntern filelist_intern;
300
301         struct FileListEntryCache filelist_cache;
302
303         /* We need to keep those info outside of actual filelist items, because those are no more persistent
304          * (only generated on demand, and freed as soon as possible).
305          * Persistent part (mere list of paths + stat info) is kept as small as possible, and filebrowser-agnostic.
306          */
307         GHash *selection_state;
308
309         short max_recursion;
310         short recursion_level;
311
312         struct BlendHandle *libfiledata;
313
314         /* Set given path as root directory, if last bool is true may change given string in place to a valid value.
315          * Returns True if valid dir. */
316         bool (*checkdirf)(struct FileList *, char *, const bool);
317
318         /* Fill filelist (to be called by read job). */
319         void (*read_jobf)(struct FileList *, const char *, short *, short *, float *, ThreadMutex *);
320
321         /* Filter an entry of current filelist. */
322         bool (*filterf)(struct FileListInternEntry *, const char *, FileListFilter *);
323 } FileList;
324
325 /* FileList.flags */
326 enum {
327         FL_FORCE_RESET    = 1 << 0,
328         FL_IS_READY       = 1 << 1,
329         FL_IS_PENDING     = 1 << 2,
330         FL_NEED_SORTING   = 1 << 3,
331         FL_NEED_FILTERING = 1 << 4,
332 };
333
334 #define SPECIAL_IMG_SIZE 48
335 #define SPECIAL_IMG_ROWS 4
336 #define SPECIAL_IMG_COLS 4
337
338 enum {
339         SPECIAL_IMG_FOLDER      = 0,
340         SPECIAL_IMG_PARENT      = 1,
341         SPECIAL_IMG_REFRESH     = 2,
342         SPECIAL_IMG_BLENDFILE   = 3,
343         SPECIAL_IMG_SOUNDFILE   = 4,
344         SPECIAL_IMG_MOVIEFILE   = 5,
345         SPECIAL_IMG_PYTHONFILE  = 6,
346         SPECIAL_IMG_TEXTFILE    = 7,
347         SPECIAL_IMG_FONTFILE    = 8,
348         SPECIAL_IMG_UNKNOWNFILE = 9,
349         SPECIAL_IMG_LOADING     = 10,
350         SPECIAL_IMG_BACKUP      = 11,
351         SPECIAL_IMG_MAX
352 };
353
354 static ImBuf *gSpecialFileImages[SPECIAL_IMG_MAX];
355
356
357 static void filelist_readjob_main(struct FileList *, const char *, short *, short *, float *, ThreadMutex *);
358 static void filelist_readjob_lib(struct FileList *, const char *, short *, short *, float *, ThreadMutex *);
359 static void filelist_readjob_dir(struct FileList *, const char *, short *, short *, float *, ThreadMutex *);
360
361 /* helper, could probably go in BKE actually? */
362 static int groupname_to_code(const char *group);
363 static unsigned int groupname_to_filter_id(const char *group);
364
365 static void filelist_filter_clear(FileList *filelist);
366 static void filelist_cache_clear(FileListEntryCache *cache, size_t new_size);
367
368 /* ********** Sort helpers ********** */
369
370 static int compare_direntry_generic(const FileListInternEntry *entry1, const FileListInternEntry *entry2)
371 {
372         /* type is equal to stat.st_mode */
373
374         if (entry1->typeflag & FILE_TYPE_DIR) {
375                 if (entry2->typeflag & FILE_TYPE_DIR) {
376                         /* If both entries are tagged as dirs, we make a 'sub filter' that shows first the real dirs,
377                          * then libs (.blend files), then categories in libs. */
378                         if (entry1->typeflag & FILE_TYPE_BLENDERLIB) {
379                                 if (!(entry2->typeflag & FILE_TYPE_BLENDERLIB)) {
380                                         return 1;
381                                 }
382                         }
383                         else if (entry2->typeflag & FILE_TYPE_BLENDERLIB) {
384                                 return -1;
385                         }
386                         else if (entry1->typeflag & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) {
387                                 if (!(entry2->typeflag & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP))) {
388                                         return 1;
389                                 }
390                         }
391                         else if (entry2->typeflag & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) {
392                                 return -1;
393                         }
394                 }
395                 else {
396                         return -1;
397                 }
398         }
399         else if (entry2->typeflag & FILE_TYPE_DIR) {
400                 return 1;
401         }
402
403         /* make sure "." and ".." are always first */
404         if (FILENAME_IS_CURRENT(entry1->relpath)) return -1;
405         if (FILENAME_IS_CURRENT(entry2->relpath)) return 1;
406         if (FILENAME_IS_PARENT(entry1->relpath)) return -1;
407         if (FILENAME_IS_PARENT(entry2->relpath)) return 1;
408
409         return 0;
410 }
411
412 static int compare_name(void *UNUSED(user_data), const void *a1, const void *a2)
413 {
414         const FileListInternEntry *entry1 = a1;
415         const FileListInternEntry *entry2 = a2;
416         char *name1, *name2;
417         int ret;
418
419         if ((ret = compare_direntry_generic(entry1, entry2))) {
420                 return ret;
421         }
422
423         name1 = entry1->name;
424         name2 = entry2->name;
425
426         return BLI_natstrcmp(name1, name2);
427 }
428
429 static int compare_date(void *UNUSED(user_data), const void *a1, const void *a2)
430 {
431         const FileListInternEntry *entry1 = a1;
432         const FileListInternEntry *entry2 = a2;
433         char *name1, *name2;
434         int64_t time1, time2;
435         int ret;
436
437         if ((ret = compare_direntry_generic(entry1, entry2))) {
438                 return ret;
439         }
440
441         time1 = (int64_t)entry1->st.st_mtime;
442         time2 = (int64_t)entry2->st.st_mtime;
443         if (time1 < time2) return 1;
444         if (time1 > time2) return -1;
445
446         name1 = entry1->name;
447         name2 = entry2->name;
448
449         return BLI_natstrcmp(name1, name2);
450 }
451
452 static int compare_size(void *UNUSED(user_data), const void *a1, const void *a2)
453 {
454         const FileListInternEntry *entry1 = a1;
455         const FileListInternEntry *entry2 = a2;
456         char *name1, *name2;
457         uint64_t size1, size2;
458         int ret;
459
460         if ((ret = compare_direntry_generic(entry1, entry2))) {
461                 return ret;
462         }
463
464         size1 = entry1->st.st_size;
465         size2 = entry2->st.st_size;
466         if (size1 < size2) return 1;
467         if (size1 > size2) return -1;
468
469         name1 = entry1->name;
470         name2 = entry2->name;
471
472         return BLI_natstrcmp(name1, name2);
473 }
474
475 static int compare_extension(void *UNUSED(user_data), const void *a1, const void *a2)
476 {
477         const FileListInternEntry *entry1 = a1;
478         const FileListInternEntry *entry2 = a2;
479         char *name1, *name2;
480         int ret;
481
482         if ((ret = compare_direntry_generic(entry1, entry2))) {
483                 return ret;
484         }
485
486         if ((entry1->typeflag & FILE_TYPE_BLENDERLIB) && !(entry2->typeflag & FILE_TYPE_BLENDERLIB)) return -1;
487         if (!(entry1->typeflag & FILE_TYPE_BLENDERLIB) && (entry2->typeflag & FILE_TYPE_BLENDERLIB)) return 1;
488         if ((entry1->typeflag & FILE_TYPE_BLENDERLIB) && (entry2->typeflag & FILE_TYPE_BLENDERLIB)) {
489                 if ((entry1->typeflag & FILE_TYPE_DIR) && !(entry2->typeflag & FILE_TYPE_DIR)) return 1;
490                 if (!(entry1->typeflag & FILE_TYPE_DIR) && (entry2->typeflag & FILE_TYPE_DIR)) return -1;
491                 if (entry1->blentype < entry2->blentype) return -1;
492                 if (entry1->blentype > entry2->blentype) return 1;
493         }
494         else {
495                 const char *sufix1, *sufix2;
496
497                 if (!(sufix1 = strstr(entry1->relpath, ".blend.gz")))
498                         sufix1 = strrchr(entry1->relpath, '.');
499                 if (!(sufix2 = strstr(entry2->relpath, ".blend.gz")))
500                         sufix2 = strrchr(entry2->relpath, '.');
501                 if (!sufix1) sufix1 = "";
502                 if (!sufix2) sufix2 = "";
503
504                 if ((ret = BLI_strcasecmp(sufix1, sufix2))) {
505                         return ret;
506                 }
507         }
508
509         name1 = entry1->name;
510         name2 = entry2->name;
511
512         return BLI_natstrcmp(name1, name2);
513 }
514
515 void filelist_sort(struct FileList *filelist)
516 {
517         if ((filelist->flags & FL_NEED_SORTING) && (filelist->sort != FILE_SORT_NONE)) {
518                 switch (filelist->sort) {
519                         case FILE_SORT_ALPHA:
520                                 BLI_listbase_sort_r(&filelist->filelist_intern.entries, compare_name, NULL);
521                                 break;
522                         case FILE_SORT_TIME:
523                                 BLI_listbase_sort_r(&filelist->filelist_intern.entries, compare_date, NULL);
524                                 break;
525                         case FILE_SORT_SIZE:
526                                 BLI_listbase_sort_r(&filelist->filelist_intern.entries, compare_size, NULL);
527                                 break;
528                         case FILE_SORT_EXTENSION:
529                                 BLI_listbase_sort_r(&filelist->filelist_intern.entries, compare_extension, NULL);
530                                 break;
531                         case FILE_SORT_NONE:  /* Should never reach this point! */
532                         default:
533                                 BLI_assert(0);
534                                 break;
535                 }
536
537                 filelist_filter_clear(filelist);
538                 filelist->flags &= ~FL_NEED_SORTING;
539         }
540 }
541
542 void filelist_setsorting(struct FileList *filelist, const short sort)
543 {
544         if (filelist->sort != sort) {
545                 filelist->sort = sort;
546                 filelist->flags |= FL_NEED_SORTING;
547         }
548 }
549
550 /* ********** Filter helpers ********** */
551
552 static bool is_hidden_file(const char *filename, FileListFilter *filter)
553 {
554         char *sep = (char *)BLI_last_slash(filename);
555         bool is_hidden = false;
556
557         if (filter->flags & FLF_HIDE_DOT) {
558                 if (filename[0] == '.' && filename[1] != '.' && filename[1] != '\0') {
559                         is_hidden = true; /* ignore .file */
560                 }
561                 else {
562                         int len = strlen(filename);
563                         if ((len > 0) && (filename[len - 1] == '~')) {
564                                 is_hidden = true;  /* ignore file~ */
565                         }
566                 }
567         }
568         if (!is_hidden && (filter->flags & FLF_HIDE_PARENT)) {
569                 if (filename[0] == '.' && filename[1] == '.' && filename[2] == '\0') {
570                         is_hidden = true; /* ignore .. */
571                 }
572         }
573         if (!is_hidden && ((filename[0] == '.') && (filename[1] == '\0'))) {
574                 is_hidden = true; /* ignore . */
575         }
576         /* filename might actually be a piece of path, in which case we have to check all its parts. */
577         if (!is_hidden && sep) {
578                 char tmp_filename[FILE_MAX_LIBEXTRA];
579
580                 BLI_strncpy(tmp_filename, filename, sizeof(tmp_filename));
581                 sep = tmp_filename + (sep - filename);
582                 while (sep) {
583                         BLI_assert(sep[1] != '\0');
584                         if (is_hidden_file(sep + 1, filter)) {
585                                 is_hidden = true;
586                                 break;
587                         }
588                         *sep = '\0';
589                         sep = (char *)BLI_last_slash(tmp_filename);
590                 }
591         }
592         return is_hidden;
593 }
594
595 static bool is_filtered_file(FileListInternEntry *file, const char *UNUSED(root), FileListFilter *filter)
596 {
597         bool is_filtered = !is_hidden_file(file->relpath, filter);
598
599         if (is_filtered && (filter->flags & FLF_DO_FILTER) && !FILENAME_IS_CURRPAR(file->relpath)) {
600                 /* We only check for types if some type are enabled in filtering. */
601                 if (filter->filter) {
602                         if (file->typeflag & FILE_TYPE_DIR) {
603                                 if (file->typeflag & (FILE_TYPE_BLENDERLIB | FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) {
604                                         if (!(filter->filter & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP))) {
605                                                 is_filtered = false;
606                                         }
607                                 }
608                                 else {
609                                         if (!(filter->filter & FILE_TYPE_FOLDER)) {
610                                                 is_filtered = false;
611                                         }
612                                 }
613                         }
614                         else {
615                                 if (!(file->typeflag & filter->filter)) {
616                                         is_filtered = false;
617                                 }
618                         }
619                 }
620                 if (is_filtered && (filter->filter_search[0] != '\0')) {
621                         if (fnmatch(filter->filter_search, file->relpath, FNM_CASEFOLD) != 0) {
622                                 is_filtered = false;
623                         }
624                 }
625         }
626
627         return is_filtered;
628 }
629
630 static bool is_filtered_lib(FileListInternEntry *file, const char *root, FileListFilter *filter)
631 {
632         bool is_filtered;
633         char path[FILE_MAX_LIBEXTRA], dir[FILE_MAX_LIBEXTRA], *group, *name;
634
635         BLI_join_dirfile(path, sizeof(path), root, file->relpath);
636
637         if (BLO_library_path_explode(path, dir, &group, &name)) {
638                 is_filtered = !is_hidden_file(file->relpath, filter);
639                 if (is_filtered && (filter->flags & FLF_DO_FILTER) && !FILENAME_IS_CURRPAR(file->relpath)) {
640                         /* We only check for types if some type are enabled in filtering. */
641                         if (filter->filter || filter->filter_id) {
642                                 if (file->typeflag & FILE_TYPE_DIR) {
643                                         if (file->typeflag & (FILE_TYPE_BLENDERLIB | FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) {
644                                                 if (!(filter->filter & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP))) {
645                                                         is_filtered = false;
646                                                 }
647                                         }
648                                         else {
649                                                 if (!(filter->filter & FILE_TYPE_FOLDER)) {
650                                                         is_filtered = false;
651                                                 }
652                                         }
653                                 }
654                                 if (is_filtered && group) {
655                                         if (!name && (filter->flags & FLF_HIDE_LIB_DIR)) {
656                                                 is_filtered = false;
657                                         }
658                                         else {
659                                                 unsigned int filter_id = groupname_to_filter_id(group);
660                                                 if (!(filter_id & filter->filter_id)) {
661                                                         is_filtered = false;
662                                                 }
663                                         }
664                                 }
665                         }
666                         if (is_filtered && (filter->filter_search[0] != '\0')) {
667                                 if (fnmatch(filter->filter_search, file->relpath, FNM_CASEFOLD) != 0) {
668                                         is_filtered = false;
669                                 }
670                         }
671                 }
672         }
673         else {
674                 is_filtered = is_filtered_file(file, root, filter);
675         }
676
677         return is_filtered;
678 }
679
680 static bool is_filtered_main(FileListInternEntry *file, const char *UNUSED(dir), FileListFilter *filter)
681 {
682         return !is_hidden_file(file->relpath, filter);
683 }
684
685 static void filelist_filter_clear(FileList *filelist)
686 {
687         filelist->flags |= FL_NEED_FILTERING;
688 }
689
690 void filelist_filter(FileList *filelist)
691 {
692         int num_filtered = 0;
693         const int num_files = filelist->filelist.nbr_entries;
694         FileListInternEntry **filtered_tmp, *file;
695
696         if (filelist->filelist.nbr_entries == 0) {
697                 return;
698         }
699
700         if (!(filelist->flags & FL_NEED_FILTERING)) {
701                 /* Assume it has already been filtered, nothing else to do! */
702                 return;
703         }
704
705         filelist->filter_data.flags &= ~FLF_HIDE_LIB_DIR;
706         if (filelist->max_recursion) {
707                 /* Never show lib ID 'categories' directories when we are in 'flat' mode, unless
708                  * root path is a blend file. */
709                 char dir[FILE_MAX_LIBEXTRA];
710                 if (!filelist_islibrary(filelist, dir, NULL)) {
711                         filelist->filter_data.flags |= FLF_HIDE_LIB_DIR;
712                 }
713         }
714
715         filtered_tmp = MEM_mallocN(sizeof(*filtered_tmp) * (size_t)num_files, __func__);
716
717         /* Filter remap & count how many files are left after filter in a single loop. */
718         for (file = filelist->filelist_intern.entries.first; file; file = file->next) {
719                 if (filelist->filterf(file, filelist->filelist.root, &filelist->filter_data)) {
720                         filtered_tmp[num_filtered++] = file;
721                 }
722         }
723
724         if (filelist->filelist_intern.filtered) {
725                 MEM_freeN(filelist->filelist_intern.filtered);
726         }
727         filelist->filelist_intern.filtered = MEM_mallocN(sizeof(*filelist->filelist_intern.filtered) * (size_t)num_filtered,
728                                                          __func__);
729         memcpy(filelist->filelist_intern.filtered, filtered_tmp,
730                sizeof(*filelist->filelist_intern.filtered) * (size_t)num_filtered);
731         filelist->filelist.nbr_entries_filtered = num_filtered;
732 //      printf("Filetered: %d over %d entries\n", num_filtered, filelist->filelist.nbr_entries);
733
734         filelist_cache_clear(&filelist->filelist_cache, filelist->filelist_cache.size);
735         filelist->flags &= ~FL_NEED_FILTERING;
736
737         MEM_freeN(filtered_tmp);
738 }
739
740 void filelist_setfilter_options(FileList *filelist, const bool do_filter,
741                                 const bool hide_dot, const bool hide_parent,
742                                 const unsigned int filter, const unsigned int filter_id,
743                                 const char *filter_glob, const char *filter_search)
744 {
745         bool update = false;
746
747         if (((filelist->filter_data.flags & FLF_DO_FILTER) != 0) != (do_filter != 0)) {
748                 filelist->filter_data.flags ^= FLF_DO_FILTER;
749                 update = true;
750         }
751         if (((filelist->filter_data.flags & FLF_HIDE_DOT) != 0) != (hide_dot != 0)) {
752                 filelist->filter_data.flags ^= FLF_HIDE_DOT;
753                 update = true;
754         }
755         if (((filelist->filter_data.flags & FLF_HIDE_PARENT) != 0) != (hide_parent != 0)) {
756                 filelist->filter_data.flags ^= FLF_HIDE_PARENT;
757                 update = true;
758         }
759         if ((filelist->filter_data.filter != filter) || (filelist->filter_data.filter_id != filter_id)) {
760                 filelist->filter_data.filter = filter;
761                 filelist->filter_data.filter_id = filter_id;
762                 update = true;
763         }
764         if (!STREQ(filelist->filter_data.filter_glob, filter_glob)) {
765                 BLI_strncpy(filelist->filter_data.filter_glob, filter_glob, sizeof(filelist->filter_data.filter_glob));
766                 update = true;
767         }
768         if ((BLI_strcmp_ignore_pad(filelist->filter_data.filter_search, filter_search, '*') != 0)) {
769                 BLI_strncpy_ensure_pad(filelist->filter_data.filter_search, filter_search, '*',
770                                        sizeof(filelist->filter_data.filter_search));
771                 update = true;
772         }
773
774         if (update) {
775                 /* And now, free filtered data so that we know we have to filter again. */
776                 filelist_filter_clear(filelist);
777         }
778 }
779
780 /* ********** Icon/image helpers ********** */
781
782 void filelist_init_icons(void)
783 {
784         short x, y, k;
785         ImBuf *bbuf;
786         ImBuf *ibuf;
787
788         BLI_assert(G.background == false);
789
790 #ifdef WITH_HEADLESS
791         bbuf = NULL;
792 #else
793         bbuf = IMB_ibImageFromMemory((unsigned char *)datatoc_prvicons_png, datatoc_prvicons_png_size, IB_rect, NULL, "<splash>");
794 #endif
795         if (bbuf) {
796                 for (y = 0; y < SPECIAL_IMG_ROWS; y++) {
797                         for (x = 0; x < SPECIAL_IMG_COLS; x++) {
798                                 int tile = SPECIAL_IMG_COLS * y + x;
799                                 if (tile < SPECIAL_IMG_MAX) {
800                                         ibuf = IMB_allocImBuf(SPECIAL_IMG_SIZE, SPECIAL_IMG_SIZE, 32, IB_rect);
801                                         for (k = 0; k < SPECIAL_IMG_SIZE; k++) {
802                                                 memcpy(&ibuf->rect[k * SPECIAL_IMG_SIZE], &bbuf->rect[(k + y * SPECIAL_IMG_SIZE) * SPECIAL_IMG_SIZE * SPECIAL_IMG_COLS + x * SPECIAL_IMG_SIZE], SPECIAL_IMG_SIZE * sizeof(int));
803                                         }
804                                         gSpecialFileImages[tile] = ibuf;
805                                 }
806                         }
807                 }
808                 IMB_freeImBuf(bbuf);
809         }
810 }
811
812 void filelist_free_icons(void)
813 {
814         int i;
815
816         BLI_assert(G.background == false);
817
818         for (i = 0; i < SPECIAL_IMG_MAX; ++i) {
819                 IMB_freeImBuf(gSpecialFileImages[i]);
820                 gSpecialFileImages[i] = NULL;
821         }
822 }
823
824 void filelist_imgsize(struct FileList *filelist, short w, short h)
825 {
826         filelist->prv_w = w;
827         filelist->prv_h = h;
828 }
829
830 static FileDirEntry *filelist_geticon_get_file(struct FileList *filelist, const int index)
831 {
832         BLI_assert(G.background == false);
833
834         return filelist_file(filelist, index);
835 }
836
837 ImBuf *filelist_getimage(struct FileList *filelist, const int index)
838 {
839         FileDirEntry *file = filelist_geticon_get_file(filelist, index);
840
841         return file->image;
842 }
843
844 static ImBuf *filelist_geticon_image_ex(const unsigned int typeflag, const char *relpath)
845 {
846         ImBuf *ibuf = NULL;
847
848         if (typeflag & FILE_TYPE_DIR) {
849                 if (FILENAME_IS_PARENT(relpath)) {
850                         ibuf = gSpecialFileImages[SPECIAL_IMG_PARENT];
851                 }
852                 else if (FILENAME_IS_CURRENT(relpath)) {
853                         ibuf = gSpecialFileImages[SPECIAL_IMG_REFRESH];
854                 }
855                 else {
856                         ibuf = gSpecialFileImages[SPECIAL_IMG_FOLDER];
857                 }
858         }
859         else if (typeflag & FILE_TYPE_BLENDER) {
860                 ibuf = gSpecialFileImages[SPECIAL_IMG_BLENDFILE];
861         }
862         else if (typeflag & FILE_TYPE_BLENDERLIB) {
863                 ibuf = gSpecialFileImages[SPECIAL_IMG_UNKNOWNFILE];
864         }
865         else if (typeflag & (FILE_TYPE_MOVIE)) {
866                 ibuf = gSpecialFileImages[SPECIAL_IMG_MOVIEFILE];
867         }
868         else if (typeflag & FILE_TYPE_SOUND) {
869                 ibuf = gSpecialFileImages[SPECIAL_IMG_SOUNDFILE];
870         }
871         else if (typeflag & FILE_TYPE_PYSCRIPT) {
872                 ibuf = gSpecialFileImages[SPECIAL_IMG_PYTHONFILE];
873         }
874         else if (typeflag & FILE_TYPE_FTFONT) {
875                 ibuf = gSpecialFileImages[SPECIAL_IMG_FONTFILE];
876         }
877         else if (typeflag & FILE_TYPE_TEXT) {
878                 ibuf = gSpecialFileImages[SPECIAL_IMG_TEXTFILE];
879         }
880         else if (typeflag & FILE_TYPE_IMAGE) {
881                 ibuf = gSpecialFileImages[SPECIAL_IMG_LOADING];
882         }
883         else if (typeflag & FILE_TYPE_BLENDER_BACKUP) {
884                 ibuf = gSpecialFileImages[SPECIAL_IMG_BACKUP];
885         }
886         else {
887                 ibuf = gSpecialFileImages[SPECIAL_IMG_UNKNOWNFILE];
888         }
889
890         return ibuf;
891 }
892
893 ImBuf *filelist_geticon_image(struct FileList *filelist, const int index)
894 {
895         FileDirEntry *file = filelist_geticon_get_file(filelist, index);
896
897         return filelist_geticon_image_ex(file->typeflag, file->relpath);
898 }
899
900 static int filelist_geticon_ex(
901         const int typeflag, const int blentype, const char *relpath, const bool is_main, const bool ignore_libdir)
902 {
903         if ((typeflag & FILE_TYPE_DIR) && !(ignore_libdir && (typeflag & (FILE_TYPE_BLENDERLIB | FILE_TYPE_BLENDER)))) {
904                 if (FILENAME_IS_PARENT(relpath)) {
905                         return is_main ? ICON_FILE_PARENT : ICON_NONE;
906                 }
907                 else if (typeflag & FILE_TYPE_APPLICATIONBUNDLE) {
908                         return ICON_UGLYPACKAGE;
909                 }
910                 else if (typeflag & FILE_TYPE_BLENDER) {
911                         return ICON_FILE_BLEND;
912                 }
913                 else if (is_main) {
914                         /* Do not return icon for folders if icons are not 'main' draw type (e.g. when used over previews). */
915                         return ICON_FILE_FOLDER;
916                 }
917         }
918
919         if (typeflag & FILE_TYPE_BLENDER)
920                 return ICON_FILE_BLEND;
921         else if (typeflag & FILE_TYPE_BLENDER_BACKUP)
922                 return ICON_FILE_BACKUP;
923         else if (typeflag & FILE_TYPE_IMAGE)
924                 return ICON_FILE_IMAGE;
925         else if (typeflag & FILE_TYPE_MOVIE)
926                 return ICON_FILE_MOVIE;
927         else if (typeflag & FILE_TYPE_PYSCRIPT)
928                 return ICON_FILE_SCRIPT;
929         else if (typeflag & FILE_TYPE_SOUND)
930                 return ICON_FILE_SOUND;
931         else if (typeflag & FILE_TYPE_FTFONT)
932                 return ICON_FILE_FONT;
933         else if (typeflag & FILE_TYPE_BTX)
934                 return ICON_FILE_BLANK;
935         else if (typeflag & FILE_TYPE_COLLADA)
936                 return ICON_FILE_BLANK;
937         else if (typeflag & FILE_TYPE_ALEMBIC)
938                 return ICON_FILE_BLANK;
939         else if (typeflag & FILE_TYPE_TEXT)
940                 return ICON_FILE_TEXT;
941         else if (typeflag & FILE_TYPE_BLENDERLIB) {
942                 const int ret = UI_idcode_icon_get(blentype);
943                 if (ret != ICON_NONE) {
944                         return ret;
945                 }
946         }
947         return is_main ? ICON_FILE_BLANK : ICON_NONE;
948 }
949
950 int filelist_geticon(struct FileList *filelist, const int index, const bool is_main)
951 {
952         FileDirEntry *file = filelist_geticon_get_file(filelist, index);
953
954         return filelist_geticon_ex(file->typeflag, file->blentype, file->relpath, is_main, false);
955 }
956
957 /* ********** Main ********** */
958
959 static bool filelist_checkdir_dir(struct FileList *UNUSED(filelist), char *r_dir, const bool do_change)
960 {
961         if (do_change) {
962                 BLI_make_exist(r_dir);
963                 return true;
964         }
965         else {
966                 return BLI_is_dir(r_dir);
967         }
968 }
969
970 static bool filelist_checkdir_lib(struct FileList *UNUSED(filelist), char *r_dir, const bool do_change)
971 {
972         char tdir[FILE_MAX_LIBEXTRA];
973         char *name;
974
975         const bool is_valid = (BLI_is_dir(r_dir) ||
976                                (BLO_library_path_explode(r_dir, tdir, NULL, &name) && BLI_is_file(tdir) && !name));
977
978         if (do_change && !is_valid) {
979                 /* if not a valid library, we need it to be a valid directory! */
980                 BLI_make_exist(r_dir);
981                 return true;
982         }
983         return is_valid;
984 }
985
986 static bool filelist_checkdir_main(struct FileList *filelist, char *r_dir, const bool do_change)
987 {
988         /* TODO */
989         return filelist_checkdir_lib(filelist, r_dir, do_change);
990 }
991
992 static void filelist_entry_clear(FileDirEntry *entry)
993 {
994         if (entry->name) {
995                 MEM_freeN(entry->name);
996         }
997         if (entry->description) {
998                 MEM_freeN(entry->description);
999         }
1000         if (entry->relpath) {
1001                 MEM_freeN(entry->relpath);
1002         }
1003         if (entry->image) {
1004                 IMB_freeImBuf(entry->image);
1005         }
1006         /* For now, consider FileDirEntryRevision::poin as not owned here, so no need to do anything about it */
1007
1008         if (!BLI_listbase_is_empty(&entry->variants)) {
1009                 FileDirEntryVariant *var;
1010
1011                 for (var = entry->variants.first; var; var = var->next) {
1012                         if (var->name) {
1013                                 MEM_freeN(var->name);
1014                         }
1015                         if (var->description) {
1016                                 MEM_freeN(var->description);
1017                         }
1018
1019                         if (!BLI_listbase_is_empty(&var->revisions)) {
1020                                 FileDirEntryRevision *rev;
1021
1022                                 for (rev = var->revisions.first; rev; rev = rev->next) {
1023                                         if (rev->comment) {
1024                                                 MEM_freeN(rev->comment);
1025                                         }
1026                                 }
1027
1028                                 BLI_freelistN(&var->revisions);
1029                         }
1030                 }
1031
1032                 /* TODO: tags! */
1033
1034                 BLI_freelistN(&entry->variants);
1035         }
1036         else if (entry->entry) {
1037                 MEM_freeN(entry->entry);
1038         }
1039 }
1040
1041 static void filelist_entry_free(FileDirEntry *entry)
1042 {
1043         filelist_entry_clear(entry);
1044         MEM_freeN(entry);
1045 }
1046
1047 static void filelist_direntryarr_free(FileDirEntryArr *array)
1048 {
1049 #if 0
1050         FileDirEntry *entry, *entry_next;
1051
1052         for (entry = array->entries.first; entry; entry = entry_next) {
1053                 entry_next = entry->next;
1054                 filelist_entry_free(entry);
1055         }
1056         BLI_listbase_clear(&array->entries);
1057 #else
1058         BLI_assert(BLI_listbase_is_empty(&array->entries));
1059 #endif
1060         array->nbr_entries = 0;
1061         array->nbr_entries_filtered = -1;
1062         array->entry_idx_start = -1;
1063         array->entry_idx_end = -1;
1064 }
1065
1066 static void filelist_intern_entry_free(FileListInternEntry *entry)
1067 {
1068         if (entry->relpath) {
1069                 MEM_freeN(entry->relpath);
1070         }
1071         if (entry->name) {
1072                 MEM_freeN(entry->name);
1073         }
1074         MEM_freeN(entry);
1075 }
1076
1077 static void filelist_intern_free(FileListIntern *filelist_intern)
1078 {
1079         FileListInternEntry *entry, *entry_next;
1080
1081         for (entry = filelist_intern->entries.first; entry; entry = entry_next) {
1082                 entry_next = entry->next;
1083                 filelist_intern_entry_free(entry);
1084         }
1085         BLI_listbase_clear(&filelist_intern->entries);
1086
1087         MEM_SAFE_FREE(filelist_intern->filtered);
1088 }
1089
1090 static void filelist_cache_preview_runf(TaskPool *__restrict pool, void *taskdata, int UNUSED(threadid))
1091 {
1092         FileListEntryCache *cache = BLI_task_pool_userdata(pool);
1093         FileListEntryPreview *preview = taskdata;
1094
1095         ThumbSource source = 0;
1096
1097 //      printf("%s: Start (%d)...\n", __func__, threadid);
1098
1099 //      printf("%s: %d - %s - %p\n", __func__, preview->index, preview->path, preview->img);
1100         BLI_assert(preview->flags & (FILE_TYPE_IMAGE | FILE_TYPE_MOVIE | FILE_TYPE_FTFONT |
1101                                      FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP | FILE_TYPE_BLENDERLIB));
1102
1103         if (preview->flags & FILE_TYPE_IMAGE) {
1104                 source = THB_SOURCE_IMAGE;
1105         }
1106         else if (preview->flags & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP | FILE_TYPE_BLENDERLIB)) {
1107                 source = THB_SOURCE_BLEND;
1108         }
1109         else if (preview->flags & FILE_TYPE_MOVIE) {
1110                 source = THB_SOURCE_MOVIE;
1111         }
1112         else if (preview->flags & FILE_TYPE_FTFONT) {
1113                 source = THB_SOURCE_FONT;
1114         }
1115
1116         IMB_thumb_path_lock(preview->path);
1117         preview->img = IMB_thumb_manage(preview->path, THB_LARGE, source);
1118         IMB_thumb_path_unlock(preview->path);
1119
1120         /* Used to tell free func to not free anything.
1121          * Note that we do not care about cas result here,
1122          * we only want value attribution itself to be atomic (and memory barier).*/
1123         atomic_cas_uint32(&preview->flags, preview->flags, 0);
1124         BLI_thread_queue_push(cache->previews_done, preview);
1125
1126 //      printf("%s: End (%d)...\n", __func__, threadid);
1127 }
1128
1129 static void filelist_cache_preview_freef(TaskPool * __restrict UNUSED(pool), void *taskdata, int UNUSED(threadid))
1130 {
1131         FileListEntryPreview *preview = taskdata;
1132
1133         /* If preview->flag is empty, it means that preview has already been generated and added to done queue,
1134          * we do not own it anymore. */
1135         if (preview->flags) {
1136                 if (preview->img) {
1137                         IMB_freeImBuf(preview->img);
1138                 }
1139                 MEM_freeN(preview);
1140         }
1141 }
1142
1143 static void filelist_cache_preview_ensure_running(FileListEntryCache *cache)
1144 {
1145         if (!cache->previews_pool) {
1146                 TaskScheduler *scheduler = BLI_task_scheduler_get();
1147
1148                 cache->previews_pool = BLI_task_pool_create_background(scheduler, cache);
1149                 cache->previews_done = BLI_thread_queue_init();
1150
1151                 IMB_thumb_locks_acquire();
1152         }
1153 }
1154
1155 static void filelist_cache_previews_clear(FileListEntryCache *cache)
1156 {
1157         FileListEntryPreview *preview;
1158
1159         if (cache->previews_pool) {
1160                 BLI_task_pool_cancel(cache->previews_pool);
1161
1162                 while ((preview = BLI_thread_queue_pop_timeout(cache->previews_done, 0))) {
1163 //                      printf("%s: DONE %d - %s - %p\n", __func__, preview->index, preview->path, preview->img);
1164                         if (preview->img) {
1165                                 IMB_freeImBuf(preview->img);
1166                         }
1167                         MEM_freeN(preview);
1168                 }
1169         }
1170 }
1171
1172 static void filelist_cache_previews_free(FileListEntryCache *cache)
1173 {
1174         if (cache->previews_pool) {
1175                 BLI_thread_queue_nowait(cache->previews_done);
1176
1177                 filelist_cache_previews_clear(cache);
1178
1179                 BLI_thread_queue_free(cache->previews_done);
1180                 BLI_task_pool_free(cache->previews_pool);
1181                 cache->previews_pool = NULL;
1182                 cache->previews_done = NULL;
1183
1184                 IMB_thumb_locks_release();
1185         }
1186
1187         cache->flags &= ~FLC_PREVIEWS_ACTIVE;
1188 }
1189
1190 static void filelist_cache_previews_push(FileList *filelist, FileDirEntry *entry, const int index)
1191 {
1192         FileListEntryCache *cache = &filelist->filelist_cache;
1193
1194         BLI_assert(cache->flags & FLC_PREVIEWS_ACTIVE);
1195
1196         if (!entry->image &&
1197             !(entry->flags & FILE_ENTRY_INVALID_PREVIEW) &&
1198             (entry->typeflag & (FILE_TYPE_IMAGE | FILE_TYPE_MOVIE | FILE_TYPE_FTFONT |
1199                                 FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP | FILE_TYPE_BLENDERLIB)))
1200         {
1201                 FileListEntryPreview *preview = MEM_mallocN(sizeof(*preview), __func__);
1202                 BLI_join_dirfile(preview->path, sizeof(preview->path), filelist->filelist.root, entry->relpath);
1203                 preview->index = index;
1204                 preview->flags = entry->typeflag;
1205                 preview->img = NULL;
1206 //              printf("%s: %d - %s - %p\n", __func__, preview->index, preview->path, preview->img);
1207
1208                 filelist_cache_preview_ensure_running(cache);
1209                 BLI_task_pool_push_ex(cache->previews_pool, filelist_cache_preview_runf, preview,
1210                                       true, filelist_cache_preview_freef, TASK_PRIORITY_LOW);
1211         }
1212 }
1213
1214 static void filelist_cache_init(FileListEntryCache *cache, size_t cache_size)
1215 {
1216         BLI_listbase_clear(&cache->cached_entries);
1217
1218         cache->block_cursor = cache->block_start_index = cache->block_center_index = cache->block_end_index = 0;
1219         cache->block_entries = MEM_mallocN(sizeof(*cache->block_entries) * cache_size, __func__);
1220
1221         cache->misc_entries = BLI_ghash_ptr_new_ex(__func__, cache_size);
1222         cache->misc_entries_indices = MEM_mallocN(sizeof(*cache->misc_entries_indices) * cache_size, __func__);
1223         copy_vn_i(cache->misc_entries_indices, cache_size, -1);
1224         cache->misc_cursor = 0;
1225
1226         /* XXX This assumes uint is 32 bits and uuid is 128 bits (char[16]), be careful! */
1227         cache->uuids = BLI_ghash_new_ex(
1228                            BLI_ghashutil_uinthash_v4_p, BLI_ghashutil_uinthash_v4_cmp, __func__, cache_size * 2);
1229
1230         cache->size = cache_size;
1231         cache->flags = FLC_IS_INIT;
1232 }
1233
1234 static void filelist_cache_free(FileListEntryCache *cache)
1235 {
1236         FileDirEntry *entry, *entry_next;
1237
1238         if (!(cache->flags & FLC_IS_INIT)) {
1239                 return;
1240         }
1241
1242         filelist_cache_previews_free(cache);
1243
1244         MEM_freeN(cache->block_entries);
1245
1246         BLI_ghash_free(cache->misc_entries, NULL, NULL);
1247         MEM_freeN(cache->misc_entries_indices);
1248
1249         BLI_ghash_free(cache->uuids, NULL, NULL);
1250
1251         for (entry = cache->cached_entries.first; entry; entry = entry_next) {
1252                 entry_next = entry->next;
1253                 filelist_entry_free(entry);
1254         }
1255         BLI_listbase_clear(&cache->cached_entries);
1256 }
1257
1258 static void filelist_cache_clear(FileListEntryCache *cache, size_t new_size)
1259 {
1260         FileDirEntry *entry, *entry_next;
1261
1262         if (!(cache->flags & FLC_IS_INIT)) {
1263                 return;
1264         }
1265
1266         filelist_cache_previews_clear(cache);
1267
1268         cache->block_cursor = cache->block_start_index = cache->block_center_index = cache->block_end_index = 0;
1269         if (new_size != cache->size) {
1270                 cache->block_entries = MEM_reallocN(cache->block_entries, sizeof(*cache->block_entries) * new_size);
1271         }
1272
1273         BLI_ghash_clear_ex(cache->misc_entries, NULL, NULL, new_size);
1274         if (new_size != cache->size) {
1275                 cache->misc_entries_indices = MEM_reallocN(cache->misc_entries_indices,
1276                                                            sizeof(*cache->misc_entries_indices) * new_size);
1277         }
1278         copy_vn_i(cache->misc_entries_indices, new_size, -1);
1279
1280         BLI_ghash_clear_ex(cache->uuids, NULL, NULL, new_size * 2);
1281
1282         cache->size = new_size;
1283
1284         for (entry = cache->cached_entries.first; entry; entry = entry_next) {
1285                 entry_next = entry->next;
1286                 filelist_entry_free(entry);
1287         }
1288         BLI_listbase_clear(&cache->cached_entries);
1289 }
1290
1291 FileList *filelist_new(short type)
1292 {
1293         FileList *p = MEM_callocN(sizeof(*p), __func__);
1294
1295         filelist_cache_init(&p->filelist_cache, FILELIST_ENTRYCACHESIZE_DEFAULT);
1296
1297         p->selection_state = BLI_ghash_new(BLI_ghashutil_uinthash_v4_p, BLI_ghashutil_uinthash_v4_cmp, __func__);
1298
1299         switch (type) {
1300                 case FILE_MAIN:
1301                         p->checkdirf = filelist_checkdir_main;
1302                         p->read_jobf = filelist_readjob_main;
1303                         p->filterf = is_filtered_main;
1304                         break;
1305                 case FILE_LOADLIB:
1306                         p->checkdirf = filelist_checkdir_lib;
1307                         p->read_jobf = filelist_readjob_lib;
1308                         p->filterf = is_filtered_lib;
1309                         break;
1310                 default:
1311                         p->checkdirf = filelist_checkdir_dir;
1312                         p->read_jobf = filelist_readjob_dir;
1313                         p->filterf = is_filtered_file;
1314                         break;
1315         }
1316         return p;
1317 }
1318
1319 void filelist_clear_ex(struct FileList *filelist, const bool do_cache, const bool do_selection)
1320 {
1321         if (!filelist) {
1322                 return;
1323         }
1324
1325         filelist_filter_clear(filelist);
1326
1327         if (do_cache) {
1328                 filelist_cache_clear(&filelist->filelist_cache, filelist->filelist_cache.size);
1329         }
1330
1331         filelist_intern_free(&filelist->filelist_intern);
1332
1333         filelist_direntryarr_free(&filelist->filelist);
1334
1335         if (do_selection && filelist->selection_state) {
1336                 BLI_ghash_clear(filelist->selection_state, MEM_freeN, NULL);
1337         }
1338 }
1339
1340 void filelist_clear(struct FileList *filelist)
1341 {
1342         filelist_clear_ex(filelist, true, true);
1343 }
1344
1345 void filelist_free(struct FileList *filelist)
1346 {
1347         if (!filelist) {
1348                 printf("Attempting to delete empty filelist.\n");
1349                 return;
1350         }
1351
1352         filelist_clear_ex(filelist, false, false);  /* No need to clear cache & selection_state, we free them anyway. */
1353         filelist_cache_free(&filelist->filelist_cache);
1354
1355         if (filelist->selection_state) {
1356                 BLI_ghash_free(filelist->selection_state, MEM_freeN, NULL);
1357                 filelist->selection_state = NULL;
1358         }
1359
1360         memset(&filelist->filter_data, 0, sizeof(filelist->filter_data));
1361
1362         filelist->flags &= ~(FL_NEED_SORTING | FL_NEED_FILTERING);
1363         filelist->sort = FILE_SORT_NONE;
1364 }
1365
1366 void filelist_freelib(struct FileList *filelist)
1367 {
1368         if (filelist->libfiledata)
1369                 BLO_blendhandle_close(filelist->libfiledata);
1370         filelist->libfiledata = NULL;
1371 }
1372
1373 BlendHandle *filelist_lib(struct FileList *filelist)
1374 {
1375         return filelist->libfiledata;
1376 }
1377
1378 static const char *fileentry_uiname(const char *root, const char *relpath, const int typeflag, char *buff)
1379 {
1380         char *name = NULL;
1381
1382         if (typeflag & FILE_TYPE_BLENDERLIB) {
1383                 char abspath[FILE_MAX_LIBEXTRA];
1384                 char *group;
1385
1386                 BLI_join_dirfile(abspath, sizeof(abspath), root, relpath);
1387                 BLO_library_path_explode(abspath, buff, &group, &name);
1388                 if (!name) {
1389                         name = group;
1390                 }
1391         }
1392         /* Depending on platforms, 'my_file.blend/..' might be viewed as dir or not... */
1393         if (!name) {
1394                 if (typeflag & FILE_TYPE_DIR) {
1395                         name = (char *)relpath;
1396                 }
1397                 else {
1398                         name = (char *)BLI_path_basename(relpath);
1399                 }
1400         }
1401         BLI_assert(name);
1402
1403         return name;
1404 }
1405
1406 const char *filelist_dir(struct FileList *filelist)
1407 {
1408         return filelist->filelist.root;
1409 }
1410
1411 bool filelist_is_dir(struct FileList *filelist, const char *path)
1412 {
1413         return filelist->checkdirf(filelist, (char *)path, false);
1414 }
1415
1416 /**
1417  * May modify in place given r_dir, which is expected to be FILE_MAX_LIBEXTRA length.
1418  */
1419 void filelist_setdir(struct FileList *filelist, char *r_dir)
1420 {
1421         BLI_assert(strlen(r_dir) < FILE_MAX_LIBEXTRA);
1422
1423         BLI_cleanup_dir(BKE_main_blendfile_path_from_global(), r_dir);
1424         const bool is_valid_path = filelist->checkdirf(filelist, r_dir, true);
1425         BLI_assert(is_valid_path);
1426         UNUSED_VARS_NDEBUG(is_valid_path);
1427
1428         if (!STREQ(filelist->filelist.root, r_dir)) {
1429                 BLI_strncpy(filelist->filelist.root, r_dir, sizeof(filelist->filelist.root));
1430                 filelist->flags |= FL_FORCE_RESET;
1431         }
1432 }
1433
1434 void filelist_setrecursion(struct FileList *filelist, const int recursion_level)
1435 {
1436         if (filelist->max_recursion != recursion_level) {
1437                 filelist->max_recursion = recursion_level;
1438                 filelist->flags |= FL_FORCE_RESET;
1439         }
1440 }
1441
1442 bool filelist_force_reset(struct FileList *filelist)
1443 {
1444         return (filelist->flags & FL_FORCE_RESET) != 0;
1445 }
1446
1447 bool filelist_is_ready(struct FileList *filelist)
1448 {
1449         return (filelist->flags & FL_IS_READY) != 0;
1450 }
1451
1452 bool filelist_pending(struct FileList *filelist)
1453 {
1454         return (filelist->flags & FL_IS_PENDING) != 0;
1455 }
1456
1457 /**
1458  * Limited version of full update done by space_file's file_refresh(), to be used by operators and such.
1459  * Ensures given filelist is ready to be used (i.e. it is filtered and sorted), unless it is tagged for a full refresh.
1460  */
1461 int filelist_files_ensure(FileList *filelist)
1462 {
1463         if (!filelist_force_reset(filelist) || !filelist_empty(filelist)) {
1464                 filelist_sort(filelist);
1465                 filelist_filter(filelist);
1466         }
1467
1468         return filelist->filelist.nbr_entries_filtered;
1469 }
1470
1471 static FileDirEntry *filelist_file_create_entry(FileList *filelist, const int index)
1472 {
1473         FileListInternEntry *entry = filelist->filelist_intern.filtered[index];
1474         FileListEntryCache *cache = &filelist->filelist_cache;
1475         FileDirEntry *ret;
1476         FileDirEntryRevision *rev;
1477
1478         ret = MEM_callocN(sizeof(*ret), __func__);
1479         rev = MEM_callocN(sizeof(*rev), __func__);
1480
1481         rev->size = (uint64_t)entry->st.st_size;
1482
1483         rev->time = (int64_t)entry->st.st_mtime;
1484
1485         ret->entry = rev;
1486         ret->relpath = BLI_strdup(entry->relpath);
1487         ret->name = BLI_strdup(entry->name);
1488         ret->description = BLI_strdupcat(filelist->filelist.root, entry->relpath);
1489         memcpy(ret->uuid, entry->uuid, sizeof(ret->uuid));
1490         ret->blentype = entry->blentype;
1491         ret->typeflag = entry->typeflag;
1492
1493         BLI_addtail(&cache->cached_entries, ret);
1494         return ret;
1495 }
1496
1497 static void filelist_file_release_entry(FileList *filelist, FileDirEntry *entry)
1498 {
1499         BLI_remlink(&filelist->filelist_cache.cached_entries, entry);
1500         filelist_entry_free(entry);
1501 }
1502
1503 static FileDirEntry *filelist_file_ex(struct FileList *filelist, const int index, const bool use_request)
1504 {
1505         FileDirEntry *ret = NULL, *old;
1506         FileListEntryCache *cache = &filelist->filelist_cache;
1507         const size_t cache_size = cache->size;
1508         int old_index;
1509
1510         if ((index < 0) || (index >= filelist->filelist.nbr_entries_filtered)) {
1511                 return ret;
1512         }
1513
1514         if (index >= cache->block_start_index && index < cache->block_end_index) {
1515                 const int idx = (index - cache->block_start_index + cache->block_cursor) % cache_size;
1516                 return cache->block_entries[idx];
1517         }
1518
1519         if ((ret = BLI_ghash_lookup(cache->misc_entries, SET_INT_IN_POINTER(index)))) {
1520                 return ret;
1521         }
1522
1523         if (!use_request) {
1524                 return NULL;
1525         }
1526
1527 //      printf("requesting file %d (not yet cached)\n", index);
1528
1529         /* Else, we have to add new entry to 'misc' cache - and possibly make room for it first! */
1530         ret = filelist_file_create_entry(filelist, index);
1531         old_index = cache->misc_entries_indices[cache->misc_cursor];
1532         if ((old = BLI_ghash_popkey(cache->misc_entries, SET_INT_IN_POINTER(old_index), NULL))) {
1533                 BLI_ghash_remove(cache->uuids, old->uuid, NULL, NULL);
1534                 filelist_file_release_entry(filelist, old);
1535         }
1536         BLI_ghash_insert(cache->misc_entries, SET_INT_IN_POINTER(index), ret);
1537         BLI_ghash_insert(cache->uuids, ret->uuid, ret);
1538
1539         cache->misc_entries_indices[cache->misc_cursor] = index;
1540         cache->misc_cursor = (cache->misc_cursor + 1) % cache_size;
1541
1542 #if 0  /* Actually no, only block cached entries should have preview imho. */
1543         if (cache->previews_pool) {
1544                 filelist_cache_previews_push(filelist, ret, index);
1545         }
1546 #endif
1547
1548         return ret;
1549 }
1550
1551 FileDirEntry *filelist_file(struct FileList *filelist, int index)
1552 {
1553         return filelist_file_ex(filelist, index, true);
1554 }
1555
1556 int filelist_file_findpath(struct FileList *filelist, const char *filename)
1557 {
1558         int fidx = -1;
1559
1560         if (filelist->filelist.nbr_entries_filtered < 0) {
1561                 return fidx;
1562         }
1563
1564         /* XXX TODO Cache could probably use a ghash on paths too? Not really urgent though.
1565          *          This is only used to find again renamed entry, annoying but looks hairy to get rid of it currently. */
1566
1567         for (fidx = 0; fidx < filelist->filelist.nbr_entries_filtered; fidx++) {
1568                 FileListInternEntry *entry = filelist->filelist_intern.filtered[fidx];
1569                 if (STREQ(entry->relpath, filename)) {
1570                         return fidx;
1571                 }
1572         }
1573
1574         return -1;
1575 }
1576
1577 FileDirEntry *filelist_entry_find_uuid(struct FileList *filelist, const int uuid[4])
1578 {
1579         if (filelist->filelist.nbr_entries_filtered < 0) {
1580                 return NULL;
1581         }
1582
1583         if (filelist->filelist_cache.uuids) {
1584                 FileDirEntry *entry = BLI_ghash_lookup(filelist->filelist_cache.uuids, uuid);
1585                 if (entry) {
1586                         return entry;
1587                 }
1588         }
1589
1590         {
1591                 int fidx;
1592
1593                 for (fidx = 0; fidx < filelist->filelist.nbr_entries_filtered; fidx++) {
1594                         FileListInternEntry *entry = filelist->filelist_intern.filtered[fidx];
1595                         if (memcmp(entry->uuid, uuid, sizeof(entry->uuid)) == 0) {
1596                                 return filelist_file(filelist, fidx);
1597                         }
1598                 }
1599         }
1600
1601         return NULL;
1602 }
1603
1604 void filelist_file_cache_slidingwindow_set(FileList *filelist, size_t window_size)
1605 {
1606         /* Always keep it power of 2, in [256, 8192] range for now, cache being app. twice bigger than requested window. */
1607         size_t size = 256;
1608         window_size *= 2;
1609
1610         while (size < window_size && size < 8192) {
1611                 size *= 2;
1612         }
1613
1614         if (size != filelist->filelist_cache.size) {
1615                 filelist_cache_clear(&filelist->filelist_cache, size);
1616         }
1617 }
1618
1619 /* Helpers, low-level, they assume cursor + size <= cache_size */
1620 static bool filelist_file_cache_block_create(FileList *filelist, const int start_index, const int size, int cursor)
1621 {
1622         FileListEntryCache *cache = &filelist->filelist_cache;
1623
1624         {
1625                 int i, idx;
1626
1627                 for (i = 0, idx = start_index; i < size; i++, idx++, cursor++) {
1628                         FileDirEntry *entry;
1629
1630                         /* That entry might have already been requested and stored in misc cache... */
1631                         if ((entry = BLI_ghash_popkey(cache->misc_entries, SET_INT_IN_POINTER(idx), NULL)) == NULL) {
1632                                 entry = filelist_file_create_entry(filelist, idx);
1633                                 BLI_ghash_insert(cache->uuids, entry->uuid, entry);
1634                         }
1635                         cache->block_entries[cursor] = entry;
1636                 }
1637                 return true;
1638         }
1639
1640         return false;
1641 }
1642
1643 static void filelist_file_cache_block_release(struct FileList *filelist, const int size, int cursor)
1644 {
1645         FileListEntryCache *cache = &filelist->filelist_cache;
1646
1647         {
1648                 int i;
1649
1650                 for (i = 0; i < size; i++, cursor++) {
1651                         FileDirEntry *entry = cache->block_entries[cursor];
1652 //                      printf("%s: release cacheidx %d (%%p %%s)\n", __func__, cursor/*, cache->block_entries[cursor], cache->block_entries[cursor]->relpath*/);
1653                         BLI_ghash_remove(cache->uuids, entry->uuid, NULL, NULL);
1654                         filelist_file_release_entry(filelist, entry);
1655 #ifndef NDEBUG
1656                         cache->block_entries[cursor] = NULL;
1657 #endif
1658                 }
1659         }
1660 }
1661
1662 /* Load in cache all entries "around" given index (as much as block cache may hold). */
1663 bool filelist_file_cache_block(struct FileList *filelist, const int index)
1664 {
1665         FileListEntryCache *cache = &filelist->filelist_cache;
1666         const size_t cache_size = cache->size;
1667
1668         const int nbr_entries = filelist->filelist.nbr_entries_filtered;
1669         int start_index = max_ii(0, index - (cache_size / 2));
1670         int end_index = min_ii(nbr_entries, index + (cache_size / 2));
1671         int i;
1672         const bool full_refresh = (filelist->flags & FL_IS_READY) == 0;
1673
1674         if ((index < 0) || (index >= nbr_entries)) {
1675 //              printf("Wrong index %d ([%d:%d])", index, 0, nbr_entries);
1676                 return false;
1677         }
1678
1679         /* Maximize cached range! */
1680         if ((end_index - start_index) < cache_size) {
1681                 if (start_index == 0) {
1682                         end_index = min_ii(nbr_entries, start_index + cache_size);
1683                 }
1684                 else if (end_index == nbr_entries) {
1685                         start_index = max_ii(0, end_index - cache_size);
1686                 }
1687         }
1688
1689         BLI_assert((end_index - start_index) <= cache_size) ;
1690
1691 //      printf("%s: [%d:%d] around index %d (current cache: [%d:%d])\n", __func__,
1692 //             start_index, end_index, index, cache->block_start_index, cache->block_end_index);
1693
1694         /* If we have something to (re)cache... */
1695         if (full_refresh || (start_index != cache->block_start_index) || (end_index != cache->block_end_index)) {
1696                 if (full_refresh || (start_index >= cache->block_end_index) || (end_index <= cache->block_start_index)) {
1697                         int size1 = cache->block_end_index - cache->block_start_index;
1698                         int size2 = 0;
1699                         int idx1 = cache->block_cursor, idx2 = 0;
1700
1701 //                      printf("Full Recaching!\n");
1702
1703                         if (cache->flags & FLC_PREVIEWS_ACTIVE) {
1704                                 filelist_cache_previews_clear(cache);
1705                         }
1706
1707                         if (idx1 + size1 > cache_size) {
1708                                 size2 = idx1 + size1 - cache_size;
1709                                 size1 -= size2;
1710                                 filelist_file_cache_block_release(filelist, size2, idx2);
1711                         }
1712                         filelist_file_cache_block_release(filelist, size1, idx1);
1713
1714                         cache->block_start_index = cache->block_end_index = cache->block_cursor = 0;
1715
1716                         /* New cached block does not overlap existing one, simple. */
1717                         if (!filelist_file_cache_block_create(filelist, start_index, end_index - start_index, 0)) {
1718                                 return false;
1719                         }
1720
1721                         cache->block_start_index = start_index;
1722                         cache->block_end_index = end_index;
1723                 }
1724                 else {
1725 //                      printf("Partial Recaching!\n");
1726
1727                         /* At this point, we know we keep part of currently cached entries, so update previews if needed,
1728                          * and remove everything from working queue - we'll add all newly needed entries at the end. */
1729                         if (cache->flags & FLC_PREVIEWS_ACTIVE) {
1730                                 filelist_cache_previews_update(filelist);
1731                                 filelist_cache_previews_clear(cache);
1732                         }
1733
1734 //                      printf("\tpreview cleaned up...\n");
1735
1736                         if (start_index > cache->block_start_index) {
1737                                 int size1 = start_index - cache->block_start_index;
1738                                 int size2 = 0;
1739                                 int idx1 = cache->block_cursor, idx2 = 0;
1740
1741 //                              printf("\tcache releasing: [%d:%d] (%d, %d)\n", cache->block_start_index, cache->block_start_index + size1, cache->block_cursor, size1);
1742
1743                                 if (idx1 + size1 > cache_size) {
1744                                         size2 = idx1 + size1 - cache_size;
1745                                         size1 -= size2;
1746                                         filelist_file_cache_block_release(filelist, size2, idx2);
1747                                 }
1748                                 filelist_file_cache_block_release(filelist, size1, idx1);
1749
1750                                 cache->block_cursor = (idx1 + size1 + size2) % cache_size;
1751                                 cache->block_start_index = start_index;
1752                         }
1753                         if (end_index < cache->block_end_index) {
1754                                 int size1 = cache->block_end_index - end_index;
1755                                 int size2 = 0;
1756                                 int idx1, idx2 = 0;
1757
1758 //                              printf("\tcache releasing: [%d:%d] (%d)\n", cache->block_end_index - size1, cache->block_end_index, cache->block_cursor);
1759
1760                                 idx1 = (cache->block_cursor + end_index - cache->block_start_index) % cache_size;
1761                                 if (idx1 + size1 > cache_size) {
1762                                         size2 = idx1 + size1 - cache_size;
1763                                         size1 -= size2;
1764                                         filelist_file_cache_block_release(filelist, size2, idx2);
1765                                 }
1766                                 filelist_file_cache_block_release(filelist, size1, idx1);
1767
1768                                 cache->block_end_index = end_index;
1769                         }
1770
1771 //                      printf("\tcache cleaned up...\n");
1772
1773                         if (start_index < cache->block_start_index) {
1774                                 /* Add (request) needed entries before already cached ones. */
1775                                 /* Note: We need some index black magic to wrap around (cycle) inside our cache_size array... */
1776                                 int size1 = cache->block_start_index - start_index;
1777                                 int size2 = 0;
1778                                 int idx1, idx2;
1779
1780                                 if (size1 > cache->block_cursor) {
1781                                         size2 = size1;
1782                                         size1 -= cache->block_cursor;
1783                                         size2 -= size1;
1784                                         idx2 = 0;
1785                                         idx1 = cache_size - size1;
1786                                 }
1787                                 else {
1788                                         idx1 = cache->block_cursor - size1;
1789                                 }
1790
1791                                 if (size2) {
1792                                         if (!filelist_file_cache_block_create(filelist, start_index + size1, size2, idx2)) {
1793                                                 return false;
1794                                         }
1795                                 }
1796                                 if (!filelist_file_cache_block_create(filelist, start_index, size1, idx1)) {
1797                                         return false;
1798                                 }
1799
1800                                 cache->block_cursor = idx1;
1801                                 cache->block_start_index = start_index;
1802                         }
1803 //                      printf("\tstart-extended...\n");
1804                         if (end_index > cache->block_end_index) {
1805                                 /* Add (request) needed entries after already cached ones. */
1806                                 /* Note: We need some index black magic to wrap around (cycle) inside our cache_size array... */
1807                                 int size1 = end_index - cache->block_end_index;
1808                                 int size2 = 0;
1809                                 int idx1, idx2;
1810
1811                                 idx1 = (cache->block_cursor + end_index - cache->block_start_index - size1) % cache_size;
1812                                 if ((idx1 + size1) > cache_size) {
1813                                         size2 = size1;
1814                                         size1 = cache_size - idx1;
1815                                         size2 -= size1;
1816                                         idx2 = 0;
1817                                 }
1818
1819                                 if (size2) {
1820                                         if (!filelist_file_cache_block_create(filelist, end_index - size2, size2, idx2)) {
1821                                                 return false;
1822                                         }
1823                                 }
1824                                 if (!filelist_file_cache_block_create(filelist, end_index - size1 - size2, size1, idx1)) {
1825                                         return false;
1826                                 }
1827
1828                                 cache->block_end_index = end_index;
1829                         }
1830
1831 //                      printf("\tend-extended...\n");
1832                 }
1833         }
1834         else if ((cache->block_center_index != index) && (cache->flags & FLC_PREVIEWS_ACTIVE)) {
1835                 /* We try to always preview visible entries first, so 'restart' preview background task. */
1836                 filelist_cache_previews_update(filelist);
1837                 filelist_cache_previews_clear(cache);
1838         }
1839
1840 //      printf("Re-queueing previews...\n");
1841
1842         /* Note we try to preview first images around given index - i.e. assumed visible ones. */
1843         if (cache->flags & FLC_PREVIEWS_ACTIVE) {
1844                 for (i = 0; ((index + i) < end_index) || ((index - i) >= start_index); i++) {
1845                         if ((index - i) >= start_index) {
1846                                 const int idx = (cache->block_cursor + (index - start_index) - i) % cache_size;
1847                                 filelist_cache_previews_push(filelist, cache->block_entries[idx], index - i);
1848                         }
1849                         if ((index + i) < end_index) {
1850                                 const int idx = (cache->block_cursor + (index - start_index) + i) % cache_size;
1851                                 filelist_cache_previews_push(filelist, cache->block_entries[idx], index + i);
1852                         }
1853                 }
1854         }
1855
1856         cache->block_center_index = index;
1857
1858 //      printf("%s Finished!\n", __func__);
1859
1860         return true;
1861 }
1862
1863 void filelist_cache_previews_set(FileList *filelist, const bool use_previews)
1864 {
1865         FileListEntryCache *cache = &filelist->filelist_cache;
1866
1867         if (use_previews == ((cache->flags & FLC_PREVIEWS_ACTIVE) != 0)) {
1868                 return;
1869         }
1870         /* Do not start preview work while listing, gives nasty flickering! */
1871         else if (use_previews && (filelist->flags & FL_IS_READY)) {
1872                 cache->flags |= FLC_PREVIEWS_ACTIVE;
1873
1874                 BLI_assert((cache->previews_pool == NULL) && (cache->previews_done == NULL));
1875
1876 //              printf("%s: Init Previews...\n", __func__);
1877
1878                 /* No need to populate preview queue here, filelist_file_cache_block() handles this. */
1879         }
1880         else {
1881 //              printf("%s: Clear Previews...\n", __func__);
1882
1883                 filelist_cache_previews_free(cache);
1884         }
1885 }
1886
1887 bool filelist_cache_previews_update(FileList *filelist)
1888 {
1889         FileListEntryCache *cache = &filelist->filelist_cache;
1890         TaskPool *pool = cache->previews_pool;
1891         bool changed = false;
1892
1893         if (!pool) {
1894                 return changed;
1895         }
1896
1897 //      printf("%s: Update Previews...\n", __func__);
1898
1899         while (!BLI_thread_queue_is_empty(cache->previews_done)) {
1900                 FileListEntryPreview *preview = BLI_thread_queue_pop(cache->previews_done);
1901                 FileDirEntry *entry;
1902
1903                 /* Paranoid (should never happen currently since we consume this queue from a single thread), but... */
1904                 if (!preview) {
1905                         continue;
1906                 }
1907                 /* entry might have been removed from cache in the mean time, we do not want to cache it again here. */
1908                 entry = filelist_file_ex(filelist, preview->index, false);
1909
1910 //              printf("%s: %d - %s - %p\n", __func__, preview->index, preview->path, preview->img);
1911
1912                 if (preview->img) {
1913                         /* Due to asynchronous process, a preview for a given image may be generated several times, i.e.
1914                          * entry->image may already be set at this point. */
1915                         if (entry && !entry->image) {
1916                                 entry->image = preview->img;
1917                                 changed = true;
1918                         }
1919                         else {
1920                                 IMB_freeImBuf(preview->img);
1921                         }
1922                 }
1923                 else if (entry) {
1924                         /* We want to avoid re-processing this entry continuously!
1925                          * Note that, since entries only live in cache, preview will be retried quite often anyway. */
1926                         entry->flags |= FILE_ENTRY_INVALID_PREVIEW;
1927                 }
1928
1929                 MEM_freeN(preview);
1930         }
1931
1932         return changed;
1933 }
1934
1935 bool filelist_cache_previews_running(FileList *filelist)
1936 {
1937         FileListEntryCache *cache = &filelist->filelist_cache;
1938
1939         return (cache->previews_pool != NULL);
1940 }
1941
1942 /* would recognize .blend as well */
1943 static bool file_is_blend_backup(const char *str)
1944 {
1945         const size_t a = strlen(str);
1946         size_t b = 7;
1947         bool retval = 0;
1948
1949         if (a == 0 || b >= a) {
1950                 /* pass */
1951         }
1952         else {
1953                 const char *loc;
1954
1955                 if (a > b + 1)
1956                         b++;
1957
1958                 /* allow .blend1 .blend2 .blend32 */
1959                 loc = BLI_strcasestr(str + a - b, ".blend");
1960
1961                 if (loc)
1962                         retval = 1;
1963         }
1964
1965         return (retval);
1966 }
1967
1968 /* TODO: Maybe we should move this to BLI? On the other hand, it's using defines from spacefile area, so not sure... */
1969 int ED_path_extension_type(const char *path)
1970 {
1971         if (BLO_has_bfile_extension(path)) {
1972                 return FILE_TYPE_BLENDER;
1973         }
1974         else if (file_is_blend_backup(path)) {
1975                 return FILE_TYPE_BLENDER_BACKUP;
1976         }
1977         else if (BLI_path_extension_check(path, ".app")) {
1978                 return FILE_TYPE_APPLICATIONBUNDLE;
1979         }
1980         else if (BLI_path_extension_check(path, ".py")) {
1981                 return FILE_TYPE_PYSCRIPT;
1982         }
1983         else if (BLI_path_extension_check_n(path, ".txt", ".glsl", ".osl", ".data", ".pov", ".ini", ".mcr", ".inc", NULL)) {
1984                 return FILE_TYPE_TEXT;
1985         }
1986         else if (BLI_path_extension_check_n(path, ".ttf", ".ttc", ".pfb", ".otf", ".otc", NULL)) {
1987                 return FILE_TYPE_FTFONT;
1988         }
1989         else if (BLI_path_extension_check(path, ".btx")) {
1990                 return FILE_TYPE_BTX;
1991         }
1992         else if (BLI_path_extension_check(path, ".dae")) {
1993                 return FILE_TYPE_COLLADA;
1994         }
1995         else if (BLI_path_extension_check(path, ".abc")) {
1996                 return FILE_TYPE_ALEMBIC;
1997         }
1998         else if (BLI_path_extension_check_array(path, imb_ext_image)) {
1999                 return FILE_TYPE_IMAGE;
2000         }
2001         else if (BLI_path_extension_check(path, ".ogg")) {
2002                 if (IMB_isanim(path)) {
2003                         return FILE_TYPE_MOVIE;
2004                 }
2005                 else {
2006                         return FILE_TYPE_SOUND;
2007                 }
2008         }
2009         else if (BLI_path_extension_check_array(path, imb_ext_movie)) {
2010                 return FILE_TYPE_MOVIE;
2011         }
2012         else if (BLI_path_extension_check_array(path, imb_ext_audio)) {
2013                 return FILE_TYPE_SOUND;
2014         }
2015         return 0;
2016 }
2017
2018 static int file_extension_type(const char *dir, const char *relpath)
2019 {
2020         char path[FILE_MAX];
2021         BLI_join_dirfile(path, sizeof(path), dir, relpath);
2022         return ED_path_extension_type(path);
2023 }
2024
2025 int ED_file_extension_icon(const char *path)
2026 {
2027         const int type = ED_path_extension_type(path);
2028
2029         switch (type) {
2030                 case FILE_TYPE_BLENDER:
2031                         return ICON_FILE_BLEND;
2032                 case FILE_TYPE_BLENDER_BACKUP:
2033                         return ICON_FILE_BACKUP;
2034                 case FILE_TYPE_IMAGE:
2035                         return ICON_FILE_IMAGE;
2036                 case FILE_TYPE_MOVIE:
2037                         return ICON_FILE_MOVIE;
2038                 case FILE_TYPE_PYSCRIPT:
2039                         return ICON_FILE_SCRIPT;
2040                 case FILE_TYPE_SOUND:
2041                         return ICON_FILE_SOUND;
2042                 case FILE_TYPE_FTFONT:
2043                         return ICON_FILE_FONT;
2044                 case FILE_TYPE_BTX:
2045                         return ICON_FILE_BLANK;
2046                 case FILE_TYPE_COLLADA:
2047                         return ICON_FILE_BLANK;
2048                 case FILE_TYPE_ALEMBIC:
2049                         return ICON_FILE_BLANK;
2050                 case FILE_TYPE_TEXT:
2051                         return ICON_FILE_TEXT;
2052                 default:
2053                         return ICON_FILE_BLANK;
2054         }
2055 }
2056
2057 int filelist_empty(struct FileList *filelist)
2058 {
2059         return (filelist->filelist.nbr_entries == 0);
2060 }
2061
2062 unsigned int filelist_entry_select_set(
2063         const FileList *filelist, const FileDirEntry *entry, FileSelType select, unsigned int flag, FileCheckType check)
2064 {
2065         /* Default NULL pointer if not found is fine here! */
2066         void **es_p = BLI_ghash_lookup_p(filelist->selection_state, entry->uuid);
2067         unsigned int entry_flag = es_p ? GET_UINT_FROM_POINTER(*es_p) : 0;
2068         const unsigned int org_entry_flag = entry_flag;
2069
2070         BLI_assert(entry);
2071         BLI_assert(ELEM(check, CHECK_DIRS, CHECK_FILES, CHECK_ALL));
2072
2073         if (((check == CHECK_ALL)) ||
2074             ((check == CHECK_DIRS) && (entry->typeflag & FILE_TYPE_DIR)) ||
2075             ((check == CHECK_FILES) && !(entry->typeflag & FILE_TYPE_DIR)))
2076         {
2077                 switch (select) {
2078                         case FILE_SEL_REMOVE:
2079                                 entry_flag &= ~flag;
2080                                 break;
2081                         case FILE_SEL_ADD:
2082                                 entry_flag |= flag;
2083                                 break;
2084                         case FILE_SEL_TOGGLE:
2085                                 entry_flag ^= flag;
2086                                 break;
2087                 }
2088         }
2089
2090         if (entry_flag != org_entry_flag) {
2091                 if (es_p) {
2092                         if (entry_flag) {
2093                                 *es_p = SET_UINT_IN_POINTER(entry_flag);
2094                         }
2095                         else {
2096                                 BLI_ghash_remove(filelist->selection_state, entry->uuid, MEM_freeN, NULL);
2097                         }
2098                 }
2099                 else if (entry_flag) {
2100                         void *key = MEM_mallocN(sizeof(entry->uuid), __func__);
2101                         memcpy(key, entry->uuid, sizeof(entry->uuid));
2102                         BLI_ghash_insert(filelist->selection_state, key, SET_UINT_IN_POINTER(entry_flag));
2103                 }
2104         }
2105
2106         return entry_flag;
2107 }
2108
2109 void filelist_entry_select_index_set(FileList *filelist, const int index, FileSelType select, unsigned int flag, FileCheckType check)
2110 {
2111         FileDirEntry *entry = filelist_file(filelist, index);
2112
2113         if (entry) {
2114                 filelist_entry_select_set(filelist, entry, select, flag, check);
2115         }
2116 }
2117
2118 void filelist_entries_select_index_range_set(
2119         FileList *filelist, FileSelection *sel, FileSelType select, unsigned int flag, FileCheckType check)
2120 {
2121         /* select all valid files between first and last indicated */
2122         if ((sel->first >= 0) && (sel->first < filelist->filelist.nbr_entries_filtered) &&
2123             (sel->last >= 0) && (sel->last < filelist->filelist.nbr_entries_filtered))
2124         {
2125                 int current_file;
2126                 for (current_file = sel->first; current_file <= sel->last; current_file++) {
2127                         filelist_entry_select_index_set(filelist, current_file, select, flag, check);
2128                 }
2129         }
2130 }
2131
2132 unsigned int filelist_entry_select_get(FileList *filelist, FileDirEntry *entry, FileCheckType check)
2133 {
2134         BLI_assert(entry);
2135         BLI_assert(ELEM(check, CHECK_DIRS, CHECK_FILES, CHECK_ALL));
2136
2137         if (((check == CHECK_ALL)) ||
2138             ((check == CHECK_DIRS) && (entry->typeflag & FILE_TYPE_DIR)) ||
2139             ((check == CHECK_FILES) && !(entry->typeflag & FILE_TYPE_DIR)))
2140         {
2141                 /* Default NULL pointer if not found is fine here! */
2142                 return GET_UINT_FROM_POINTER(BLI_ghash_lookup(filelist->selection_state, entry->uuid));
2143         }
2144
2145         return 0;
2146 }
2147
2148 unsigned int filelist_entry_select_index_get(FileList *filelist, const int index, FileCheckType check)
2149 {
2150         FileDirEntry *entry = filelist_file(filelist, index);
2151
2152         if (entry) {
2153                 return filelist_entry_select_get(filelist, entry, check);
2154         }
2155
2156         return 0;
2157 }
2158
2159 /* WARNING! dir must be FILE_MAX_LIBEXTRA long! */
2160 bool filelist_islibrary(struct FileList *filelist, char *dir, char **group)
2161 {
2162         return BLO_library_path_explode(filelist->filelist.root, dir, group, NULL);
2163 }
2164
2165 static int groupname_to_code(const char *group)
2166 {
2167         char buf[BLO_GROUP_MAX];
2168         char *lslash;
2169
2170         BLI_assert(group);
2171
2172         BLI_strncpy(buf, group, sizeof(buf));
2173         lslash = (char *)BLI_last_slash(buf);
2174         if (lslash)
2175                 lslash[0] = '\0';
2176
2177         return buf[0] ? BKE_idcode_from_name(buf) : 0;
2178 }
2179
2180 static unsigned int groupname_to_filter_id(const char *group)
2181 {
2182         int id_code = groupname_to_code(group);
2183
2184         return BKE_idcode_to_idfilter(id_code);
2185 }
2186
2187 /**
2188  * From here, we are in 'Job Context', i.e. have to be careful about sharing stuff between background working thread
2189  * and main one (used by UI among other things).
2190  */
2191 typedef struct TodoDir {
2192         int level;
2193         char *dir;
2194 } TodoDir;
2195
2196 static int filelist_readjob_list_dir(
2197         const char *root, ListBase *entries, const char *filter_glob,
2198         const bool do_lib, const char *main_name, const bool skip_currpar)
2199 {
2200         struct direntry *files;
2201         int nbr_files, nbr_entries = 0;
2202
2203         nbr_files = BLI_filelist_dir_contents(root, &files);
2204         if (files) {
2205                 int i = nbr_files;
2206                 while (i--) {
2207                         FileListInternEntry *entry;
2208
2209                         if (skip_currpar && FILENAME_IS_CURRPAR(files[i].relname)) {
2210                                 continue;
2211                         }
2212
2213                         entry = MEM_callocN(sizeof(*entry), __func__);
2214                         entry->relpath = MEM_dupallocN(files[i].relname);
2215                         entry->st = files[i].s;
2216
2217                         /* Set file type. */
2218                         if (S_ISDIR(files[i].s.st_mode)) {
2219                                 entry->typeflag = FILE_TYPE_DIR;
2220                         }
2221                         else if (do_lib && BLO_has_bfile_extension(entry->relpath)) {
2222                                 /* If we are considering .blend files as libs, promote them to directory status. */
2223                                 char name[FILE_MAX];
2224
2225                                 entry->typeflag = FILE_TYPE_BLENDER;
2226
2227                                 BLI_join_dirfile(name, sizeof(name), root, entry->relpath);
2228
2229                                 /* prevent current file being used as acceptable dir */
2230                                 if (BLI_path_cmp(main_name, name) != 0) {
2231                                         entry->typeflag |= FILE_TYPE_DIR;
2232                                 }
2233                         }
2234                         /* Otherwise, do not check extensions for directories! */
2235                         else if (!(entry->typeflag & FILE_TYPE_DIR)) {
2236                                 entry->typeflag = file_extension_type(root, entry->relpath);
2237                                 if (filter_glob[0] && BLI_path_extension_check_glob(entry->relpath, filter_glob)) {
2238                                         entry->typeflag |= FILE_TYPE_OPERATOR;
2239                                 }
2240                         }
2241
2242                         BLI_addtail(entries, entry);
2243                         nbr_entries++;
2244                 }
2245                 BLI_filelist_free(files, nbr_files);
2246         }
2247         return nbr_entries;
2248 }
2249
2250 static int filelist_readjob_list_lib(const char *root, ListBase *entries, const bool skip_currpar)
2251 {
2252         FileListInternEntry *entry;
2253         LinkNode *ln, *names;
2254         int i, nnames, idcode = 0, nbr_entries = 0;
2255         char dir[FILE_MAX_LIBEXTRA], *group;
2256         bool ok;
2257
2258         struct BlendHandle *libfiledata = NULL;
2259
2260         /* name test */
2261         ok = BLO_library_path_explode(root, dir, &group, NULL);
2262         if (!ok) {
2263                 return nbr_entries;
2264         }
2265
2266         /* there we go */
2267         libfiledata = BLO_blendhandle_from_file(dir, NULL);
2268         if (libfiledata == NULL) {
2269                 return nbr_entries;
2270         }
2271
2272         /* memory for strings is passed into filelist[i].entry->relpath and freed in filelist_entry_free. */
2273         if (group) {
2274                 idcode = groupname_to_code(group);
2275                 names = BLO_blendhandle_get_datablock_names(libfiledata, idcode, &nnames);
2276         }
2277         else {
2278                 names = BLO_blendhandle_get_linkable_groups(libfiledata);
2279                 nnames = BLI_linklist_count(names);
2280         }
2281
2282         BLO_blendhandle_close(libfiledata);
2283
2284         if (!skip_currpar) {
2285                 entry = MEM_callocN(sizeof(*entry), __func__);
2286                 entry->relpath = BLI_strdup(FILENAME_PARENT);
2287                 entry->typeflag |= (FILE_TYPE_BLENDERLIB | FILE_TYPE_DIR);
2288                 BLI_addtail(entries, entry);
2289                 nbr_entries++;
2290         }
2291
2292         for (i = 0, ln = names; i < nnames; i++, ln = ln->next) {
2293                 const char *blockname = ln->link;
2294
2295                 entry = MEM_callocN(sizeof(*entry), __func__);
2296                 entry->relpath = BLI_strdup(blockname);
2297                 entry->typeflag |= FILE_TYPE_BLENDERLIB;
2298                 if (!(group && idcode)) {
2299                         entry->typeflag |= FILE_TYPE_DIR;
2300                         entry->blentype = groupname_to_code(blockname);
2301                 }
2302                 else {
2303                         entry->blentype = idcode;
2304                 }
2305                 BLI_addtail(entries, entry);
2306                 nbr_entries++;
2307         }
2308
2309         BLI_linklist_free(names, free);
2310
2311         return nbr_entries;
2312 }
2313
2314 #if 0
2315 /* Kept for reference here, in case we want to add back that feature later. We do not need it currently. */
2316 /* Code ***NOT*** updated for job stuff! */
2317 static void filelist_readjob_main_rec(Main *bmain, FileList *filelist)
2318 {
2319         ID *id;
2320         FileDirEntry *files, *firstlib = NULL;
2321         ListBase *lb;
2322         int a, fake, idcode, ok, totlib, totbl;
2323
2324         // filelist->type = FILE_MAIN; // XXX TODO: add modes to filebrowser
2325
2326         BLI_assert(filelist->filelist.entries == NULL);
2327
2328         if (filelist->filelist.root[0] == '/') filelist->filelist.root[0] = '\0';
2329
2330         if (filelist->filelist.root[0]) {
2331                 idcode = groupname_to_code(filelist->filelist.root);
2332                 if (idcode == 0) filelist->filelist.root[0] = '\0';
2333         }
2334
2335         if (filelist->dir[0] == 0) {
2336                 /* make directories */
2337 #ifdef WITH_FREESTYLE
2338                 filelist->filelist.nbr_entries = 24;
2339 #else
2340                 filelist->filelist.nbr_entries = 23;
2341 #endif
2342                 filelist_resize(filelist, filelist->filelist.nbr_entries);
2343
2344                 for (a = 0; a < filelist->filelist.nbr_entries; a++) {
2345                         filelist->filelist.entries[a].typeflag |= FILE_TYPE_DIR;
2346                 }
2347
2348                 filelist->filelist.entries[0].entry->relpath = BLI_strdup(FILENAME_PARENT);
2349                 filelist->filelist.entries[1].entry->relpath = BLI_strdup("Scene");
2350                 filelist->filelist.entries[2].entry->relpath = BLI_strdup("Object");
2351                 filelist->filelist.entries[3].entry->relpath = BLI_strdup("Mesh");
2352                 filelist->filelist.entries[4].entry->relpath = BLI_strdup("Curve");
2353                 filelist->filelist.entries[5].entry->relpath = BLI_strdup("Metaball");
2354                 filelist->filelist.entries[6].entry->relpath = BLI_strdup("Material");
2355                 filelist->filelist.entries[7].entry->relpath = BLI_strdup("Texture");
2356                 filelist->filelist.entries[8].entry->relpath = BLI_strdup("Image");
2357                 filelist->filelist.entries[9].entry->relpath = BLI_strdup("Ika");
2358                 filelist->filelist.entries[10].entry->relpath = BLI_strdup("Wave");
2359                 filelist->filelist.entries[11].entry->relpath = BLI_strdup("Lattice");
2360                 filelist->filelist.entries[12].entry->relpath = BLI_strdup("Light");
2361                 filelist->filelist.entries[13].entry->relpath = BLI_strdup("Camera");
2362                 filelist->filelist.entries[14].entry->relpath = BLI_strdup("Ipo");
2363                 filelist->filelist.entries[15].entry->relpath = BLI_strdup("World");
2364                 filelist->filelist.entries[16].entry->relpath = BLI_strdup("Screen");
2365                 filelist->filelist.entries[17].entry->relpath = BLI_strdup("VFont");
2366                 filelist->filelist.entries[18].entry->relpath = BLI_strdup("Text");
2367                 filelist->filelist.entries[19].entry->relpath = BLI_strdup("Armature");
2368                 filelist->filelist.entries[20].entry->relpath = BLI_strdup("Action");
2369                 filelist->filelist.entries[21].entry->relpath = BLI_strdup("NodeTree");
2370                 filelist->filelist.entries[22].entry->relpath = BLI_strdup("Speaker");
2371 #ifdef WITH_FREESTYLE
2372                 filelist->filelist.entries[23].entry->relpath = BLI_strdup("FreestyleLineStyle");
2373 #endif
2374         }
2375         else {
2376                 /* make files */
2377                 idcode = groupname_to_code(filelist->filelist.root);
2378
2379                 lb = which_libbase(bmain, idcode);
2380                 if (lb == NULL) return;
2381
2382                 filelist->filelist.nbr_entries = 0;
2383                 for (id = lb->first; id; id = id->next) {
2384                         if (!(filelist->filter_data.flags & FLF_HIDE_DOT) || id->name[2] != '.') {
2385                                 filelist->filelist.nbr_entries++;
2386                         }
2387                 }
2388
2389                 /* XXX TODO: if databrowse F4 or append/link filelist->flags & FLF_HIDE_PARENT has to be set */
2390                 if (!(filelist->filter_data.flags & FLF_HIDE_PARENT))
2391                         filelist->filelist.nbr_entries++;
2392
2393                 if (filelist->filelist.nbr_entries > 0) {
2394                         filelist_resize(filelist, filelist->filelist.nbr_entries);
2395                 }
2396
2397                 files = filelist->filelist.entries;
2398
2399                 if (!(filelist->filter_data.flags & FLF_HIDE_PARENT)) {
2400                         files->entry->relpath = BLI_strdup(FILENAME_PARENT);
2401                         files->typeflag |= FILE_TYPE_DIR;
2402
2403                         files++;
2404                 }
2405
2406                 totlib = totbl = 0;
2407                 for (id = lb->first; id; id = id->next) {
2408                         ok = 1;
2409                         if (ok) {
2410                                 if (!(filelist->filter_data.flags & FLF_HIDE_DOT) || id->name[2] != '.') {
2411                                         if (id->lib == NULL) {
2412                                                 files->entry->relpath = BLI_strdup(id->name + 2);
2413                                         }
2414                                         else {
2415                                                 char relname[FILE_MAX + (MAX_ID_NAME - 2) + 3];
2416                                                 BLI_snprintf(relname, sizeof(relname), "%s | %s", id->lib->name, id->name + 2);
2417                                                 files->entry->relpath = BLI_strdup(relname);
2418                                         }
2419 //                                      files->type |= S_IFREG;
2420 #if 0               /* XXX TODO show the selection status of the objects */
2421                                         if (!filelist->has_func) { /* F4 DATA BROWSE */
2422                                                 if (idcode == ID_OB) {
2423                                                         if ( ((Object *)id)->flag & SELECT) files->entry->selflag |= FILE_SEL_SELECTED;
2424                                                 }
2425                                                 else if (idcode == ID_SCE) {
2426                                                         if ( ((Scene *)id)->r.scemode & R_BG_RENDER) files->entry->selflag |= FILE_SEL_SELECTED;
2427                                                 }
2428                                         }
2429 #endif
2430 //                                      files->entry->nr = totbl + 1;
2431                                         files->entry->poin = id;
2432                                         fake = id->flag & LIB_FAKEUSER;
2433                                         if (idcode == ID_MA || idcode == ID_TE || idcode == ID_LA || idcode == ID_WO || idcode == ID_IM) {
2434                                                 files->typeflag |= FILE_TYPE_IMAGE;
2435                                         }
2436 //                                      if      (id->lib && fake) BLI_snprintf(files->extra, sizeof(files->entry->extra), "LF %d",    id->us);
2437 //                                      else if (id->lib)         BLI_snprintf(files->extra, sizeof(files->entry->extra), "L    %d",  id->us);
2438 //                                      else if (fake)            BLI_snprintf(files->extra, sizeof(files->entry->extra), "F    %d",  id->us);
2439 //                                      else                      BLI_snprintf(files->extra, sizeof(files->entry->extra), "      %d", id->us);
2440
2441                                         if (id->lib) {
2442                                                 if (totlib == 0) firstlib = files;
2443                                                 totlib++;
2444                                         }
2445
2446                                         files++;
2447                                 }
2448                                 totbl++;
2449                         }
2450                 }
2451
2452                 /* only qsort of library blocks */
2453                 if (totlib > 1) {
2454                         qsort(firstlib, totlib, sizeof(*files), compare_name);
2455                 }
2456         }
2457 }
2458 #endif
2459
2460 static void filelist_readjob_do(
2461         const bool do_lib,
2462         FileList *filelist, const char *main_name, short *stop, short *do_update, float *progress, ThreadMutex *lock)
2463 {
2464         ListBase entries = {0};
2465         BLI_Stack *todo_dirs;
2466         TodoDir *td_dir;
2467         char dir[FILE_MAX_LIBEXTRA];
2468         char filter_glob[FILE_MAXFILE];
2469         const char *root = filelist->filelist.root;
2470         const int max_recursion = filelist->max_recursion;
2471         int nbr_done_dirs = 0, nbr_todo_dirs = 1;
2472
2473 //      BLI_assert(filelist->filtered == NULL);
2474         BLI_assert(BLI_listbase_is_empty(&filelist->filelist.entries) && (filelist->filelist.nbr_entries == 0));
2475
2476         todo_dirs = BLI_stack_new(sizeof(*td_dir), __func__);
2477         td_dir = BLI_stack_push_r(todo_dirs);
2478         td_dir->level = 1;
2479
2480         BLI_strncpy(dir, filelist->filelist.root, sizeof(dir));
2481         BLI_strncpy(filter_glob, filelist->filter_data.filter_glob, sizeof(filter_glob));
2482
2483         BLI_cleanup_dir(main_name, dir);
2484         td_dir->dir = BLI_strdup(dir);
2485
2486         while (!BLI_stack_is_empty(todo_dirs) && !(*stop)) {
2487                 FileListInternEntry *entry;
2488                 int nbr_entries = 0;
2489                 bool is_lib = do_lib;
2490
2491                 char *subdir;
2492                 char rel_subdir[FILE_MAX_LIBEXTRA];
2493                 int recursion_level;
2494                 bool skip_currpar;
2495
2496                 td_dir = BLI_stack_peek(todo_dirs);
2497                 subdir = td_dir->dir;
2498                 recursion_level = td_dir->level;
2499                 skip_currpar = (recursion_level > 1);
2500
2501                 BLI_stack_discard(todo_dirs);
2502
2503                 /* ARRRG! We have to be very careful *not to use* common BLI_path_util helpers over entry->relpath itself
2504                  * (nor any path containing it), since it may actually be a datablock name inside .blend file,
2505                  * which can have slashes and backslashes! See T46827.
2506                  * Note that in the end, this means we 'cache' valid relative subdir once here, this is actually better. */
2507                 BLI_strncpy(rel_subdir, subdir, sizeof(rel_subdir));
2508                 BLI_cleanup_dir(root, rel_subdir);
2509                 BLI_path_rel(rel_subdir, root);
2510
2511                 if (do_lib) {
2512                         nbr_entries = filelist_readjob_list_lib(subdir, &entries, skip_currpar);
2513                 }
2514                 if (!nbr_entries) {
2515                         is_lib = false;
2516                         nbr_entries = filelist_readjob_list_dir(subdir, &entries, filter_glob, do_lib, main_name, skip_currpar);
2517                 }
2518
2519                 for (entry = entries.first; entry; entry = entry->next) {
2520                         BLI_join_dirfile(dir, sizeof(dir), rel_subdir, entry->relpath);
2521
2522                         /* Generate our entry uuid. Abusing uuid as an uint32, shall be more than enough here,
2523                          * things would crash way before we overflow that counter!
2524                          * Using an atomic operation to avoid having to lock thread...
2525                          * Note that we do not really need this here currently, since there is a single listing thread, but better
2526                          * remain consistent about threading! */
2527                         *((uint32_t *)entry->uuid) = atomic_add_and_fetch_uint32((uint32_t *)filelist->filelist_intern.curr_uuid, 1);
2528
2529                         /* Only thing we change in direntry here, so we need to free it first. */
2530                         MEM_freeN(entry->relpath);
2531                         entry->relpath = BLI_strdup(dir + 2);  /* + 2 to remove '//' added by BLI_path_rel to rel_subdir */
2532                         entry->name = BLI_strdup(fileentry_uiname(root, entry->relpath, entry->typeflag, dir));
2533
2534                         /* Here we decide whether current filedirentry is to be listed too, or not. */
2535                         if (max_recursion && (is_lib || (recursion_level <= max_recursion))) {
2536                                 if (((entry->typeflag & FILE_TYPE_DIR) == 0) || FILENAME_IS_CURRPAR(entry->relpath)) {
2537                                         /* Skip... */
2538                                 }
2539                                 else if (!is_lib && (recursion_level >= max_recursion) &&
2540                                          ((entry->typeflag & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) == 0))
2541                                 {
2542                                         /* Do not recurse in real directories in this case, only in .blend libs. */
2543                                 }
2544                                 else {
2545                                         /* We have a directory we want to list, add it to todo list! */
2546                                         BLI_join_dirfile(dir, sizeof(dir), root, entry->relpath);
2547                                         BLI_cleanup_dir(main_name, dir);
2548                                         td_dir = BLI_stack_push_r(todo_dirs);
2549                                         td_dir->level = recursion_level + 1;
2550                                         td_dir->dir = BLI_strdup(dir);
2551                                         nbr_todo_dirs++;
2552                                 }
2553                         }
2554                 }
2555
2556                 if (nbr_entries) {
2557                         BLI_mutex_lock(lock);
2558
2559                         *do_update = true;
2560
2561                         BLI_movelisttolist(&filelist->filelist.entries, &entries);
2562                         filelist->filelist.nbr_entries += nbr_entries;
2563
2564                         BLI_mutex_unlock(lock);
2565                 }
2566
2567                 nbr_done_dirs++;
2568                 *progress = (float)nbr_done_dirs / (float)nbr_todo_dirs;
2569                 MEM_freeN(subdir);
2570         }
2571
2572         /* If we were interrupted by stop, stack may not be empty and we need to free pending dir paths. */
2573         while (!BLI_stack_is_empty(todo_dirs)) {
2574                 td_dir = BLI_stack_peek(todo_dirs);
2575                 MEM_freeN(td_dir->dir);
2576                 BLI_stack_discard(todo_dirs);
2577         }
2578         BLI_stack_free(todo_dirs);
2579 }
2580
2581 static void filelist_readjob_dir(
2582         FileList *filelist, const char *main_name, short *stop, short *do_update, float *progress, ThreadMutex *lock)
2583 {
2584         filelist_readjob_do(false, filelist, main_name, stop, do_update, progress, lock);
2585 }
2586
2587 static void filelist_readjob_lib(
2588         FileList *filelist, const char *main_name, short *stop, short *do_update, float *progress, ThreadMutex *lock)
2589 {
2590         filelist_readjob_do(true, filelist, main_name, stop, do_update, progress, lock);
2591 }
2592
2593 static void filelist_readjob_main(
2594         FileList *filelist, const char *main_name, short *stop, short *do_update, float *progress, ThreadMutex *lock)
2595 {
2596         /* TODO! */
2597         filelist_readjob_dir(filelist, main_name, stop, do_update, progress, lock);
2598 }
2599
2600
2601 typedef struct FileListReadJob {
2602         ThreadMutex lock;
2603         char main_name[FILE_MAX];
2604         struct FileList *filelist;
2605         struct FileList *tmp_filelist;  /* XXX We may use a simpler struct here... just a linked list and root path? */
2606 } FileListReadJob;
2607
2608 static void filelist_readjob_startjob(void *flrjv, short *stop, short *do_update, float *progress)
2609 {
2610         FileListReadJob *flrj = flrjv;
2611
2612 //      printf("START filelist reading (%d files, main thread: %d)\n",
2613 //             flrj->filelist->filelist.nbr_entries, BLI_thread_is_main());
2614
2615         BLI_mutex_lock(&flrj->lock);
2616
2617         BLI_assert((flrj->tmp_filelist == NULL) && flrj->filelist);
2618
2619         flrj->tmp_filelist = MEM_dupallocN(flrj->filelist);
2620
2621         BLI_listbase_clear(&flrj->tmp_filelist->filelist.entries);
2622         flrj->tmp_filelist->filelist.nbr_entries = 0;
2623
2624         flrj->tmp_filelist->filelist_intern.filtered = NULL;
2625         BLI_listbase_clear(&flrj->tmp_filelist->filelist_intern.entries);
2626         memset(flrj->tmp_filelist->filelist_intern.curr_uuid, 0, sizeof(flrj->tmp_filelist->filelist_intern.curr_uuid));
2627
2628         flrj->tmp_filelist->libfiledata = NULL;
2629         memset(&flrj->tmp_filelist->filelist_cache, 0, sizeof(flrj->tmp_filelist->filelist_cache));
2630         flrj->tmp_filelist->selection_state = NULL;
2631
2632         BLI_mutex_unlock(&flrj->lock);
2633
2634         flrj->tmp_filelist->read_jobf(flrj->tmp_filelist, flrj->main_name, stop, do_update, progress, &flrj->lock);
2635 }
2636
2637 static void filelist_readjob_update(void *flrjv)
2638 {
2639         FileListReadJob *flrj = flrjv;
2640         FileListIntern *fl_intern = &flrj->filelist->filelist_intern;
2641         ListBase new_entries = {NULL};
2642         int nbr_entries, new_nbr_entries = 0;
2643
2644         BLI_movelisttolist(&new_entries, &fl_intern->entries);
2645         nbr_entries = flrj->filelist->filelist.nbr_entries;
2646
2647         BLI_mutex_lock(&flrj->lock);
2648
2649         if (flrj->tmp_filelist->filelist.nbr_entries) {
2650                 /* We just move everything out of 'thread context' into final list. */
2651                 new_nbr_entries = flrj->tmp_filelist->filelist.nbr_entries;
2652                 BLI_movelisttolist(&new_entries, &flrj->tmp_filelist->filelist.entries);
2653                 flrj->tmp_filelist->filelist.nbr_entries = 0;
2654         }
2655
2656         BLI_mutex_unlock(&flrj->lock);
2657
2658         if (new_nbr_entries) {
2659                 /* Do not clear selection cache, we can assume already 'selected' uuids are still valid! */
2660                 filelist_clear_ex(flrj->filelist, true, false);
2661
2662                 flrj->filelist->flags |= (FL_NEED_SORTING | FL_NEED_FILTERING);
2663         }
2664
2665         /* if no new_nbr_entries, this is NOP */
2666         BLI_movelisttolist(&fl_intern->entries, &new_entries);
2667         flrj->filelist->filelist.nbr_entries = nbr_entries + new_nbr_entries;
2668 }
2669
2670 static void filelist_readjob_endjob(void *flrjv)
2671 {
2672         FileListReadJob *flrj = flrjv;
2673
2674         /* In case there would be some dangling update... */
2675         filelist_readjob_update(flrjv);
2676
2677         flrj->filelist->flags &= ~FL_IS_PENDING;
2678         flrj->filelist->flags |= FL_IS_READY;
2679 }
2680
2681 static void filelist_readjob_free(void *flrjv)
2682 {
2683         FileListReadJob *flrj = flrjv;
2684
2685 //      printf("END filelist reading (%d files)\n", flrj->filelist->filelist.nbr_entries);
2686
2687         if (flrj->tmp_filelist) {
2688                 /* tmp_filelist shall never ever be filtered! */
2689                 BLI_assert(flrj->tmp_filelist->filelist.nbr_entries == 0);
2690                 BLI_assert(BLI_listbase_is_empty(&flrj->tmp_filelist->filelist.entries));
2691
2692                 filelist_freelib(flrj->tmp_filelist);
2693                 filelist_free(flrj->tmp_filelist);
2694                 MEM_freeN(flrj->tmp_filelist);
2695         }
2696
2697         BLI_mutex_end(&flrj->lock);
2698
2699         MEM_freeN(flrj);
2700 }
2701
2702 void filelist_readjob_start(FileList *filelist, const bContext *C)
2703 {
2704         Main *bmain = CTX_data_main(C);
2705         wmJob *wm_job;
2706         FileListReadJob *flrj;
2707
2708         /* prepare job data */
2709         flrj = MEM_callocN(sizeof(*flrj), __func__);
2710         flrj->filelist = filelist;
2711         BLI_strncpy(flrj->main_name, BKE_main_blendfile_path(bmain), sizeof(flrj->main_name));
2712
2713         filelist->flags &= ~(FL_FORCE_RESET | FL_IS_READY);
2714         filelist->flags |= FL_IS_PENDING;
2715
2716         BLI_mutex_init(&flrj->lock);
2717
2718         /* setup job */
2719         wm_job = WM_jobs_get(CTX_wm_manager(C), CTX_wm_window(C), CTX_wm_area(C), "Listing Dirs...",
2720                              WM_JOB_PROGRESS, WM_JOB_TYPE_FILESEL_READDIR);
2721         WM_jobs_customdata_set(wm_job, flrj, filelist_readjob_free);
2722         WM_jobs_timer(wm_job, 0.01, NC_SPACE | ND_SPACE_FILE_LIST, NC_SPACE | ND_SPACE_FILE_LIST);
2723         WM_jobs_callbacks(wm_job, filelist_readjob_startjob, NULL, filelist_readjob_update, filelist_readjob_endjob);
2724
2725         /* start the job */
2726         WM_jobs_start(CTX_wm_manager(C), wm_job);
2727 }
2728
2729 void filelist_readjob_stop(wmWindowManager *wm, ScrArea *sa)
2730 {
2731         WM_jobs_kill_type(wm, sa, WM_JOB_TYPE_FILESEL_READDIR);
2732 }
2733
2734 int filelist_readjob_running(wmWindowManager *wm, ScrArea *sa)
2735 {
2736         return WM_jobs_test(wm, sa, WM_JOB_TYPE_FILESEL_READDIR);
2737 }