Cleanup/refactor spacefile's 'check dir' code.
[blender.git] / source / blender / editors / space_file / filelist.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version. 
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2007 Blender Foundation.
19  * All rights reserved.
20  *
21  * The Original Code is: all of this file.
22  *
23  * Contributor(s): none yet.
24  *
25  * ***** END GPL LICENSE BLOCK *****
26  */
27
28 /** \file blender/editors/space_file/filelist.c
29  *  \ingroup spfile
30  */
31
32
33 /* global includes */
34
35 #include <stdlib.h>
36 #include <math.h>
37 #include <string.h>
38 #include <sys/stat.h>
39 #include <time.h>
40
41 #ifndef WIN32
42 #  include <unistd.h>
43 #else
44 #  include <io.h>
45 #  include <direct.h>
46 #endif   
47 #include "MEM_guardedalloc.h"
48
49 #include "BLI_blenlib.h"
50 #include "BLI_fileops.h"
51 #include "BLI_fileops_types.h"
52 #include "BLI_fnmatch.h"
53 #include "BLI_ghash.h"
54 #include "BLI_hash_md5.h"
55 #include "BLI_linklist.h"
56 #include "BLI_math.h"
57 #include "BLI_stack.h"
58 #include "BLI_task.h"
59 #include "BLI_threads.h"
60 #include "BLI_utildefines.h"
61
62 #ifdef WIN32
63 #  include "BLI_winstuff.h"
64 #endif
65
66 #include "BKE_context.h"
67 #include "BKE_global.h"
68 #include "BKE_library.h"
69 #include "BKE_icons.h"
70 #include "BKE_idcode.h"
71 #include "BKE_main.h"
72 #include "BLO_readfile.h"
73
74 #include "DNA_space_types.h"
75
76 #include "ED_datafiles.h"
77 #include "ED_fileselect.h"
78 #include "ED_screen.h"
79
80 #include "IMB_imbuf.h"
81 #include "IMB_imbuf_types.h"
82 #include "IMB_thumbs.h"
83
84 #include "PIL_time.h"
85
86 #include "WM_api.h"
87 #include "WM_types.h"
88
89 #include "UI_resources.h"
90 #include "UI_interface_icons.h"
91
92 #include "atomic_ops.h"
93
94 #include "filelist.h"
95
96
97 /* ----------------- FOLDERLIST (previous/next) -------------- */
98
99 typedef struct FolderList {
100         struct FolderList *next, *prev;
101         char *foldername;
102 } FolderList;
103
104 ListBase *folderlist_new(void)
105 {
106         ListBase *p = MEM_callocN(sizeof(*p), __func__);
107         return p;
108 }
109
110 void folderlist_popdir(struct ListBase *folderlist, char *dir)
111 {
112         const char *prev_dir;
113         struct FolderList *folder;
114         folder = folderlist->last;
115
116         if (folder) {
117                 /* remove the current directory */
118                 MEM_freeN(folder->foldername);
119                 BLI_freelinkN(folderlist, folder);
120
121                 folder = folderlist->last;
122                 if (folder) {
123                         prev_dir = folder->foldername;
124                         BLI_strncpy(dir, prev_dir, FILE_MAXDIR);
125                 }
126         }
127         /* delete the folder next or use setdir directly before PREVIOUS OP */
128 }
129
130 void folderlist_pushdir(ListBase *folderlist, const char *dir)
131 {
132         struct FolderList *folder, *previous_folder;
133         previous_folder = folderlist->last;
134
135         /* check if already exists */
136         if (previous_folder && previous_folder->foldername) {
137                 if (BLI_path_cmp(previous_folder->foldername, dir) == 0) {
138                         return;
139                 }
140         }
141
142         /* create next folder element */
143         folder = MEM_mallocN(sizeof(*folder), __func__);
144         folder->foldername = BLI_strdup(dir);
145
146         /* add it to the end of the list */
147         BLI_addtail(folderlist, folder);
148 }
149
150 const char *folderlist_peeklastdir(ListBase *folderlist)
151 {
152         struct FolderList *folder;
153
154         if (!folderlist->last)
155                 return NULL;
156
157         folder = folderlist->last;
158         return folder->foldername;
159 }
160
161 int folderlist_clear_next(struct SpaceFile *sfile)
162 {
163         struct FolderList *folder;
164
165         /* if there is no folder_next there is nothing we can clear */
166         if (!sfile->folders_next)
167                 return 0;
168
169         /* if previous_folder, next_folder or refresh_folder operators are executed it doesn't clear folder_next */
170         folder = sfile->folders_prev->last;
171         if ((!folder) || (BLI_path_cmp(folder->foldername, sfile->params->dir) == 0))
172                 return 0;
173
174         /* eventually clear flist->folders_next */
175         return 1;
176 }
177
178 /* not listbase itself */
179 void folderlist_free(ListBase *folderlist)
180 {
181         if (folderlist) {
182                 FolderList *folder;
183                 for (folder = folderlist->first; folder; folder = folder->next)
184                         MEM_freeN(folder->foldername);
185                 BLI_freelistN(folderlist);
186         }
187 }
188
189 ListBase *folderlist_duplicate(ListBase *folderlist)
190 {
191         
192         if (folderlist) {
193                 ListBase *folderlistn = MEM_callocN(sizeof(*folderlistn), __func__);
194                 FolderList *folder;
195                 
196                 BLI_duplicatelist(folderlistn, folderlist);
197                 
198                 for (folder = folderlistn->first; folder; folder = folder->next) {
199                         folder->foldername = MEM_dupallocN(folder->foldername);
200                 }
201                 return folderlistn;
202         }
203         return NULL;
204 }
205
206
207 /* ------------------FILELIST------------------------ */
208
209 typedef struct FileListInternEntry {
210         struct FileListInternEntry *next, *prev;
211
212         char uuid[16];  /* ASSET_UUID_LENGTH */
213
214         int typeflag;  /* eFileSel_File_Types */
215         int blentype;  /* ID type, in case typeflag has FILE_TYPE_BLENDERLIB set. */
216
217         char *relpath;
218         char *name;  /* not striclty needed, but used during sorting, avoids to have to recompute it there... */
219
220         BLI_stat_t st;
221 } FileListInternEntry;
222
223 typedef struct FileListIntern {
224         ListBase entries;  /* FileListInternEntry items. */
225         FileListInternEntry **filtered;
226
227         char curr_uuid[16];  /* Used to generate uuid during internal listing. */
228 } FileListIntern;
229
230 #define FILELIST_ENTRYCACHESIZE_DEFAULT 1024  /* Keep it a power of two! */
231 typedef struct FileListEntryCache {
232         size_t size;  /* The size of the cache... */
233
234         int flags;
235
236         /* This one gathers all entries from both block and misc caches. Used for easy bulk-freing. */
237         ListBase cached_entries;
238
239         /* Block cache: all entries between start and end index. used for part of the list on diplay. */
240         FileDirEntry **block_entries;
241         int block_start_index, block_end_index, block_center_index, block_cursor;
242
243         /* Misc cache: random indices, FIFO behavior.
244          * Note: Not 100% sure we actually need that, time will say. */
245         int misc_cursor;
246         int *misc_entries_indices;
247         GHash *misc_entries;
248
249         /* Allows to quickly get a cached entry from its UUID. */
250         GHash *uuids;
251
252         /* Previews handling. */
253         TaskPool *previews_pool;
254         ThreadQueue *previews_done;
255 } FileListEntryCache;
256
257 /* FileListCache.flags */
258 enum {
259         FLC_IS_INIT              = 1 << 0,
260         FLC_PREVIEWS_ACTIVE      = 1 << 1,
261 };
262
263 typedef struct FileListEntryPreview {
264         char path[FILE_MAX];
265         unsigned int flags;
266         int index;
267         ImBuf *img;
268 } FileListEntryPreview;
269
270 typedef struct FileListFilter {
271         unsigned int filter;
272         unsigned int filter_id;
273         char filter_glob[256];
274         char filter_search[66];  /* + 2 for heading/trailing implicit '*' wildcards. */
275         short flags;
276 } FileListFilter;
277
278 /* FileListFilter.flags */
279 enum {
280         FLF_HIDE_DOT     = 1 << 0,
281         FLF_HIDE_PARENT  = 1 << 1,
282         FLF_HIDE_LIB_DIR = 1 << 2,
283 };
284
285 typedef struct FileList {
286         FileDirEntryArr filelist;
287
288         short prv_w;
289         short prv_h;
290
291         short flags;
292
293         short sort;
294
295         FileListFilter filter_data;
296
297         struct FileListIntern filelist_intern;
298
299         struct FileListEntryCache filelist_cache;
300
301         /* We need to keep those info outside of actual filelist items, because those are no more persistent
302          * (only generated on demand, and freed as soon as possible).
303          * Persistent part (mere list of paths + stat info) is kept as small as possible, and filebrowser-agnostic.
304          */
305         GHash *selection_state;
306
307         short max_recursion;
308         short recursion_level;
309
310         struct BlendHandle *libfiledata;
311
312         /* Set given path as root directory, if last bool is true may change given string in place to a valid value.
313          * Returns True if valid dir. */
314         bool (*checkdirf)(struct FileList *, char *, const bool);
315
316         /* Fill filelist (to be called by read job). */
317         void (*read_jobf)(struct FileList *, const char *, short *, short *, float *, ThreadMutex *);
318
319         /* Filter an entry of current filelist. */
320         bool (*filterf)(struct FileListInternEntry *, const char *, FileListFilter *);
321 } FileList;
322
323 /* FileList.flags */
324 enum {
325         FL_FORCE_RESET    = 1 << 0,
326         FL_IS_READY       = 1 << 1,
327         FL_IS_PENDING     = 1 << 2,
328         FL_NEED_SORTING   = 1 << 3,
329         FL_NEED_FILTERING = 1 << 4,
330 };
331
332 #define SPECIAL_IMG_SIZE 48
333 #define SPECIAL_IMG_ROWS 4
334 #define SPECIAL_IMG_COLS 4
335
336 enum {
337         SPECIAL_IMG_FOLDER      = 0,
338         SPECIAL_IMG_PARENT      = 1,
339         SPECIAL_IMG_REFRESH     = 2,
340         SPECIAL_IMG_BLENDFILE   = 3,
341         SPECIAL_IMG_SOUNDFILE   = 4,
342         SPECIAL_IMG_MOVIEFILE   = 5,
343         SPECIAL_IMG_PYTHONFILE  = 6,
344         SPECIAL_IMG_TEXTFILE    = 7,
345         SPECIAL_IMG_FONTFILE    = 8,
346         SPECIAL_IMG_UNKNOWNFILE = 9,
347         SPECIAL_IMG_LOADING     = 10,
348         SPECIAL_IMG_BACKUP      = 11,
349         SPECIAL_IMG_MAX
350 };
351
352 static ImBuf *gSpecialFileImages[SPECIAL_IMG_MAX];
353
354
355 static void filelist_readjob_main(struct FileList *, const char *, short *, short *, float *, ThreadMutex *);
356 static void filelist_readjob_lib(struct FileList *, const char *, short *, short *, float *, ThreadMutex *);
357 static void filelist_readjob_dir(struct FileList *, const char *, short *, short *, float *, ThreadMutex *);
358
359 /* helper, could probably go in BKE actually? */
360 static int groupname_to_code(const char *group);
361 static unsigned int groupname_to_filter_id(const char *group);
362
363 static void filelist_filter_clear(FileList *filelist);
364 static void filelist_cache_clear(FileListEntryCache *cache, size_t new_size);
365
366 /* ********** Sort helpers ********** */
367
368 static int compare_direntry_generic(const FileListInternEntry *entry1, const FileListInternEntry *entry2)
369 {
370         /* type is equal to stat.st_mode */
371
372         if (entry1->typeflag & FILE_TYPE_DIR) {
373                 if (entry2->typeflag & FILE_TYPE_DIR) {
374                         /* If both entries are tagged as dirs, we make a 'sub filter' that shows first the real dirs,
375                          * then libs (.blend files), then categories in libs. */
376                         if (entry1->typeflag & FILE_TYPE_BLENDERLIB) {
377                                 if (!(entry2->typeflag & FILE_TYPE_BLENDERLIB)) {
378                                         return 1;
379                                 }
380                         }
381                         else if (entry2->typeflag & FILE_TYPE_BLENDERLIB) {
382                                 return -1;
383                         }
384                         else if (entry1->typeflag & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) {
385                                 if (!(entry2->typeflag & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP))) {
386                                         return 1;
387                                 }
388                         }
389                         else if (entry2->typeflag & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) {
390                                 return -1;
391                         }
392                 }
393                 else {
394                         return -1;
395                 }
396         }
397         else if (entry2->typeflag & FILE_TYPE_DIR) {
398                 return 1;
399         }
400
401         /* make sure "." and ".." are always first */
402         if (FILENAME_IS_CURRENT(entry1->relpath)) return -1;
403         if (FILENAME_IS_CURRENT(entry2->relpath)) return 1;
404         if (FILENAME_IS_PARENT(entry1->relpath)) return -1;
405         if (FILENAME_IS_PARENT(entry2->relpath)) return 1;
406         
407         return 0;
408 }
409
410 static int compare_name(void *UNUSED(user_data), const void *a1, const void *a2)
411 {
412         const FileListInternEntry *entry1 = a1;
413         const FileListInternEntry *entry2 = a2;
414         char *name1, *name2;
415         int ret;
416
417         if ((ret = compare_direntry_generic(entry1, entry2))) {
418                 return ret;
419         }
420
421         name1 = entry1->name;
422         name2 = entry2->name;
423
424         return BLI_natstrcmp(name1, name2);
425 }
426
427 static int compare_date(void *UNUSED(user_data), const void *a1, const void *a2)
428 {
429         const FileListInternEntry *entry1 = a1;
430         const FileListInternEntry *entry2 = a2;
431         char *name1, *name2;
432         int64_t time1, time2;
433         int ret;
434
435         if ((ret = compare_direntry_generic(entry1, entry2))) {
436                 return ret;
437         }
438         
439         time1 = (int64_t)entry1->st.st_mtime;
440         time2 = (int64_t)entry2->st.st_mtime;
441         if (time1 < time2) return 1;
442         if (time1 > time2) return -1;
443
444         name1 = entry1->name;
445         name2 = entry2->name;
446
447         return BLI_natstrcmp(name1, name2);
448 }
449
450 static int compare_size(void *UNUSED(user_data), const void *a1, const void *a2)
451 {
452         const FileListInternEntry *entry1 = a1;
453         const FileListInternEntry *entry2 = a2;
454         char *name1, *name2;
455         uint64_t size1, size2;
456         int ret;
457
458         if ((ret = compare_direntry_generic(entry1, entry2))) {
459                 return ret;
460         }
461         
462         size1 = entry1->st.st_size;
463         size2 = entry2->st.st_size;
464         if (size1 < size2) return 1;
465         if (size1 > size2) return -1;
466
467         name1 = entry1->name;
468         name2 = entry2->name;
469
470         return BLI_natstrcmp(name1, name2);
471 }
472
473 static int compare_extension(void *UNUSED(user_data), const void *a1, const void *a2)
474 {
475         const FileListInternEntry *entry1 = a1;
476         const FileListInternEntry *entry2 = a2;
477         char *name1, *name2;
478         int ret;
479
480         if ((ret = compare_direntry_generic(entry1, entry2))) {
481                 return ret;
482         }
483
484         if ((entry1->typeflag & FILE_TYPE_BLENDERLIB) && !(entry2->typeflag & FILE_TYPE_BLENDERLIB)) return -1;
485         if (!(entry1->typeflag & FILE_TYPE_BLENDERLIB) && (entry2->typeflag & FILE_TYPE_BLENDERLIB)) return 1;
486         if ((entry1->typeflag & FILE_TYPE_BLENDERLIB) && (entry2->typeflag & FILE_TYPE_BLENDERLIB)) {
487                 if ((entry1->typeflag & FILE_TYPE_DIR) && !(entry2->typeflag & FILE_TYPE_DIR)) return 1;
488                 if (!(entry1->typeflag & FILE_TYPE_DIR) && (entry2->typeflag & FILE_TYPE_DIR)) return -1;
489                 if (entry1->blentype < entry2->blentype) return -1;
490                 if (entry1->blentype > entry2->blentype) return 1;
491         }
492         else {
493                 const char *sufix1, *sufix2;
494
495                 if (!(sufix1 = strstr(entry1->relpath, ".blend.gz")))
496                         sufix1 = strrchr(entry1->relpath, '.');
497                 if (!(sufix2 = strstr(entry2->relpath, ".blend.gz")))
498                         sufix2 = strrchr(entry2->relpath, '.');
499                 if (!sufix1) sufix1 = "";
500                 if (!sufix2) sufix2 = "";
501
502                 if ((ret = BLI_strcasecmp(sufix1, sufix2))) {
503                         return ret;
504                 }
505         }
506
507         name1 = entry1->name;
508         name2 = entry2->name;
509
510         return BLI_natstrcmp(name1, name2);
511 }
512
513 void filelist_sort(struct FileList *filelist)
514 {
515         if ((filelist->flags & FL_NEED_SORTING) && (filelist->sort != FILE_SORT_NONE)) {
516                 switch (filelist->sort) {
517                         case FILE_SORT_ALPHA:
518                                 BLI_listbase_sort_r(&filelist->filelist_intern.entries, compare_name, NULL);
519                                 break;
520                         case FILE_SORT_TIME:
521                                 BLI_listbase_sort_r(&filelist->filelist_intern.entries, compare_date, NULL);
522                                 break;
523                         case FILE_SORT_SIZE:
524                                 BLI_listbase_sort_r(&filelist->filelist_intern.entries, compare_size, NULL);
525                                 break;
526                         case FILE_SORT_EXTENSION:
527                                 BLI_listbase_sort_r(&filelist->filelist_intern.entries, compare_extension, NULL);
528                                 break;
529                         case FILE_SORT_NONE:  /* Should never reach this point! */
530                         default:
531                                 BLI_assert(0);
532                                 break;
533                 }
534
535                 filelist_filter_clear(filelist);
536                 filelist->flags &= ~FL_NEED_SORTING;
537         }
538 }
539
540 void filelist_setsorting(struct FileList *filelist, const short sort)
541 {
542         if (filelist->sort != sort) {
543                 filelist->sort = sort;
544                 filelist->flags |= FL_NEED_SORTING;
545         }
546 }
547
548 /* ********** Filter helpers ********** */
549
550 static bool is_hidden_file(const char *filename, FileListFilter *filter)
551 {
552         char *sep = (char *)BLI_last_slash(filename);
553         bool is_hidden = false;
554
555         if (filter->flags & FLF_HIDE_DOT) {
556                 if (filename[0] == '.' && filename[1] != '.' && filename[1] != '\0') {
557                         is_hidden = true; /* ignore .file */
558                 }
559                 else {
560                         int len = strlen(filename);
561                         if ((len > 0) && (filename[len - 1] == '~')) {
562                                 is_hidden = true;  /* ignore file~ */
563                         }
564                 }
565         }
566         if (!is_hidden && (filter->flags & FLF_HIDE_PARENT)) {
567                 if (filename[0] == '.' && filename[1] == '.' && filename[2] == '\0') {
568                         is_hidden = true; /* ignore .. */
569                 }
570         }
571         if (!is_hidden && ((filename[0] == '.') && (filename[1] == '\0'))) {
572                 is_hidden = true; /* ignore . */
573         }
574         /* filename might actually be a piece of path, in which case we have to check all its parts. */
575         if (!is_hidden && sep) {
576                 char tmp_filename[FILE_MAX_LIBEXTRA];
577
578                 BLI_strncpy(tmp_filename, filename, sizeof(tmp_filename));
579                 sep = tmp_filename + (sep - filename);
580                 while (sep) {
581                         BLI_assert(sep[1] != '\0');
582                         if (is_hidden_file(sep + 1, filter)) {
583                                 is_hidden = true;
584                                 break;
585                         }
586                         *sep = '\0';
587                         sep = (char *)BLI_last_slash(tmp_filename);
588                 }
589         }
590         return is_hidden;
591 }
592
593 static bool is_filtered_file(FileListInternEntry *file, const char *UNUSED(root), FileListFilter *filter)
594 {
595         bool is_filtered = !is_hidden_file(file->relpath, filter);
596
597         if (is_filtered && filter->filter && !FILENAME_IS_CURRPAR(file->relpath)) {
598                 if (file->typeflag & FILE_TYPE_DIR) {
599                         if (file->typeflag & (FILE_TYPE_BLENDERLIB | FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) {
600                                 if (!(filter->filter & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP))) {
601                                         is_filtered = false;
602                                 }
603                         }
604                         else {
605                                 if (!(filter->filter & FILE_TYPE_FOLDER)) {
606                                         is_filtered = false;
607                                 }
608                         }
609                 }
610                 else {
611                         if (!(file->typeflag & filter->filter)) {
612                                 is_filtered = false;
613                         }
614                 }
615                 if (is_filtered && (filter->filter_search[0] != '\0')) {
616                         if (fnmatch(filter->filter_search, file->relpath, FNM_CASEFOLD) != 0) {
617                                 is_filtered = false;
618                         }
619                 }
620         }
621
622         return is_filtered;
623 }
624
625 static bool is_filtered_lib(FileListInternEntry *file, const char *root, FileListFilter *filter)
626 {
627         bool is_filtered;
628         char path[FILE_MAX_LIBEXTRA], dir[FILE_MAX_LIBEXTRA], *group, *name;
629
630         BLI_join_dirfile(path, sizeof(path), root, file->relpath);
631
632         if (BLO_library_path_explode(path, dir, &group, &name)) {
633                 is_filtered = !is_hidden_file(file->relpath, filter);
634                 if (is_filtered && filter->filter && !FILENAME_IS_CURRPAR(file->relpath)) {
635                         if (file->typeflag & FILE_TYPE_DIR) {
636                                 if (file->typeflag & (FILE_TYPE_BLENDERLIB | FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) {
637                                         if (!(filter->filter & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP))) {
638                                                 is_filtered = false;
639                                         }
640                                 }
641                                 else {
642                                         if (!(filter->filter & FILE_TYPE_FOLDER)) {
643                                                 is_filtered = false;
644                                         }
645                                 }
646                         }
647                         if (is_filtered && group) {
648                                 if (!name && (filter->flags & FLF_HIDE_LIB_DIR)) {
649                                         is_filtered = false;
650                                 }
651                                 else {
652                                         unsigned int filter_id = groupname_to_filter_id(group);
653                                         if (!(filter_id & filter->filter_id)) {
654                                                 is_filtered = false;
655                                         }
656                                 }
657                         }
658                         if (is_filtered && (filter->filter_search[0] != '\0')) {
659                                 if (fnmatch(filter->filter_search, file->relpath, FNM_CASEFOLD) != 0) {
660                                         is_filtered = false;
661                                 }
662                         }
663                 }
664         }
665         else {
666                 is_filtered = is_filtered_file(file, root, filter);
667         }
668
669         return is_filtered;
670 }
671
672 static bool is_filtered_main(FileListInternEntry *file, const char *UNUSED(dir), FileListFilter *filter)
673 {
674         return !is_hidden_file(file->relpath, filter);
675 }
676
677 static void filelist_filter_clear(FileList *filelist)
678 {
679         filelist->flags |= FL_NEED_FILTERING;
680 }
681
682 void filelist_filter(FileList *filelist)
683 {
684         int num_filtered = 0;
685         const int num_files = filelist->filelist.nbr_entries;
686         FileListInternEntry **filtered_tmp, *file;
687
688         if (filelist->filelist.nbr_entries == 0) {
689                 return;
690         }
691
692         if (!(filelist->flags & FL_NEED_FILTERING)) {
693                 /* Assume it has already been filtered, nothing else to do! */
694                 return;
695         }
696
697         filelist->filter_data.flags &= ~FLF_HIDE_LIB_DIR;
698         if (filelist->max_recursion) {
699                 /* Never show lib ID 'categories' directories when we are in 'flat' mode, unless
700                  * root path is a blend file. */
701                 char dir[FILE_MAX_LIBEXTRA];
702                 if (!filelist_islibrary(filelist, dir, NULL)) {
703                         filelist->filter_data.flags |= FLF_HIDE_LIB_DIR;
704                 }
705         }
706
707         filtered_tmp = MEM_mallocN(sizeof(*filtered_tmp) * (size_t)num_files, __func__);
708
709         /* Filter remap & count how many files are left after filter in a single loop. */
710         for (file = filelist->filelist_intern.entries.first; file; file = file->next) {
711                 if (filelist->filterf(file, filelist->filelist.root, &filelist->filter_data)) {
712                         filtered_tmp[num_filtered++] = file;
713                 }
714         }
715
716         if (filelist->filelist_intern.filtered) {
717                 MEM_freeN(filelist->filelist_intern.filtered);
718         }
719         filelist->filelist_intern.filtered = MEM_mallocN(sizeof(*filelist->filelist_intern.filtered) * (size_t)num_filtered,
720                                                          __func__);
721         memcpy(filelist->filelist_intern.filtered, filtered_tmp,
722                sizeof(*filelist->filelist_intern.filtered) * (size_t)num_filtered);
723         filelist->filelist.nbr_entries_filtered = num_filtered;
724 //      printf("Filetered: %d over %d entries\n", num_filtered, filelist->filelist.nbr_entries);
725
726         filelist_cache_clear(&filelist->filelist_cache, filelist->filelist_cache.size);
727         filelist->flags &= ~FL_NEED_FILTERING;
728
729         MEM_freeN(filtered_tmp);
730 }
731
732 void filelist_setfilter_options(FileList *filelist, const bool hide_dot, const bool hide_parent,
733                                 const unsigned int filter, const unsigned int filter_id,
734                                 const char *filter_glob, const char *filter_search)
735 {
736         bool update = false;
737
738         if (((filelist->filter_data.flags & FLF_HIDE_DOT) != 0) != (hide_dot != 0)) {
739                 filelist->filter_data.flags ^= FLF_HIDE_DOT;
740                 update = true;
741         }
742         if (((filelist->filter_data.flags & FLF_HIDE_PARENT) != 0) != (hide_parent != 0)) {
743                 filelist->filter_data.flags ^= FLF_HIDE_PARENT;
744                 update = true;
745         }
746         if ((filelist->filter_data.filter != filter) || (filelist->filter_data.filter_id != filter_id)) {
747                 filelist->filter_data.filter = filter;
748                 filelist->filter_data.filter_id = filter_id;
749                 update = true;
750         }
751         if (!STREQ(filelist->filter_data.filter_glob, filter_glob)) {
752                 BLI_strncpy(filelist->filter_data.filter_glob, filter_glob, sizeof(filelist->filter_data.filter_glob));
753                 update = true;
754         }
755         if ((BLI_strcmp_ignore_pad(filelist->filter_data.filter_search, filter_search, '*') != 0)) {
756                 BLI_strncpy_ensure_pad(filelist->filter_data.filter_search, filter_search, '*',
757                                        sizeof(filelist->filter_data.filter_search));
758                 update = true;
759         }
760
761         if (update) {
762                 /* And now, free filtered data so that we know we have to filter again. */
763                 filelist_filter_clear(filelist);
764         }
765 }
766
767 /* ********** Icon/image helpers ********** */
768
769 void filelist_init_icons(void)
770 {
771         short x, y, k;
772         ImBuf *bbuf;
773         ImBuf *ibuf;
774
775         BLI_assert(G.background == false);
776
777 #ifdef WITH_HEADLESS
778         bbuf = NULL;
779 #else
780         bbuf = IMB_ibImageFromMemory((unsigned char *)datatoc_prvicons_png, datatoc_prvicons_png_size, IB_rect, NULL, "<splash>");
781 #endif
782         if (bbuf) {
783                 for (y = 0; y < SPECIAL_IMG_ROWS; y++) {
784                         for (x = 0; x < SPECIAL_IMG_COLS; x++) {
785                                 int tile = SPECIAL_IMG_COLS * y + x;
786                                 if (tile < SPECIAL_IMG_MAX) {
787                                         ibuf = IMB_allocImBuf(SPECIAL_IMG_SIZE, SPECIAL_IMG_SIZE, 32, IB_rect);
788                                         for (k = 0; k < SPECIAL_IMG_SIZE; k++) {
789                                                 memcpy(&ibuf->rect[k * SPECIAL_IMG_SIZE], &bbuf->rect[(k + y * SPECIAL_IMG_SIZE) * SPECIAL_IMG_SIZE * SPECIAL_IMG_COLS + x * SPECIAL_IMG_SIZE], SPECIAL_IMG_SIZE * sizeof(int));
790                                         }
791                                         gSpecialFileImages[tile] = ibuf;
792                                 }
793                         }
794                 }
795                 IMB_freeImBuf(bbuf);
796         }
797 }
798
799 void filelist_free_icons(void)
800 {
801         int i;
802
803         BLI_assert(G.background == false);
804
805         for (i = 0; i < SPECIAL_IMG_MAX; ++i) {
806                 IMB_freeImBuf(gSpecialFileImages[i]);
807                 gSpecialFileImages[i] = NULL;
808         }
809 }
810
811 void filelist_imgsize(struct FileList *filelist, short w, short h)
812 {
813         filelist->prv_w = w;
814         filelist->prv_h = h;
815 }
816
817 static FileDirEntry *filelist_geticon_get_file(struct FileList *filelist, const int index)
818 {
819         BLI_assert(G.background == false);
820
821         return filelist_file(filelist, index);
822 }
823
824 ImBuf *filelist_getimage(struct FileList *filelist, const int index)
825 {
826         FileDirEntry *file = filelist_geticon_get_file(filelist, index);
827
828         return file->image;
829 }
830
831 static ImBuf *filelist_geticon_image_ex(const unsigned int typeflag, const char *relpath)
832 {
833         ImBuf *ibuf = NULL;
834
835         if (typeflag & FILE_TYPE_DIR) {
836                 if (FILENAME_IS_PARENT(relpath)) {
837                         ibuf = gSpecialFileImages[SPECIAL_IMG_PARENT];
838                 }
839                 else if (FILENAME_IS_CURRENT(relpath)) {
840                         ibuf = gSpecialFileImages[SPECIAL_IMG_REFRESH];
841                 }
842                 else {
843                         ibuf = gSpecialFileImages[SPECIAL_IMG_FOLDER];
844                 }
845         }
846         else if (typeflag & FILE_TYPE_BLENDER) {
847                 ibuf = gSpecialFileImages[SPECIAL_IMG_BLENDFILE];
848         }
849         else if (typeflag & FILE_TYPE_BLENDERLIB) {
850                 ibuf = gSpecialFileImages[SPECIAL_IMG_UNKNOWNFILE];
851         }
852         else if (typeflag & (FILE_TYPE_MOVIE)) {
853                 ibuf = gSpecialFileImages[SPECIAL_IMG_MOVIEFILE];
854         }
855         else if (typeflag & FILE_TYPE_SOUND) {
856                 ibuf = gSpecialFileImages[SPECIAL_IMG_SOUNDFILE];
857         }
858         else if (typeflag & FILE_TYPE_PYSCRIPT) {
859                 ibuf = gSpecialFileImages[SPECIAL_IMG_PYTHONFILE];
860         }
861         else if (typeflag & FILE_TYPE_FTFONT) {
862                 ibuf = gSpecialFileImages[SPECIAL_IMG_FONTFILE];
863         }
864         else if (typeflag & FILE_TYPE_TEXT) {
865                 ibuf = gSpecialFileImages[SPECIAL_IMG_TEXTFILE];
866         }
867         else if (typeflag & FILE_TYPE_IMAGE) {
868                 ibuf = gSpecialFileImages[SPECIAL_IMG_LOADING];
869         }
870         else if (typeflag & FILE_TYPE_BLENDER_BACKUP) {
871                 ibuf = gSpecialFileImages[SPECIAL_IMG_BACKUP];
872         }
873         else {
874                 ibuf = gSpecialFileImages[SPECIAL_IMG_UNKNOWNFILE];
875         }
876
877         return ibuf;
878 }
879
880 ImBuf *filelist_geticon_image(struct FileList *filelist, const int index)
881 {
882         FileDirEntry *file = filelist_geticon_get_file(filelist, index);
883
884         return filelist_geticon_image_ex(file->typeflag, file->relpath);
885 }
886
887 static int filelist_geticon_ex(
888         const int typeflag, const int blentype, const char *relpath, const bool is_main, const bool ignore_libdir)
889 {
890         if ((typeflag & FILE_TYPE_DIR) && !(ignore_libdir && (typeflag & (FILE_TYPE_BLENDERLIB | FILE_TYPE_BLENDER)))) {
891                 if (FILENAME_IS_PARENT(relpath)) {
892                         return is_main ? ICON_FILE_PARENT : ICON_NONE;
893                 }
894                 else if (typeflag & FILE_TYPE_APPLICATIONBUNDLE) {
895                         return ICON_UGLYPACKAGE;
896                 }
897                 else if (typeflag & FILE_TYPE_BLENDER) {
898                         return ICON_FILE_BLEND;
899                 }
900                 else if (is_main) {
901                         /* Do not return icon for folders if icons are not 'main' draw type (e.g. when used over previews). */
902                         return ICON_FILE_FOLDER;
903                 }
904         }
905
906         if (typeflag & FILE_TYPE_BLENDER)
907                 return ICON_FILE_BLEND;
908         else if (typeflag & FILE_TYPE_BLENDER_BACKUP)
909                 return ICON_FILE_BACKUP;
910         else if (typeflag & FILE_TYPE_IMAGE)
911                 return ICON_FILE_IMAGE;
912         else if (typeflag & FILE_TYPE_MOVIE)
913                 return ICON_FILE_MOVIE;
914         else if (typeflag & FILE_TYPE_PYSCRIPT)
915                 return ICON_FILE_SCRIPT;
916         else if (typeflag & FILE_TYPE_SOUND)
917                 return ICON_FILE_SOUND;
918         else if (typeflag & FILE_TYPE_FTFONT)
919                 return ICON_FILE_FONT;
920         else if (typeflag & FILE_TYPE_BTX)
921                 return ICON_FILE_BLANK;
922         else if (typeflag & FILE_TYPE_COLLADA)
923                 return ICON_FILE_BLANK;
924         else if (typeflag & FILE_TYPE_ALEMBIC)
925                 return ICON_FILE_BLANK;
926         else if (typeflag & FILE_TYPE_TEXT)
927                 return ICON_FILE_TEXT;
928         else if (typeflag & FILE_TYPE_BLENDERLIB) {
929                 const int ret = UI_idcode_icon_get(blentype);
930                 if (ret != ICON_NONE) {
931                         return ret;
932                 }
933         }
934         return is_main ? ICON_FILE_BLANK : ICON_NONE;
935 }
936
937 int filelist_geticon(struct FileList *filelist, const int index, const bool is_main)
938 {
939         FileDirEntry *file = filelist_geticon_get_file(filelist, index);
940
941         return filelist_geticon_ex(file->typeflag, file->blentype, file->relpath, is_main, false);
942 }
943
944 /* ********** Main ********** */
945
946 static bool filelist_checkdir_dir(struct FileList *UNUSED(filelist), char *r_dir, const bool do_change)
947 {
948         if (do_change) {
949                 BLI_make_exist(r_dir);
950                 return true;
951         }
952         else {
953                 return BLI_is_dir(r_dir);
954         }
955 }
956
957 static bool filelist_checkdir_lib(struct FileList *UNUSED(filelist), char *r_dir, const bool do_change)
958 {
959         char tdir[FILE_MAX_LIBEXTRA];
960         char *name;
961
962         const bool is_valid = (BLI_is_dir(r_dir) ||
963                                (BLO_library_path_explode(r_dir, tdir, NULL, &name) && BLI_is_file(tdir) && !name));
964
965         if (do_change && !is_valid) {
966                 /* if not a valid library, we need it to be a valid directory! */
967                 BLI_make_exist(r_dir);
968                 return true;
969         }
970         return is_valid;
971 }
972
973 static bool filelist_checkdir_main(struct FileList *filelist, char *r_dir, const bool do_change)
974 {
975         /* TODO */
976         return filelist_checkdir_lib(filelist, r_dir, do_change);
977 }
978
979 static void filelist_entry_clear(FileDirEntry *entry)
980 {
981         if (entry->name) {
982                 MEM_freeN(entry->name);
983         }
984         if (entry->description) {
985                 MEM_freeN(entry->description);
986         }
987         if (entry->relpath) {
988                 MEM_freeN(entry->relpath);
989         }
990         if (entry->image) {
991                 IMB_freeImBuf(entry->image);
992         }
993         /* For now, consider FileDirEntryRevision::poin as not owned here, so no need to do anything about it */
994
995         if (!BLI_listbase_is_empty(&entry->variants)) {
996                 FileDirEntryVariant *var;
997
998                 for (var = entry->variants.first; var; var = var->next) {
999                         if (var->name) {
1000                                 MEM_freeN(var->name);
1001                         }
1002                         if (var->description) {
1003                                 MEM_freeN(var->description);
1004                         }
1005
1006                         if (!BLI_listbase_is_empty(&var->revisions)) {
1007                                 FileDirEntryRevision *rev;
1008
1009                                 for (rev = var->revisions.first; rev; rev = rev->next) {
1010                                         if (rev->comment) {
1011                                                 MEM_freeN(rev->comment);
1012                                         }
1013                                 }
1014
1015                                 BLI_freelistN(&var->revisions);
1016                         }
1017                 }
1018
1019                 /* TODO: tags! */
1020
1021                 BLI_freelistN(&entry->variants);
1022         }
1023         else if (entry->entry) {
1024                 MEM_freeN(entry->entry);
1025         }
1026 }
1027
1028 static void filelist_entry_free(FileDirEntry *entry)
1029 {
1030         filelist_entry_clear(entry);
1031         MEM_freeN(entry);
1032 }
1033
1034 static void filelist_direntryarr_free(FileDirEntryArr *array)
1035 {
1036 #if 0
1037         FileDirEntry *entry, *entry_next;
1038
1039         for (entry = array->entries.first; entry; entry = entry_next) {
1040                 entry_next = entry->next;
1041                 filelist_entry_free(entry);
1042         }
1043         BLI_listbase_clear(&array->entries);
1044 #else
1045         BLI_assert(BLI_listbase_is_empty(&array->entries));
1046 #endif
1047         array->nbr_entries = 0;
1048         array->nbr_entries_filtered = -1;
1049         array->entry_idx_start = -1;
1050         array->entry_idx_end = -1;
1051 }
1052
1053 static void filelist_intern_entry_free(FileListInternEntry *entry)
1054 {
1055         if (entry->relpath) {
1056                 MEM_freeN(entry->relpath);
1057         }
1058         if (entry->name) {
1059                 MEM_freeN(entry->name);
1060         }
1061         MEM_freeN(entry);
1062 }
1063
1064 static void filelist_intern_free(FileListIntern *filelist_intern)
1065 {
1066         FileListInternEntry *entry, *entry_next;
1067
1068         for (entry = filelist_intern->entries.first; entry; entry = entry_next) {
1069                 entry_next = entry->next;
1070                 filelist_intern_entry_free(entry);
1071         }
1072         BLI_listbase_clear(&filelist_intern->entries);
1073
1074         MEM_SAFE_FREE(filelist_intern->filtered);
1075 }
1076
1077 static void filelist_cache_preview_runf(TaskPool *__restrict pool, void *taskdata, int UNUSED(threadid))
1078 {
1079         FileListEntryCache *cache = BLI_task_pool_userdata(pool);
1080         FileListEntryPreview *preview = taskdata;
1081
1082         ThumbSource source = 0;
1083
1084 //      printf("%s: Start (%d)...\n", __func__, threadid);
1085
1086 //      printf("%s: %d - %s - %p\n", __func__, preview->index, preview->path, preview->img);
1087         BLI_assert(preview->flags & (FILE_TYPE_IMAGE | FILE_TYPE_MOVIE | FILE_TYPE_FTFONT |
1088                                      FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP | FILE_TYPE_BLENDERLIB));
1089
1090         if (preview->flags & FILE_TYPE_IMAGE) {
1091                 source = THB_SOURCE_IMAGE;
1092         }
1093         else if (preview->flags & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP | FILE_TYPE_BLENDERLIB)) {
1094                 source = THB_SOURCE_BLEND;
1095         }
1096         else if (preview->flags & FILE_TYPE_MOVIE) {
1097                 source = THB_SOURCE_MOVIE;
1098         }
1099         else if (preview->flags & FILE_TYPE_FTFONT) {
1100                 source = THB_SOURCE_FONT;
1101         }
1102
1103         IMB_thumb_path_lock(preview->path);
1104         preview->img = IMB_thumb_manage(preview->path, THB_LARGE, source);
1105         IMB_thumb_path_unlock(preview->path);
1106
1107         preview->flags = 0;  /* Used to tell free func to not free anything! */
1108         BLI_thread_queue_push(cache->previews_done, preview);
1109
1110 //      printf("%s: End (%d)...\n", __func__, threadid);
1111 }
1112
1113 static void filelist_cache_preview_freef(TaskPool * __restrict UNUSED(pool), void *taskdata, int UNUSED(threadid))
1114 {
1115         FileListEntryPreview *preview = taskdata;
1116
1117         /* If preview->flag is empty, it means that preview has already been generated and added to done queue,
1118          * we do not own it anymore. */
1119         if (preview->flags) {
1120                 if (preview->img) {
1121                         IMB_freeImBuf(preview->img);
1122                 }
1123                 MEM_freeN(preview);
1124         }
1125 }
1126
1127 static void filelist_cache_preview_ensure_running(FileListEntryCache *cache)
1128 {
1129         if (!cache->previews_pool) {
1130                 TaskScheduler *scheduler = BLI_task_scheduler_get();
1131
1132                 cache->previews_pool = BLI_task_pool_create_background(scheduler, cache);
1133                 cache->previews_done = BLI_thread_queue_init();
1134
1135                 IMB_thumb_locks_acquire();
1136         }
1137 }
1138
1139 static void filelist_cache_previews_clear(FileListEntryCache *cache)
1140 {
1141         FileListEntryPreview *preview;
1142
1143         if (cache->previews_pool) {
1144                 BLI_task_pool_cancel(cache->previews_pool);
1145
1146                 while ((preview = BLI_thread_queue_pop_timeout(cache->previews_done, 0))) {
1147 //                      printf("%s: DONE %d - %s - %p\n", __func__, preview->index, preview->path, preview->img);
1148                         if (preview->img) {
1149                                 IMB_freeImBuf(preview->img);
1150                         }
1151                         MEM_freeN(preview);
1152                 }
1153         }
1154 }
1155
1156 static void filelist_cache_previews_free(FileListEntryCache *cache)
1157 {
1158         if (cache->previews_pool) {
1159                 BLI_thread_queue_nowait(cache->previews_done);
1160
1161                 filelist_cache_previews_clear(cache);
1162
1163                 BLI_thread_queue_free(cache->previews_done);
1164                 BLI_task_pool_free(cache->previews_pool);
1165                 cache->previews_pool = NULL;
1166                 cache->previews_done = NULL;
1167
1168                 IMB_thumb_locks_release();
1169         }
1170
1171         cache->flags &= ~FLC_PREVIEWS_ACTIVE;
1172 }
1173
1174 static void filelist_cache_previews_push(FileList *filelist, FileDirEntry *entry, const int index)
1175 {
1176         FileListEntryCache *cache = &filelist->filelist_cache;
1177
1178         BLI_assert(cache->flags & FLC_PREVIEWS_ACTIVE);
1179
1180         if (!entry->image &&
1181             !(entry->flags & FILE_ENTRY_INVALID_PREVIEW) &&
1182             (entry->typeflag & (FILE_TYPE_IMAGE | FILE_TYPE_MOVIE | FILE_TYPE_FTFONT |
1183                                 FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP | FILE_TYPE_BLENDERLIB)))
1184         {
1185                 FileListEntryPreview *preview = MEM_mallocN(sizeof(*preview), __func__);
1186                 BLI_join_dirfile(preview->path, sizeof(preview->path), filelist->filelist.root, entry->relpath);
1187                 preview->index = index;
1188                 preview->flags = entry->typeflag;
1189                 preview->img = NULL;
1190 //              printf("%s: %d - %s - %p\n", __func__, preview->index, preview->path, preview->img);
1191
1192                 filelist_cache_preview_ensure_running(cache);
1193                 BLI_task_pool_push_ex(cache->previews_pool, filelist_cache_preview_runf, preview,
1194                                       true, filelist_cache_preview_freef, TASK_PRIORITY_LOW);
1195         }
1196 }
1197
1198 static void filelist_cache_init(FileListEntryCache *cache, size_t cache_size)
1199 {
1200         BLI_listbase_clear(&cache->cached_entries);
1201
1202         cache->block_cursor = cache->block_start_index = cache->block_center_index = cache->block_end_index = 0;
1203         cache->block_entries = MEM_mallocN(sizeof(*cache->block_entries) * cache_size, __func__);
1204
1205         cache->misc_entries = BLI_ghash_ptr_new_ex(__func__, cache_size);
1206         cache->misc_entries_indices = MEM_mallocN(sizeof(*cache->misc_entries_indices) * cache_size, __func__);
1207         copy_vn_i(cache->misc_entries_indices, cache_size, -1);
1208         cache->misc_cursor = 0;
1209
1210         /* XXX This assumes uint is 32 bits and uuid is 128 bits (char[16]), be careful! */
1211         cache->uuids = BLI_ghash_new_ex(
1212                            BLI_ghashutil_uinthash_v4_p, BLI_ghashutil_uinthash_v4_cmp, __func__, cache_size * 2);
1213
1214         cache->size = cache_size;
1215         cache->flags = FLC_IS_INIT;
1216 }
1217
1218 static void filelist_cache_free(FileListEntryCache *cache)
1219 {
1220         FileDirEntry *entry, *entry_next;
1221
1222         if (!(cache->flags & FLC_IS_INIT)) {
1223                 return;
1224         }
1225
1226         filelist_cache_previews_free(cache);
1227
1228         MEM_freeN(cache->block_entries);
1229
1230         BLI_ghash_free(cache->misc_entries, NULL, NULL);
1231         MEM_freeN(cache->misc_entries_indices);
1232
1233         BLI_ghash_free(cache->uuids, NULL, NULL);
1234
1235         for (entry = cache->cached_entries.first; entry; entry = entry_next) {
1236                 entry_next = entry->next;
1237                 filelist_entry_free(entry);
1238         }
1239         BLI_listbase_clear(&cache->cached_entries);
1240 }
1241
1242 static void filelist_cache_clear(FileListEntryCache *cache, size_t new_size)
1243 {
1244         FileDirEntry *entry, *entry_next;
1245
1246         if (!(cache->flags & FLC_IS_INIT)) {
1247                 return;
1248         }
1249
1250         filelist_cache_previews_clear(cache);
1251
1252         cache->block_cursor = cache->block_start_index = cache->block_center_index = cache->block_end_index = 0;
1253         if (new_size != cache->size) {
1254                 cache->block_entries = MEM_reallocN(cache->block_entries, sizeof(*cache->block_entries) * new_size);
1255         }
1256
1257         BLI_ghash_clear_ex(cache->misc_entries, NULL, NULL, new_size);
1258         if (new_size != cache->size) {
1259                 cache->misc_entries_indices = MEM_reallocN(cache->misc_entries_indices,
1260                                                            sizeof(*cache->misc_entries_indices) * new_size);
1261         }
1262         copy_vn_i(cache->misc_entries_indices, new_size, -1);
1263
1264         BLI_ghash_clear_ex(cache->uuids, NULL, NULL, new_size * 2);
1265
1266         cache->size = new_size;
1267
1268         for (entry = cache->cached_entries.first; entry; entry = entry_next) {
1269                 entry_next = entry->next;
1270                 filelist_entry_free(entry);
1271         }
1272         BLI_listbase_clear(&cache->cached_entries);
1273 }
1274
1275 FileList *filelist_new(short type)
1276 {
1277         FileList *p = MEM_callocN(sizeof(*p), __func__);
1278
1279         filelist_cache_init(&p->filelist_cache, FILELIST_ENTRYCACHESIZE_DEFAULT);
1280
1281         p->selection_state = BLI_ghash_new(BLI_ghashutil_uinthash_v4_p, BLI_ghashutil_uinthash_v4_cmp, __func__);
1282
1283         switch (type) {
1284                 case FILE_MAIN:
1285                         p->checkdirf = filelist_checkdir_main;
1286                         p->read_jobf = filelist_readjob_main;
1287                         p->filterf = is_filtered_main;
1288                         break;
1289                 case FILE_LOADLIB:
1290                         p->checkdirf = filelist_checkdir_lib;
1291                         p->read_jobf = filelist_readjob_lib;
1292                         p->filterf = is_filtered_lib;
1293                         break;
1294                 default:
1295                         p->checkdirf = filelist_checkdir_dir;
1296                         p->read_jobf = filelist_readjob_dir;
1297                         p->filterf = is_filtered_file;
1298                         break;
1299         }
1300         return p;
1301 }
1302
1303 void filelist_clear_ex(struct FileList *filelist, const bool do_cache, const bool do_selection)
1304 {
1305         if (!filelist) {
1306                 return;
1307         }
1308
1309         filelist_filter_clear(filelist);
1310
1311         if (do_cache) {
1312                 filelist_cache_clear(&filelist->filelist_cache, filelist->filelist_cache.size);
1313         }
1314
1315         filelist_intern_free(&filelist->filelist_intern);
1316
1317         filelist_direntryarr_free(&filelist->filelist);
1318
1319         if (do_selection && filelist->selection_state) {
1320                 BLI_ghash_clear(filelist->selection_state, MEM_freeN, NULL);
1321         }
1322 }
1323
1324 void filelist_clear(struct FileList *filelist)
1325 {
1326         filelist_clear_ex(filelist, true, true);
1327 }
1328
1329 void filelist_free(struct FileList *filelist)
1330 {
1331         if (!filelist) {
1332                 printf("Attempting to delete empty filelist.\n");
1333                 return;
1334         }
1335         
1336         filelist_clear_ex(filelist, false, false);  /* No need to clear cache & selection_state, we free them anyway. */
1337         filelist_cache_free(&filelist->filelist_cache);
1338
1339         if (filelist->selection_state) {
1340                 BLI_ghash_free(filelist->selection_state, MEM_freeN, NULL);
1341                 filelist->selection_state = NULL;
1342         }
1343
1344         memset(&filelist->filter_data, 0, sizeof(filelist->filter_data));
1345
1346         filelist->flags &= ~(FL_NEED_SORTING | FL_NEED_FILTERING);
1347         filelist->sort = FILE_SORT_NONE;
1348 }
1349
1350 void filelist_freelib(struct FileList *filelist)
1351 {
1352         if (filelist->libfiledata)
1353                 BLO_blendhandle_close(filelist->libfiledata);
1354         filelist->libfiledata = NULL;
1355 }
1356
1357 BlendHandle *filelist_lib(struct FileList *filelist)
1358 {
1359         return filelist->libfiledata;
1360 }
1361
1362 static const char *fileentry_uiname(const char *root, const char *relpath, const int typeflag, char *buff)
1363 {
1364         char *name = NULL;
1365
1366         if (typeflag & FILE_TYPE_BLENDERLIB) {
1367                 char abspath[FILE_MAX_LIBEXTRA];
1368                 char *group;
1369
1370                 BLI_join_dirfile(abspath, sizeof(abspath), root, relpath);
1371                 BLO_library_path_explode(abspath, buff, &group, &name);
1372                 if (!name) {
1373                         name = group;
1374                 }
1375         }
1376         /* Depending on platforms, 'my_file.blend/..' might be viewed as dir or not... */
1377         if (!name) {
1378                 if (typeflag & FILE_TYPE_DIR) {
1379                         name = (char *)relpath;
1380                 }
1381                 else {
1382                         name = (char *)BLI_path_basename(relpath);
1383                 }
1384         }
1385         BLI_assert(name);
1386
1387         return name;
1388 }
1389
1390 const char *filelist_dir(struct FileList *filelist)
1391 {
1392         return filelist->filelist.root;
1393 }
1394
1395 bool filelist_is_dir(struct FileList *filelist, const char *path)
1396 {
1397         return filelist->checkdirf(filelist, (char *)path, false);
1398 }
1399
1400 /**
1401  * May modify in place given r_dir, which is expected to be FILE_MAX_LIBEXTRA length.
1402  */
1403 void filelist_setdir(struct FileList *filelist, char *r_dir)
1404 {
1405         BLI_assert(strlen(r_dir) < FILE_MAX_LIBEXTRA);
1406
1407         BLI_cleanup_dir(G.main->name, r_dir);
1408         BLI_assert(filelist->checkdirf(filelist, r_dir, true));
1409
1410         if (!STREQ(filelist->filelist.root, r_dir)) {
1411                 BLI_strncpy(filelist->filelist.root, r_dir, sizeof(filelist->filelist.root));
1412                 filelist->flags |= FL_FORCE_RESET;
1413         }
1414 }
1415
1416 void filelist_setrecursion(struct FileList *filelist, const int recursion_level)
1417 {
1418         if (filelist->max_recursion != recursion_level) {
1419                 filelist->max_recursion = recursion_level;
1420                 filelist->flags |= FL_FORCE_RESET;
1421         }
1422 }
1423
1424 bool filelist_force_reset(struct FileList *filelist)
1425 {
1426         return (filelist->flags & FL_FORCE_RESET) != 0;
1427 }
1428
1429 bool filelist_is_ready(struct FileList *filelist)
1430 {
1431         return (filelist->flags & FL_IS_READY) != 0;
1432 }
1433
1434 bool filelist_pending(struct FileList *filelist)
1435 {
1436         return (filelist->flags & FL_IS_PENDING) != 0;
1437 }
1438
1439 /**
1440  * Limited version of full update done by space_file's file_refresh(), to be used by operators and such.
1441  * Ensures given filelist is ready to be used (i.e. it is filtered and sorted), unless it is tagged for a full refresh.
1442  */
1443 int filelist_files_ensure(FileList *filelist)
1444 {
1445         if (!filelist_force_reset(filelist) || !filelist_empty(filelist)) {
1446                 filelist_sort(filelist);
1447                 filelist_filter(filelist);
1448         }
1449
1450         return filelist->filelist.nbr_entries_filtered;
1451 }
1452
1453 static FileDirEntry *filelist_file_create_entry(FileList *filelist, const int index)
1454 {
1455         FileListInternEntry *entry = filelist->filelist_intern.filtered[index];
1456         FileListEntryCache *cache = &filelist->filelist_cache;
1457         FileDirEntry *ret;
1458         FileDirEntryRevision *rev;
1459
1460         ret = MEM_callocN(sizeof(*ret), __func__);
1461         rev = MEM_callocN(sizeof(*rev), __func__);
1462
1463         rev->size = (uint64_t)entry->st.st_size;
1464
1465         rev->time = (int64_t)entry->st.st_mtime;
1466
1467         ret->entry = rev;
1468         ret->relpath = BLI_strdup(entry->relpath);
1469         ret->name = BLI_strdup(entry->name);
1470         ret->description = BLI_strdupcat(filelist->filelist.root, entry->relpath);
1471         memcpy(ret->uuid, entry->uuid, sizeof(ret->uuid));
1472         ret->blentype = entry->blentype;
1473         ret->typeflag = entry->typeflag;
1474
1475         BLI_addtail(&cache->cached_entries, ret);
1476         return ret;
1477 }
1478
1479 static void filelist_file_release_entry(FileList *filelist, FileDirEntry *entry)
1480 {
1481         BLI_remlink(&filelist->filelist_cache.cached_entries, entry);
1482         filelist_entry_free(entry);
1483 }
1484
1485 static FileDirEntry *filelist_file_ex(struct FileList *filelist, const int index, const bool use_request)
1486 {
1487         FileDirEntry *ret = NULL, *old;
1488         FileListEntryCache *cache = &filelist->filelist_cache;
1489         const size_t cache_size = cache->size;
1490         int old_index;
1491
1492         if ((index < 0) || (index >= filelist->filelist.nbr_entries_filtered)) {
1493                 return ret;
1494         }
1495
1496         if (index >= cache->block_start_index && index < cache->block_end_index) {
1497                 const int idx = (index - cache->block_start_index + cache->block_cursor) % cache_size;
1498                 return cache->block_entries[idx];
1499         }
1500
1501         if ((ret = BLI_ghash_lookup(cache->misc_entries, SET_INT_IN_POINTER(index)))) {
1502                 return ret;
1503         }
1504
1505         if (!use_request) {
1506                 return NULL;
1507         }
1508
1509 //      printf("requesting file %d (not yet cached)\n", index);
1510
1511         /* Else, we have to add new entry to 'misc' cache - and possibly make room for it first! */
1512         ret = filelist_file_create_entry(filelist, index);
1513         old_index = cache->misc_entries_indices[cache->misc_cursor];
1514         if ((old = BLI_ghash_popkey(cache->misc_entries, SET_INT_IN_POINTER(old_index), NULL))) {
1515                 BLI_ghash_remove(cache->uuids, old->uuid, NULL, NULL);
1516                 filelist_file_release_entry(filelist, old);
1517         }
1518         BLI_ghash_insert(cache->misc_entries, SET_INT_IN_POINTER(index), ret);
1519         BLI_ghash_insert(cache->uuids, ret->uuid, ret);
1520
1521         cache->misc_entries_indices[cache->misc_cursor] = index;
1522         cache->misc_cursor = (cache->misc_cursor + 1) % cache_size;
1523
1524 #if 0  /* Actually no, only block cached entries should have preview imho. */
1525         if (cache->previews_pool) {
1526                 filelist_cache_previews_push(filelist, ret, index);
1527         }
1528 #endif
1529
1530         return ret;
1531 }
1532
1533 FileDirEntry *filelist_file(struct FileList *filelist, int index)
1534 {
1535         return filelist_file_ex(filelist, index, true);
1536 }
1537
1538 int filelist_file_findpath(struct FileList *filelist, const char *filename)
1539 {
1540         int fidx = -1;
1541         
1542         if (filelist->filelist.nbr_entries_filtered < 0) {
1543                 return fidx;
1544         }
1545
1546         /* XXX TODO Cache could probably use a ghash on paths too? Not really urgent though.
1547          *          This is only used to find again renamed entry, annoying but looks hairy to get rid of it currently. */
1548
1549         for (fidx = 0; fidx < filelist->filelist.nbr_entries_filtered; fidx++) {
1550                 FileListInternEntry *entry = filelist->filelist_intern.filtered[fidx];
1551                 if (STREQ(entry->relpath, filename)) {
1552                         return fidx;
1553                 }
1554         }
1555
1556         return -1;
1557 }
1558
1559 FileDirEntry *filelist_entry_find_uuid(struct FileList *filelist, const int uuid[4])
1560 {
1561         if (filelist->filelist.nbr_entries_filtered < 0) {
1562                 return NULL;
1563         }
1564
1565         if (filelist->filelist_cache.uuids) {
1566                 FileDirEntry *entry = BLI_ghash_lookup(filelist->filelist_cache.uuids, uuid);
1567                 if (entry) {
1568                         return entry;
1569                 }
1570         }
1571
1572         {
1573                 int fidx;
1574
1575                 for (fidx = 0; fidx < filelist->filelist.nbr_entries_filtered; fidx++) {
1576                         FileListInternEntry *entry = filelist->filelist_intern.filtered[fidx];
1577                         if (memcmp(entry->uuid, uuid, sizeof(entry->uuid)) == 0) {
1578                                 return filelist_file(filelist, fidx);
1579                         }
1580                 }
1581         }
1582
1583         return NULL;
1584 }
1585
1586 void filelist_file_cache_slidingwindow_set(FileList *filelist, size_t window_size)
1587 {
1588         /* Always keep it power of 2, in [256, 8192] range for now, cache being app. twice bigger than requested window. */
1589         size_t size = 256;
1590         window_size *= 2;
1591
1592         while (size < window_size && size < 8192) {
1593                 size *= 2;
1594         }
1595
1596         if (size != filelist->filelist_cache.size) {
1597                 filelist_cache_clear(&filelist->filelist_cache, size);
1598         }
1599 }
1600
1601 /* Helpers, low-level, they assume cursor + size <= cache_size */
1602 static bool filelist_file_cache_block_create(FileList *filelist, const int start_index, const int size, int cursor)
1603 {
1604         FileListEntryCache *cache = &filelist->filelist_cache;
1605
1606         {
1607                 int i, idx;
1608
1609                 for (i = 0, idx = start_index; i < size; i++, idx++, cursor++) {
1610                         FileDirEntry *entry;
1611
1612                         /* That entry might have already been requested and stored in misc cache... */
1613                         if ((entry = BLI_ghash_popkey(cache->misc_entries, SET_INT_IN_POINTER(idx), NULL)) == NULL) {
1614                                 entry = filelist_file_create_entry(filelist, idx);
1615                                 BLI_ghash_insert(cache->uuids, entry->uuid, entry);
1616                         }
1617                         cache->block_entries[cursor] = entry;
1618                 }
1619                 return true;
1620         }
1621
1622         return false;
1623 }
1624
1625 static void filelist_file_cache_block_release(struct FileList *filelist, const int size, int cursor)
1626 {
1627         FileListEntryCache *cache = &filelist->filelist_cache;
1628
1629         {
1630                 int i;
1631
1632                 for (i = 0; i < size; i++, cursor++) {
1633                         FileDirEntry *entry = cache->block_entries[cursor];
1634 //                      printf("%s: release cacheidx %d (%%p %%s)\n", __func__, cursor/*, cache->block_entries[cursor], cache->block_entries[cursor]->relpath*/);
1635                         BLI_ghash_remove(cache->uuids, entry->uuid, NULL, NULL);
1636                         filelist_file_release_entry(filelist, entry);
1637 #ifndef NDEBUG
1638                         cache->block_entries[cursor] = NULL;
1639 #endif
1640                 }
1641         }
1642 }
1643
1644 /* Load in cache all entries "around" given index (as much as block cache may hold). */
1645 bool filelist_file_cache_block(struct FileList *filelist, const int index)
1646 {
1647         FileListEntryCache *cache = &filelist->filelist_cache;
1648         const size_t cache_size = cache->size;
1649
1650         const int nbr_entries = filelist->filelist.nbr_entries_filtered;
1651         int start_index = max_ii(0, index - (cache_size / 2));
1652         int end_index = min_ii(nbr_entries, index + (cache_size / 2));
1653         int i;
1654
1655         if ((index < 0) || (index >= nbr_entries)) {
1656 //              printf("Wrong index %d ([%d:%d])", index, 0, nbr_entries);
1657                 return false;
1658         }
1659
1660         /* Maximize cached range! */
1661         if ((end_index - start_index) < cache_size) {
1662                 if (start_index == 0) {
1663                         end_index = min_ii(nbr_entries, start_index + cache_size);
1664                 }
1665                 else if (end_index == nbr_entries) {
1666                         start_index = max_ii(0, end_index - cache_size);
1667                 }
1668         }
1669
1670         BLI_assert((end_index - start_index) <= cache_size) ;
1671
1672 //      printf("%s: [%d:%d] around index %d (current cache: [%d:%d])\n", __func__,
1673 //             start_index, end_index, index, cache->block_start_index, cache->block_end_index);
1674
1675         /* If we have something to (re)cache... */
1676         if ((start_index != cache->block_start_index) || (end_index != cache->block_end_index)) {
1677                 if ((start_index >= cache->block_end_index) || (end_index <= cache->block_start_index)) {
1678                         int size1 = cache->block_end_index - cache->block_start_index;
1679                         int size2 = 0;
1680                         int idx1 = cache->block_cursor, idx2 = 0;
1681
1682 //                      printf("Full Recaching!\n");
1683
1684                         if (cache->flags & FLC_PREVIEWS_ACTIVE) {
1685                                 filelist_cache_previews_clear(cache);
1686                         }
1687
1688                         if (idx1 + size1 > cache_size) {
1689                                 size2 = idx1 + size1 - cache_size;
1690                                 size1 -= size2;
1691                                 filelist_file_cache_block_release(filelist, size2, idx2);
1692                         }
1693                         filelist_file_cache_block_release(filelist, size1, idx1);
1694
1695                         cache->block_start_index = cache->block_end_index = cache->block_cursor = 0;
1696
1697                         /* New cached block does not overlap existing one, simple. */
1698                         if (!filelist_file_cache_block_create(filelist, start_index, end_index - start_index, 0)) {
1699                                 return false;
1700                         }
1701
1702                         cache->block_start_index = start_index;
1703                         cache->block_end_index = end_index;
1704                 }
1705                 else {
1706 //                      printf("Partial Recaching!\n");
1707
1708                         /* At this point, we know we keep part of currently cached entries, so update previews if needed,
1709                          * and remove everything from working queue - we'll add all newly needed entries at the end. */
1710                         if (cache->flags & FLC_PREVIEWS_ACTIVE) {
1711                                 filelist_cache_previews_update(filelist);
1712                                 filelist_cache_previews_clear(cache);
1713                         }
1714
1715 //                      printf("\tpreview cleaned up...\n");
1716
1717                         if (start_index > cache->block_start_index) {
1718                                 int size1 = start_index - cache->block_start_index;
1719                                 int size2 = 0;
1720                                 int idx1 = cache->block_cursor, idx2 = 0;
1721
1722 //                              printf("\tcache releasing: [%d:%d] (%d, %d)\n", cache->block_start_index, cache->block_start_index + size1, cache->block_cursor, size1);
1723
1724                                 if (idx1 + size1 > cache_size) {
1725                                         size2 = idx1 + size1 - cache_size;
1726                                         size1 -= size2;
1727                                         filelist_file_cache_block_release(filelist, size2, idx2);
1728                                 }
1729                                 filelist_file_cache_block_release(filelist, size1, idx1);
1730
1731                                 cache->block_cursor = (idx1 + size1 + size2) % cache_size;
1732                                 cache->block_start_index = start_index;
1733                         }
1734                         if (end_index < cache->block_end_index) {
1735                                 int size1 = cache->block_end_index - end_index;
1736                                 int size2 = 0;
1737                                 int idx1, idx2 = 0;
1738
1739 //                              printf("\tcache releasing: [%d:%d] (%d)\n", cache->block_end_index - size1, cache->block_end_index, cache->block_cursor);
1740
1741                                 idx1 = (cache->block_cursor + end_index - cache->block_start_index) % cache_size;
1742                                 if (idx1 + size1 > cache_size) {
1743                                         size2 = idx1 + size1 - cache_size;
1744                                         size1 -= size2;
1745                                         filelist_file_cache_block_release(filelist, size2, idx2);
1746                                 }
1747                                 filelist_file_cache_block_release(filelist, size1, idx1);
1748
1749                                 cache->block_end_index = end_index;
1750                         }
1751
1752 //                      printf("\tcache cleaned up...\n");
1753
1754                         if (start_index < cache->block_start_index) {
1755                                 /* Add (request) needed entries before already cached ones. */
1756                                 /* Note: We need some index black magic to wrap around (cycle) inside our cache_size array... */
1757                                 int size1 = cache->block_start_index - start_index;
1758                                 int size2 = 0;
1759                                 int idx1, idx2;
1760
1761                                 if (size1 > cache->block_cursor) {
1762                                         size2 = size1;
1763                                         size1 -= cache->block_cursor;
1764                                         size2 -= size1;
1765                                         idx2 = 0;
1766                                         idx1 = cache_size - size1;
1767                                 }
1768                                 else {
1769                                         idx1 = cache->block_cursor - size1;
1770                                 }
1771
1772                                 if (size2) {
1773                                         if (!filelist_file_cache_block_create(filelist, start_index + size1, size2, idx2)) {
1774                                                 return false;
1775                                         }
1776                                 }
1777                                 if (!filelist_file_cache_block_create(filelist, start_index, size1, idx1)) {
1778                                         return false;
1779                                 }
1780
1781                                 cache->block_cursor = idx1;
1782                                 cache->block_start_index = start_index;
1783                         }
1784 //                      printf("\tstart-extended...\n");
1785                         if (end_index > cache->block_end_index) {
1786                                 /* Add (request) needed entries after already cached ones. */
1787                                 /* Note: We need some index black magic to wrap around (cycle) inside our cache_size array... */
1788                                 int size1 = end_index - cache->block_end_index;
1789                                 int size2 = 0;
1790                                 int idx1, idx2;
1791
1792                                 idx1 = (cache->block_cursor + end_index - cache->block_start_index - size1) % cache_size;
1793                                 if ((idx1 + size1) > cache_size) {
1794                                         size2 = size1;
1795                                         size1 = cache_size - idx1;
1796                                         size2 -= size1;
1797                                         idx2 = 0;
1798                                 }
1799
1800                                 if (size2) {
1801                                         if (!filelist_file_cache_block_create(filelist, end_index - size2, size2, idx2)) {
1802                                                 return false;
1803                                         }
1804                                 }
1805                                 if (!filelist_file_cache_block_create(filelist, end_index - size1 - size2, size1, idx1)) {
1806                                         return false;
1807                                 }
1808
1809                                 cache->block_end_index = end_index;
1810                         }
1811
1812 //                      printf("\tend-extended...\n");
1813                 }
1814         }
1815         else if ((cache->block_center_index != index) && (cache->flags & FLC_PREVIEWS_ACTIVE)) {
1816                 /* We try to always preview visible entries first, so 'restart' preview background task. */
1817                 filelist_cache_previews_update(filelist);
1818                 filelist_cache_previews_clear(cache);
1819         }
1820
1821 //      printf("Re-queueing previews...\n");
1822
1823         /* Note we try to preview first images around given index - i.e. assumed visible ones. */
1824         if (cache->flags & FLC_PREVIEWS_ACTIVE) {
1825                 for (i = 0; ((index + i) < end_index) || ((index - i) >= start_index); i++) {
1826                         if ((index - i) >= start_index) {
1827                                 const int idx = (cache->block_cursor + (index - start_index) - i) % cache_size;
1828                                 filelist_cache_previews_push(filelist, cache->block_entries[idx], index - i);
1829                         }
1830                         if ((index + i) < end_index) {
1831                                 const int idx = (cache->block_cursor + (index - start_index) + i) % cache_size;
1832                                 filelist_cache_previews_push(filelist, cache->block_entries[idx], index + i);
1833                         }
1834                 }
1835         }
1836
1837         cache->block_center_index = index;
1838
1839 //      printf("%s Finished!\n", __func__);
1840
1841         return true;
1842 }
1843
1844 void filelist_cache_previews_set(FileList *filelist, const bool use_previews)
1845 {
1846         FileListEntryCache *cache = &filelist->filelist_cache;
1847
1848         if (use_previews == ((cache->flags & FLC_PREVIEWS_ACTIVE) != 0)) {
1849                 return;
1850         }
1851         /* Do not start preview work while listing, gives nasty flickering! */
1852         else if (use_previews && (filelist->flags & FL_IS_READY)) {
1853                 cache->flags |= FLC_PREVIEWS_ACTIVE;
1854
1855                 BLI_assert((cache->previews_pool == NULL) && (cache->previews_done == NULL));
1856
1857 //              printf("%s: Init Previews...\n", __func__);
1858
1859                 /* No need to populate preview queue here, filelist_file_cache_block() handles this. */
1860         }
1861         else {
1862 //              printf("%s: Clear Previews...\n", __func__);
1863
1864                 filelist_cache_previews_free(cache);
1865         }
1866 }
1867
1868 bool filelist_cache_previews_update(FileList *filelist)
1869 {
1870         FileListEntryCache *cache = &filelist->filelist_cache;
1871         TaskPool *pool = cache->previews_pool;
1872         bool changed = false;
1873
1874         if (!pool) {
1875                 return changed;
1876         }
1877
1878 //      printf("%s: Update Previews...\n", __func__);
1879
1880         while (!BLI_thread_queue_is_empty(cache->previews_done)) {
1881                 FileListEntryPreview *preview = BLI_thread_queue_pop(cache->previews_done);
1882                 FileDirEntry *entry;
1883
1884                 /* Paranoid (should never happen currently since we consume this queue from a single thread), but... */
1885                 if (!preview) {
1886                         continue;
1887                 }
1888                 /* entry might have been removed from cache in the mean time, we do not want to cache it again here. */
1889                 entry = filelist_file_ex(filelist, preview->index, false);
1890
1891 //              printf("%s: %d - %s - %p\n", __func__, preview->index, preview->path, preview->img);
1892
1893                 if (preview->img) {
1894                         /* Due to asynchronous process, a preview for a given image may be generated several times, i.e.
1895                          * entry->image may already be set at this point. */
1896                         if (entry && !entry->image) {
1897                                 entry->image = preview->img;
1898                                 changed = true;
1899                         }
1900                         else {
1901                                 IMB_freeImBuf(preview->img);
1902                         }
1903                 }
1904                 else if (entry) {
1905                         /* We want to avoid re-processing this entry continuously!
1906                          * Note that, since entries only live in cache, preview will be retried quite often anyway. */
1907                         entry->flags |= FILE_ENTRY_INVALID_PREVIEW;
1908                 }
1909
1910                 MEM_freeN(preview);
1911         }
1912
1913         return changed;
1914 }
1915
1916 bool filelist_cache_previews_running(FileList *filelist)
1917 {
1918         FileListEntryCache *cache = &filelist->filelist_cache;
1919
1920         return (cache->previews_pool != NULL);
1921 }
1922
1923 /* would recognize .blend as well */
1924 static bool file_is_blend_backup(const char *str)
1925 {
1926         const size_t a = strlen(str);
1927         size_t b = 7;
1928         bool retval = 0;
1929
1930         if (a == 0 || b >= a) {
1931                 /* pass */
1932         }
1933         else {
1934                 const char *loc;
1935                 
1936                 if (a > b + 1)
1937                         b++;
1938                 
1939                 /* allow .blend1 .blend2 .blend32 */
1940                 loc = BLI_strcasestr(str + a - b, ".blend");
1941                 
1942                 if (loc)
1943                         retval = 1;
1944         }
1945         
1946         return (retval);
1947 }
1948
1949 /* TODO: Maybe we should move this to BLI? On the other hand, it's using defines from spacefile area, so not sure... */
1950 int ED_path_extension_type(const char *path)
1951 {
1952         if (BLO_has_bfile_extension(path)) {
1953                 return FILE_TYPE_BLENDER;
1954         }
1955         else if (file_is_blend_backup(path)) {
1956                 return FILE_TYPE_BLENDER_BACKUP;
1957         }
1958         else if (BLI_testextensie(path, ".app")) {
1959                 return FILE_TYPE_APPLICATIONBUNDLE;
1960         }
1961         else if (BLI_testextensie(path, ".py")) {
1962                 return FILE_TYPE_PYSCRIPT;
1963         }
1964         else if (BLI_testextensie_n(path, ".txt", ".glsl", ".osl", ".data", NULL)) {
1965                 return FILE_TYPE_TEXT;
1966         }
1967         else if (BLI_testextensie_n(path, ".ttf", ".ttc", ".pfb", ".otf", ".otc", NULL)) {
1968                 return FILE_TYPE_FTFONT;
1969         }
1970         else if (BLI_testextensie(path, ".btx")) {
1971                 return FILE_TYPE_BTX;
1972         }
1973         else if (BLI_testextensie(path, ".dae")) {
1974                 return FILE_TYPE_COLLADA;
1975         }
1976         else if (BLI_testextensie(path, ".abc")) {
1977                 return FILE_TYPE_ALEMBIC;
1978         }
1979         else if (BLI_testextensie_array(path, imb_ext_image) ||
1980                  (G.have_quicktime && BLI_testextensie_array(path, imb_ext_image_qt)))
1981         {
1982                 return FILE_TYPE_IMAGE;
1983         }
1984         else if (BLI_testextensie(path, ".ogg")) {
1985                 if (IMB_isanim(path)) {
1986                         return FILE_TYPE_MOVIE;
1987                 }
1988                 else {
1989                         return FILE_TYPE_SOUND;
1990                 }
1991         }
1992         else if (BLI_testextensie_array(path, imb_ext_movie)) {
1993                 return FILE_TYPE_MOVIE;
1994         }
1995         else if (BLI_testextensie_array(path, imb_ext_audio)) {
1996                 return FILE_TYPE_SOUND;
1997         }
1998         return 0;
1999 }
2000
2001 static int file_extension_type(const char *dir, const char *relpath)
2002 {
2003         char path[FILE_MAX];
2004         BLI_join_dirfile(path, sizeof(path), dir, relpath);
2005         return ED_path_extension_type(path);
2006 }
2007
2008 int ED_file_extension_icon(const char *path)
2009 {
2010         const int type = ED_path_extension_type(path);
2011         
2012         switch (type) {
2013                 case FILE_TYPE_BLENDER:
2014                         return ICON_FILE_BLEND;
2015                 case FILE_TYPE_BLENDER_BACKUP:
2016                         return ICON_FILE_BACKUP;
2017                 case FILE_TYPE_IMAGE:
2018                         return ICON_FILE_IMAGE;
2019                 case FILE_TYPE_MOVIE:
2020                         return ICON_FILE_MOVIE;
2021                 case FILE_TYPE_PYSCRIPT:
2022                         return ICON_FILE_SCRIPT;
2023                 case FILE_TYPE_SOUND:
2024                         return ICON_FILE_SOUND;
2025                 case FILE_TYPE_FTFONT:
2026                         return ICON_FILE_FONT;
2027                 case FILE_TYPE_BTX:
2028                         return ICON_FILE_BLANK;
2029                 case FILE_TYPE_COLLADA:
2030                         return ICON_FILE_BLANK;
2031                 case FILE_TYPE_ALEMBIC:
2032                         return ICON_FILE_BLANK;
2033                 case FILE_TYPE_TEXT:
2034                         return ICON_FILE_TEXT;
2035                 default:
2036                         return ICON_FILE_BLANK;
2037         }
2038 }
2039
2040 int filelist_empty(struct FileList *filelist)
2041 {
2042         return (filelist->filelist.nbr_entries == 0);
2043 }
2044
2045 unsigned int filelist_entry_select_set(
2046         const FileList *filelist, const FileDirEntry *entry, FileSelType select, unsigned int flag, FileCheckType check)
2047 {
2048         /* Default NULL pointer if not found is fine here! */
2049         void **es_p = BLI_ghash_lookup_p(filelist->selection_state, entry->uuid);
2050         unsigned int entry_flag = es_p ? GET_UINT_FROM_POINTER(*es_p) : 0;
2051         const unsigned int org_entry_flag = entry_flag;
2052
2053         BLI_assert(entry);
2054         BLI_assert(ELEM(check, CHECK_DIRS, CHECK_FILES, CHECK_ALL));
2055
2056         if (((check == CHECK_ALL)) ||
2057             ((check == CHECK_DIRS) && (entry->typeflag & FILE_TYPE_DIR)) ||
2058             ((check == CHECK_FILES) && !(entry->typeflag & FILE_TYPE_DIR)))
2059         {
2060                 switch (select) {
2061                         case FILE_SEL_REMOVE:
2062                                 entry_flag &= ~flag;
2063                                 break;
2064                         case FILE_SEL_ADD:
2065                                 entry_flag |= flag;
2066                                 break;
2067                         case FILE_SEL_TOGGLE:
2068                                 entry_flag ^= flag;
2069                                 break;
2070                 }
2071         }
2072
2073         if (entry_flag != org_entry_flag) {
2074                 if (es_p) {
2075                         if (entry_flag) {
2076                                 *es_p = SET_UINT_IN_POINTER(entry_flag);
2077                         }
2078                         else {
2079                                 BLI_ghash_remove(filelist->selection_state, entry->uuid, MEM_freeN, NULL);
2080                         }
2081                 }
2082                 else if (entry_flag) {
2083                         void *key = MEM_mallocN(sizeof(entry->uuid), __func__);
2084                         memcpy(key, entry->uuid, sizeof(entry->uuid));
2085                         BLI_ghash_insert(filelist->selection_state, key, SET_UINT_IN_POINTER(entry_flag));
2086                 }
2087         }
2088
2089         return entry_flag;
2090 }
2091
2092 void filelist_entry_select_index_set(FileList *filelist, const int index, FileSelType select, unsigned int flag, FileCheckType check)
2093 {
2094         FileDirEntry *entry = filelist_file(filelist, index);
2095
2096         if (entry) {
2097                 filelist_entry_select_set(filelist, entry, select, flag, check);
2098         }
2099 }
2100
2101 void filelist_entries_select_index_range_set(
2102         FileList *filelist, FileSelection *sel, FileSelType select, unsigned int flag, FileCheckType check)
2103 {
2104         /* select all valid files between first and last indicated */
2105         if ((sel->first >= 0) && (sel->first < filelist->filelist.nbr_entries_filtered) &&
2106             (sel->last >= 0) && (sel->last < filelist->filelist.nbr_entries_filtered))
2107         {
2108                 int current_file;
2109                 for (current_file = sel->first; current_file <= sel->last; current_file++) {
2110                         filelist_entry_select_index_set(filelist, current_file, select, flag, check);
2111                 }
2112         }
2113 }
2114
2115 unsigned int filelist_entry_select_get(FileList *filelist, FileDirEntry *entry, FileCheckType check)
2116 {
2117         BLI_assert(entry);
2118         BLI_assert(ELEM(check, CHECK_DIRS, CHECK_FILES, CHECK_ALL));
2119
2120         if (((check == CHECK_ALL)) ||
2121             ((check == CHECK_DIRS) && (entry->typeflag & FILE_TYPE_DIR)) ||
2122             ((check == CHECK_FILES) && !(entry->typeflag & FILE_TYPE_DIR)))
2123         {
2124                 /* Default NULL pointer if not found is fine here! */
2125                 return GET_UINT_FROM_POINTER(BLI_ghash_lookup(filelist->selection_state, entry->uuid));
2126         }
2127
2128         return 0;
2129 }
2130
2131 unsigned int filelist_entry_select_index_get(FileList *filelist, const int index, FileCheckType check)
2132 {
2133         FileDirEntry *entry = filelist_file(filelist, index);
2134
2135         if (entry) {
2136                 return filelist_entry_select_get(filelist, entry, check);
2137         }
2138
2139         return 0;
2140 }
2141
2142 /* WARNING! dir must be FILE_MAX_LIBEXTRA long! */
2143 bool filelist_islibrary(struct FileList *filelist, char *dir, char **group)
2144 {
2145         return BLO_library_path_explode(filelist->filelist.root, dir, group, NULL);
2146 }
2147
2148 static int groupname_to_code(const char *group)
2149 {
2150         char buf[BLO_GROUP_MAX];
2151         char *lslash;
2152
2153         BLI_assert(group);
2154
2155         BLI_strncpy(buf, group, sizeof(buf));
2156         lslash = (char *)BLI_last_slash(buf);
2157         if (lslash)
2158                 lslash[0] = '\0';
2159
2160         return buf[0] ? BKE_idcode_from_name(buf) : 0;
2161 }
2162
2163 static unsigned int groupname_to_filter_id(const char *group)
2164 {
2165         int id_code = groupname_to_code(group);
2166
2167         return BKE_idcode_to_idfilter(id_code);
2168 }
2169
2170 /**
2171  * From here, we are in 'Job Context', i.e. have to be careful about sharing stuff between background working thread
2172  * and main one (used by UI among other things).
2173  */
2174 typedef struct TodoDir {
2175         int level;
2176         char *dir;
2177 } TodoDir;
2178
2179 static int filelist_readjob_list_dir(
2180         const char *root, ListBase *entries, const char *filter_glob,
2181         const bool do_lib, const char *main_name, const bool skip_currpar)
2182 {
2183         struct direntry *files;
2184         int nbr_files, nbr_entries = 0;
2185
2186         nbr_files = BLI_filelist_dir_contents(root, &files);
2187         if (files) {
2188                 int i = nbr_files;
2189                 while (i--) {
2190                         FileListInternEntry *entry;
2191
2192                         if (skip_currpar && FILENAME_IS_CURRPAR(files[i].relname)) {
2193                                 continue;
2194                         }
2195
2196                         entry = MEM_callocN(sizeof(*entry), __func__);
2197                         entry->relpath = MEM_dupallocN(files[i].relname);
2198                         entry->st = files[i].s;
2199
2200                         /* Set file type. */
2201                         if (S_ISDIR(files[i].s.st_mode)) {
2202                                 entry->typeflag = FILE_TYPE_DIR;
2203                         }
2204                         else if (do_lib && BLO_has_bfile_extension(entry->relpath)) {
2205                                 /* If we are considering .blend files as libs, promote them to directory status. */
2206                                 char name[FILE_MAX];
2207
2208                                 entry->typeflag = FILE_TYPE_BLENDER;
2209
2210                                 BLI_join_dirfile(name, sizeof(name), root, entry->relpath);
2211
2212                                 /* prevent current file being used as acceptable dir */
2213                                 if (BLI_path_cmp(main_name, name) != 0) {
2214                                         entry->typeflag |= FILE_TYPE_DIR;
2215                                 }
2216                         }
2217                         /* Otherwise, do not check extensions for directories! */
2218                         else if (!(entry->typeflag & FILE_TYPE_DIR)) {
2219                                 entry->typeflag = file_extension_type(root, entry->relpath);
2220                                 if (filter_glob[0] && BLI_testextensie_glob(entry->relpath, filter_glob)) {
2221                                         entry->typeflag |= FILE_TYPE_OPERATOR;
2222                                 }
2223                         }
2224
2225                         BLI_addtail(entries, entry);
2226                         nbr_entries++;
2227                 }
2228                 BLI_filelist_free(files, nbr_files);
2229         }
2230         return nbr_entries;
2231 }
2232
2233 static int filelist_readjob_list_lib(const char *root, ListBase *entries, const bool skip_currpar)
2234 {
2235         FileListInternEntry *entry;
2236         LinkNode *ln, *names;
2237         int i, nnames, idcode = 0, nbr_entries = 0;
2238         char dir[FILE_MAX_LIBEXTRA], *group;
2239         bool ok;
2240
2241         struct BlendHandle *libfiledata = NULL;
2242
2243         /* name test */
2244         ok = BLO_library_path_explode(root, dir, &group, NULL);
2245         if (!ok) {
2246                 return nbr_entries;
2247         }
2248
2249         /* there we go */
2250         libfiledata = BLO_blendhandle_from_file(dir, NULL);
2251         if (libfiledata == NULL) {
2252                 return nbr_entries;
2253         }
2254
2255         /* memory for strings is passed into filelist[i].entry->relpath and freed in filelist_entry_free. */
2256         if (group) {
2257                 idcode = groupname_to_code(group);
2258                 names = BLO_blendhandle_get_datablock_names(libfiledata, idcode, &nnames);
2259         }
2260         else {
2261                 names = BLO_blendhandle_get_linkable_groups(libfiledata);
2262                 nnames = BLI_linklist_count(names);
2263         }
2264
2265         BLO_blendhandle_close(libfiledata);
2266
2267         if (!skip_currpar) {
2268                 entry = MEM_callocN(sizeof(*entry), __func__);
2269                 entry->relpath = BLI_strdup(FILENAME_PARENT);
2270                 entry->typeflag |= (FILE_TYPE_BLENDERLIB | FILE_TYPE_DIR);
2271                 BLI_addtail(entries, entry);
2272                 nbr_entries++;
2273         }
2274
2275         for (i = 0, ln = names; i < nnames; i++, ln = ln->next) {
2276                 const char *blockname = ln->link;
2277
2278                 entry = MEM_callocN(sizeof(*entry), __func__);
2279                 entry->relpath = BLI_strdup(blockname);
2280                 entry->typeflag |= FILE_TYPE_BLENDERLIB;
2281                 if (!(group && idcode)) {
2282                         entry->typeflag |= FILE_TYPE_DIR;
2283                         entry->blentype = groupname_to_code(blockname);
2284                 }
2285                 else {
2286                         entry->blentype = idcode;
2287                 }
2288                 BLI_addtail(entries, entry);
2289                 nbr_entries++;
2290         }
2291
2292         BLI_linklist_free(names, free);
2293
2294         return nbr_entries;
2295 }
2296
2297 #if 0
2298 /* Kept for reference here, in case we want to add back that feature later. We do not need it currently. */
2299 /* Code ***NOT*** updated for job stuff! */
2300 static void filelist_readjob_main_rec(struct FileList *filelist)
2301 {
2302         ID *id;
2303         FileDirEntry *files, *firstlib = NULL;
2304         ListBase *lb;
2305         int a, fake, idcode, ok, totlib, totbl;
2306         
2307         // filelist->type = FILE_MAIN; // XXX TODO: add modes to filebrowser
2308
2309         BLI_assert(filelist->filelist.entries == NULL);
2310
2311         if (filelist->filelist.root[0] == '/') filelist->filelist.root[0] = '\0';
2312
2313         if (filelist->filelist.root[0]) {
2314                 idcode = groupname_to_code(filelist->filelist.root);
2315                 if (idcode == 0) filelist->filelist.root[0] = '\0';
2316         }
2317
2318         if (filelist->dir[0] == 0) {
2319                 /* make directories */
2320 #ifdef WITH_FREESTYLE
2321                 filelist->filelist.nbr_entries = 24;
2322 #else
2323                 filelist->filelist.nbr_entries = 23;
2324 #endif
2325                 filelist_resize(filelist, filelist->filelist.nbr_entries);
2326
2327                 for (a = 0; a < filelist->filelist.nbr_entries; a++) {
2328                         filelist->filelist.entries[a].typeflag |= FILE_TYPE_DIR;
2329                 }
2330
2331                 filelist->filelist.entries[0].entry->relpath = BLI_strdup(FILENAME_PARENT);
2332                 filelist->filelist.entries[1].entry->relpath = BLI_strdup("Scene");
2333                 filelist->filelist.entries[2].entry->relpath = BLI_strdup("Object");
2334                 filelist->filelist.entries[3].entry->relpath = BLI_strdup("Mesh");
2335                 filelist->filelist.entries[4].entry->relpath = BLI_strdup("Curve");
2336                 filelist->filelist.entries[5].entry->relpath = BLI_strdup("Metaball");
2337                 filelist->filelist.entries[6].entry->relpath = BLI_strdup("Material");
2338                 filelist->filelist.entries[7].entry->relpath = BLI_strdup("Texture");
2339                 filelist->filelist.entries[8].entry->relpath = BLI_strdup("Image");
2340                 filelist->filelist.entries[9].entry->relpath = BLI_strdup("Ika");
2341                 filelist->filelist.entries[10].entry->relpath = BLI_strdup("Wave");
2342                 filelist->filelist.entries[11].entry->relpath = BLI_strdup("Lattice");
2343                 filelist->filelist.entries[12].entry->relpath = BLI_strdup("Lamp");
2344                 filelist->filelist.entries[13].entry->relpath = BLI_strdup("Camera");
2345                 filelist->filelist.entries[14].entry->relpath = BLI_strdup("Ipo");
2346                 filelist->filelist.entries[15].entry->relpath = BLI_strdup("World");
2347                 filelist->filelist.entries[16].entry->relpath = BLI_strdup("Screen");
2348                 filelist->filelist.entries[17].entry->relpath = BLI_strdup("VFont");
2349                 filelist->filelist.entries[18].entry->relpath = BLI_strdup("Text");
2350                 filelist->filelist.entries[19].entry->relpath = BLI_strdup("Armature");
2351                 filelist->filelist.entries[20].entry->relpath = BLI_strdup("Action");
2352                 filelist->filelist.entries[21].entry->relpath = BLI_strdup("NodeTree");
2353                 filelist->filelist.entries[22].entry->relpath = BLI_strdup("Speaker");
2354 #ifdef WITH_FREESTYLE
2355                 filelist->filelist.entries[23].entry->relpath = BLI_strdup("FreestyleLineStyle");
2356 #endif
2357         }
2358         else {
2359                 /* make files */
2360                 idcode = groupname_to_code(filelist->filelist.root);
2361
2362                 lb = which_libbase(G.main, idcode);
2363                 if (lb == NULL) return;
2364
2365                 filelist->filelist.nbr_entries = 0;
2366                 for (id = lb->first; id; id = id->next) {
2367                         if (!(filelist->filter_data.flags & FLF_HIDE_DOT) || id->name[2] != '.') {
2368                                 filelist->filelist.nbr_entries++;
2369                         }
2370                 }
2371
2372                 /* XXX TODO: if databrowse F4 or append/link filelist->flags & FLF_HIDE_PARENT has to be set */
2373                 if (!(filelist->filter_data.flags & FLF_HIDE_PARENT))
2374                         filelist->filelist.nbr_entries++;
2375
2376                 if (filelist->filelist.nbr_entries > 0) {
2377                         filelist_resize(filelist, filelist->filelist.nbr_entries);
2378                 }
2379
2380                 files = filelist->filelist.entries;
2381                 
2382                 if (!(filelist->filter_data.flags & FLF_HIDE_PARENT)) {
2383                         files->entry->relpath = BLI_strdup(FILENAME_PARENT);
2384                         files->typeflag |= FILE_TYPE_DIR;
2385
2386                         files++;
2387                 }
2388
2389                 totlib = totbl = 0;
2390                 for (id = lb->first; id; id = id->next) {
2391                         ok = 1;
2392                         if (ok) {
2393                                 if (!(filelist->filter_data.flags & FLF_HIDE_DOT) || id->name[2] != '.') {
2394                                         if (id->lib == NULL) {
2395                                                 files->entry->relpath = BLI_strdup(id->name + 2);
2396                                         }
2397                                         else {
2398                                                 char relname[FILE_MAX + (MAX_ID_NAME - 2) + 3];
2399                                                 BLI_snprintf(relname, sizeof(relname), "%s | %s", id->lib->name, id->name + 2);
2400                                                 files->entry->relpath = BLI_strdup(relname);
2401                                         }
2402 //                                      files->type |= S_IFREG;
2403 #if 0               /* XXX TODO show the selection status of the objects */
2404                                         if (!filelist->has_func) { /* F4 DATA BROWSE */
2405                                                 if (idcode == ID_OB) {
2406                                                         if ( ((Object *)id)->flag & SELECT) files->entry->selflag |= FILE_SEL_SELECTED;
2407                                                 }
2408                                                 else if (idcode == ID_SCE) {
2409                                                         if ( ((Scene *)id)->r.scemode & R_BG_RENDER) files->entry->selflag |= FILE_SEL_SELECTED;
2410                                                 }
2411                                         }
2412 #endif
2413 //                                      files->entry->nr = totbl + 1;
2414                                         files->entry->poin = id;
2415                                         fake = id->flag & LIB_FAKEUSER;
2416                                         if (idcode == ID_MA || idcode == ID_TE || idcode == ID_LA || idcode == ID_WO || idcode == ID_IM) {
2417                                                 files->typeflag |= FILE_TYPE_IMAGE;
2418                                         }
2419 //                                      if      (id->lib && fake) BLI_snprintf(files->extra, sizeof(files->entry->extra), "LF %d",    id->us);
2420 //                                      else if (id->lib)         BLI_snprintf(files->extra, sizeof(files->entry->extra), "L    %d",  id->us);
2421 //                                      else if (fake)            BLI_snprintf(files->extra, sizeof(files->entry->extra), "F    %d",  id->us);
2422 //                                      else                      BLI_snprintf(files->extra, sizeof(files->entry->extra), "      %d", id->us);
2423
2424                                         if (id->lib) {
2425                                                 if (totlib == 0) firstlib = files;
2426                                                 totlib++;
2427                                         }
2428
2429                                         files++;
2430                                 }
2431                                 totbl++;
2432                         }
2433                 }
2434
2435                 /* only qsort of library blocks */
2436                 if (totlib > 1) {
2437                         qsort(firstlib, totlib, sizeof(*files), compare_name);
2438                 }
2439         }
2440 }
2441 #endif
2442
2443 static void filelist_readjob_do(
2444         const bool do_lib,
2445         FileList *filelist, const char *main_name, short *stop, short *do_update, float *progress, ThreadMutex *lock)
2446 {
2447         ListBase entries = {0};
2448         BLI_Stack *todo_dirs;
2449         TodoDir *td_dir;
2450         char dir[FILE_MAX_LIBEXTRA];
2451         char filter_glob[64];  /* TODO should be define! */
2452         const char *root = filelist->filelist.root;
2453         const int max_recursion = filelist->max_recursion;
2454         int nbr_done_dirs = 0, nbr_todo_dirs = 1;
2455
2456 //      BLI_assert(filelist->filtered == NULL);
2457         BLI_assert(BLI_listbase_is_empty(&filelist->filelist.entries) && (filelist->filelist.nbr_entries == 0));
2458
2459         todo_dirs = BLI_stack_new(sizeof(*td_dir), __func__);
2460         td_dir = BLI_stack_push_r(todo_dirs);
2461         td_dir->level = 1;
2462
2463         BLI_strncpy(dir, filelist->filelist.root, sizeof(dir));
2464         BLI_strncpy(filter_glob, filelist->filter_data.filter_glob, sizeof(filter_glob));
2465
2466         BLI_cleanup_dir(main_name, dir);
2467         td_dir->dir = BLI_strdup(dir);
2468
2469         while (!BLI_stack_is_empty(todo_dirs) && !(*stop)) {
2470                 FileListInternEntry *entry;
2471                 int nbr_entries = 0;
2472                 bool is_lib = do_lib;
2473
2474                 char *subdir;
2475                 char rel_subdir[FILE_MAX_LIBEXTRA];
2476                 int recursion_level;
2477                 bool skip_currpar;
2478
2479                 td_dir = BLI_stack_peek(todo_dirs);
2480                 subdir = td_dir->dir;
2481                 recursion_level = td_dir->level;
2482                 skip_currpar = (recursion_level > 1);
2483
2484                 BLI_stack_discard(todo_dirs);
2485
2486                 /* ARRRG! We have to be very careful *not to use* common BLI_path_util helpers over entry->relpath itself
2487                  * (nor any path containing it), since it may actually be a datablock name inside .blend file,
2488                  * which can have slashes and backslashes! See T46827.
2489                  * Note that in the end, this means we 'cache' valid relative subdir once here, this is actually better. */
2490                 BLI_strncpy(rel_subdir, subdir, sizeof(rel_subdir));
2491                 BLI_cleanup_dir(root, rel_subdir);
2492                 BLI_path_rel(rel_subdir, root);
2493
2494                 if (do_lib) {
2495                         nbr_entries = filelist_readjob_list_lib(subdir, &entries, skip_currpar);
2496                 }
2497                 if (!nbr_entries) {
2498                         is_lib = false;
2499                         nbr_entries = filelist_readjob_list_dir(subdir, &entries, filter_glob, do_lib, main_name, skip_currpar);
2500                 }
2501
2502                 for (entry = entries.first; entry; entry = entry->next) {
2503                         BLI_join_dirfile(dir, sizeof(dir), rel_subdir, entry->relpath);
2504
2505                         /* Generate our entry uuid. Abusing uuid as an uint32, shall be more than enough here,
2506                          * things would crash way before we overflow that counter!
2507                          * Using an atomic operation to avoid having to lock thread...
2508                          * Note that we do not really need this here currently, since there is a single listing thread, but better
2509                          * remain consistent about threading! */
2510                         *((uint32_t *)entry->uuid) = atomic_add_uint32((uint32_t *)filelist->filelist_intern.curr_uuid, 1);
2511
2512                         /* Only thing we change in direntry here, so we need to free it first. */
2513                         MEM_freeN(entry->relpath);
2514                         entry->relpath = BLI_strdup(dir + 2);  /* + 2 to remove '//' added by BLI_path_rel to rel_subdir */
2515                         entry->name = BLI_strdup(fileentry_uiname(root, entry->relpath, entry->typeflag, dir));
2516
2517                         /* Here we decide whether current filedirentry is to be listed too, or not. */
2518                         if (max_recursion && (is_lib || (recursion_level <= max_recursion))) {
2519                                 if (((entry->typeflag & FILE_TYPE_DIR) == 0) || FILENAME_IS_CURRPAR(entry->relpath)) {
2520                                         /* Skip... */
2521                                 }
2522                                 else if (!is_lib && (recursion_level >= max_recursion) &&
2523                                          ((entry->typeflag & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) == 0))
2524                                 {
2525                                         /* Do not recurse in real directories in this case, only in .blend libs. */
2526                                 }
2527                                 else {
2528                                         /* We have a directory we want to list, add it to todo list! */
2529                                         BLI_join_dirfile(dir, sizeof(dir), root, entry->relpath);
2530                                         BLI_cleanup_dir(main_name, dir);
2531                                         td_dir = BLI_stack_push_r(todo_dirs);
2532                                         td_dir->level = recursion_level + 1;
2533                                         td_dir->dir = BLI_strdup(dir);
2534                                         nbr_todo_dirs++;
2535                                 }
2536                         }
2537                 }
2538
2539                 if (nbr_entries) {
2540                         BLI_mutex_lock(lock);
2541
2542                         *do_update = true;
2543
2544                         BLI_movelisttolist(&filelist->filelist.entries, &entries);
2545                         filelist->filelist.nbr_entries += nbr_entries;
2546
2547                         BLI_mutex_unlock(lock);
2548                 }
2549
2550                 nbr_done_dirs++;
2551                 *progress = (float)nbr_done_dirs / (float)nbr_todo_dirs;
2552                 MEM_freeN(subdir);
2553         }
2554
2555         /* If we were interrupted by stop, stack may not be empty and we need to free pending dir paths. */
2556         while (!BLI_stack_is_empty(todo_dirs)) {
2557                 td_dir = BLI_stack_peek(todo_dirs);
2558                 MEM_freeN(td_dir->dir);
2559                 BLI_stack_discard(todo_dirs);
2560         }
2561         BLI_stack_free(todo_dirs);
2562 }
2563
2564 static void filelist_readjob_dir(
2565         FileList *filelist, const char *main_name, short *stop, short *do_update, float *progress, ThreadMutex *lock)
2566 {
2567         filelist_readjob_do(false, filelist, main_name, stop, do_update, progress, lock);
2568 }
2569
2570 static void filelist_readjob_lib(
2571         FileList *filelist, const char *main_name, short *stop, short *do_update, float *progress, ThreadMutex *lock)
2572 {
2573         filelist_readjob_do(true, filelist, main_name, stop, do_update, progress, lock);
2574 }
2575
2576 static void filelist_readjob_main(
2577         FileList *filelist, const char *main_name, short *stop, short *do_update, float *progress, ThreadMutex *lock)
2578 {
2579         /* TODO! */
2580         filelist_readjob_dir(filelist, main_name, stop, do_update, progress, lock);
2581 }
2582
2583
2584 typedef struct FileListReadJob {
2585         ThreadMutex lock;
2586         char main_name[FILE_MAX];
2587         struct FileList *filelist;
2588         struct FileList *tmp_filelist;  /* XXX We may use a simpler struct here... just a linked list and root path? */
2589 } FileListReadJob;
2590
2591 static void filelist_readjob_startjob(void *flrjv, short *stop, short *do_update, float *progress)
2592 {
2593         FileListReadJob *flrj = flrjv;
2594
2595 //      printf("START filelist reading (%d files, main thread: %d)\n",
2596 //             flrj->filelist->filelist.nbr_entries, BLI_thread_is_main());
2597
2598         BLI_mutex_lock(&flrj->lock);
2599
2600         BLI_assert((flrj->tmp_filelist == NULL) && flrj->filelist);
2601
2602         flrj->tmp_filelist = MEM_dupallocN(flrj->filelist);
2603
2604         BLI_listbase_clear(&flrj->tmp_filelist->filelist.entries);
2605         flrj->tmp_filelist->filelist.nbr_entries = 0;
2606
2607         flrj->tmp_filelist->filelist_intern.filtered = NULL;
2608         BLI_listbase_clear(&flrj->tmp_filelist->filelist_intern.entries);
2609         memset(flrj->tmp_filelist->filelist_intern.curr_uuid, 0, sizeof(flrj->tmp_filelist->filelist_intern.curr_uuid));
2610
2611         flrj->tmp_filelist->libfiledata = NULL;
2612         memset(&flrj->tmp_filelist->filelist_cache, 0, sizeof(flrj->tmp_filelist->filelist_cache));
2613         flrj->tmp_filelist->selection_state = NULL;
2614
2615         BLI_mutex_unlock(&flrj->lock);
2616
2617         flrj->tmp_filelist->read_jobf(flrj->tmp_filelist, flrj->main_name, stop, do_update, progress, &flrj->lock);
2618 }
2619
2620 static void filelist_readjob_update(void *flrjv)
2621 {
2622         FileListReadJob *flrj = flrjv;
2623         FileListIntern *fl_intern = &flrj->filelist->filelist_intern;
2624         ListBase new_entries = {NULL};
2625         int nbr_entries, new_nbr_entries = 0;
2626
2627         BLI_movelisttolist(&new_entries, &fl_intern->entries);
2628         nbr_entries = flrj->filelist->filelist.nbr_entries;
2629
2630         BLI_mutex_lock(&flrj->lock);
2631
2632         if (flrj->tmp_filelist->filelist.nbr_entries) {
2633                 /* We just move everything out of 'thread context' into final list. */
2634                 new_nbr_entries = flrj->tmp_filelist->filelist.nbr_entries;
2635                 BLI_movelisttolist(&new_entries, &flrj->tmp_filelist->filelist.entries);
2636                 flrj->tmp_filelist->filelist.nbr_entries = 0;
2637         }
2638
2639         BLI_mutex_unlock(&flrj->lock);
2640
2641         if (new_nbr_entries) {
2642                 /* Do not clear selection cache, we can assume already 'selected' uuids are still valid! */
2643                 filelist_clear_ex(flrj->filelist, true, false);
2644
2645                 flrj->filelist->flags |= (FL_NEED_SORTING | FL_NEED_FILTERING);
2646         }
2647
2648         /* if no new_nbr_entries, this is NOP */
2649         BLI_movelisttolist(&fl_intern->entries, &new_entries);
2650         flrj->filelist->filelist.nbr_entries = nbr_entries + new_nbr_entries;
2651 }
2652
2653 static void filelist_readjob_endjob(void *flrjv)
2654 {
2655         FileListReadJob *flrj = flrjv;
2656
2657         /* In case there would be some dangling update... */
2658         filelist_readjob_update(flrjv);
2659
2660         flrj->filelist->flags &= ~FL_IS_PENDING;
2661         flrj->filelist->flags |= FL_IS_READY;
2662 }
2663
2664 static void filelist_readjob_free(void *flrjv)
2665 {
2666         FileListReadJob *flrj = flrjv;
2667
2668 //      printf("END filelist reading (%d files)\n", flrj->filelist->filelist.nbr_entries);
2669
2670         if (flrj->tmp_filelist) {
2671                 /* tmp_filelist shall never ever be filtered! */
2672                 BLI_assert(flrj->tmp_filelist->filelist.nbr_entries == 0);
2673                 BLI_assert(BLI_listbase_is_empty(&flrj->tmp_filelist->filelist.entries));
2674
2675                 filelist_freelib(flrj->tmp_filelist);
2676                 filelist_free(flrj->tmp_filelist);
2677                 MEM_freeN(flrj->tmp_filelist);
2678         }
2679
2680         BLI_mutex_end(&flrj->lock);
2681
2682         MEM_freeN(flrj);
2683 }
2684
2685 void filelist_readjob_start(FileList *filelist, const bContext *C)
2686 {
2687         wmJob *wm_job;
2688         FileListReadJob *flrj;
2689
2690         /* prepare job data */
2691         flrj = MEM_callocN(sizeof(*flrj), __func__);
2692         flrj->filelist = filelist;
2693         BLI_strncpy(flrj->main_name, G.main->name, sizeof(flrj->main_name));
2694
2695         filelist->flags &= ~(FL_FORCE_RESET | FL_IS_READY);
2696         filelist->flags |= FL_IS_PENDING;
2697
2698         BLI_mutex_init(&flrj->lock);
2699
2700         /* setup job */
2701         wm_job = WM_jobs_get(CTX_wm_manager(C), CTX_wm_window(C), CTX_wm_area(C), "Listing Dirs...",
2702                              WM_JOB_PROGRESS, WM_JOB_TYPE_FILESEL_READDIR);
2703         WM_jobs_customdata_set(wm_job, flrj, filelist_readjob_free);
2704         WM_jobs_timer(wm_job, 0.01, NC_SPACE | ND_SPACE_FILE_LIST, NC_SPACE | ND_SPACE_FILE_LIST);
2705         WM_jobs_callbacks(wm_job, filelist_readjob_startjob, NULL, filelist_readjob_update, filelist_readjob_endjob);
2706
2707         /* start the job */
2708         WM_jobs_start(CTX_wm_manager(C), wm_job);
2709 }
2710
2711 void filelist_readjob_stop(wmWindowManager *wm, ScrArea *sa)
2712 {
2713         WM_jobs_kill_type(wm, sa, WM_JOB_TYPE_FILESEL_READDIR);
2714 }
2715
2716 int filelist_readjob_running(wmWindowManager *wm, ScrArea *sa)
2717 {
2718         return WM_jobs_test(wm, sa, WM_JOB_TYPE_FILESEL_READDIR);
2719 }