Cycles: Extra tweaks to performance of header expansion
[blender.git] / intern / cycles / util / util_path.cpp
1 /*
2  * Copyright 2011-2013 Blender Foundation
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16
17 #include "util/util_debug.h"
18 #include "util/util_md5.h"
19 #include "util/util_path.h"
20 #include "util/util_string.h"
21
22 #include <OpenImageIO/filesystem.h>
23 #include <OpenImageIO/strutil.h>
24 #include <OpenImageIO/sysutil.h>
25
26 OIIO_NAMESPACE_USING
27
28 #include <stdio.h>
29
30 #include <sys/stat.h>
31
32 #if defined(_WIN32)
33 #  define DIR_SEP '\\'
34 #  define DIR_SEP_ALT '/'
35 #  include <direct.h>
36 #else
37 #  define DIR_SEP '/'
38 #  include <dirent.h>
39 #  include <pwd.h>
40 #  include <unistd.h>
41 #  include <sys/types.h>
42 #endif
43
44 #ifdef HAVE_SHLWAPI_H
45 #  include <shlwapi.h>
46 #endif
47
48 #include "util/util_map.h"
49 #include "util/util_windows.h"
50
51 CCL_NAMESPACE_BEGIN
52
53 #ifdef _WIN32
54 #  if defined(_MSC_VER) || defined(__MINGW64__)
55 typedef struct _stat64 path_stat_t;
56 #  elif defined(__MINGW32__)
57 typedef struct _stati64 path_stat_t;
58 #  else
59 typedef struct _stat path_stat_t;
60 #  endif
61 #  ifndef S_ISDIR
62 #    define S_ISDIR(x) (((x) & _S_IFDIR) == _S_IFDIR)
63 #  endif
64 #else
65 typedef struct stat path_stat_t;
66 #endif
67
68 static string cached_path = "";
69 static string cached_user_path = "";
70 static string cached_xdg_cache_path = "";
71
72 namespace {
73
74 #ifdef _WIN32
75 class directory_iterator {
76 public:
77         class path_info {
78         public:
79                 path_info(const string& path,
80                           const WIN32_FIND_DATAW& find_data)
81                 : path_(path),
82                   find_data_(find_data)
83                 {
84                 }
85
86                 string path() {
87                         return path_join(path_, string_from_wstring(find_data_.cFileName));
88                 }
89         protected:
90                 const string& path_;
91                 const WIN32_FIND_DATAW& find_data_;
92         };
93
94         directory_iterator()
95         : path_info_("", find_data_),
96           h_find_(INVALID_HANDLE_VALUE)
97         {
98         }
99
100         explicit directory_iterator(const string& path)
101         : path_(path),
102           path_info_(path, find_data_)
103         {
104                 string wildcard = path;
105                 if(wildcard[wildcard.size() - 1] != DIR_SEP) {
106                         wildcard += DIR_SEP;
107                 }
108                 wildcard += "*";
109                 h_find_ = FindFirstFileW(string_to_wstring(wildcard).c_str(),
110                                          &find_data_);
111                 if(h_find_ != INVALID_HANDLE_VALUE) {
112                         skip_dots();
113                 }
114         }
115
116         ~directory_iterator()
117         {
118                 if(h_find_ != INVALID_HANDLE_VALUE) {
119                         FindClose(h_find_);
120                 }
121         }
122
123         directory_iterator& operator++()
124         {
125                 step();
126                 return *this;
127         }
128
129         path_info* operator-> ()
130         {
131                 return &path_info_;
132         }
133
134         bool operator!=(const directory_iterator& other)
135         {
136                 return h_find_ != other.h_find_;
137         }
138
139 protected:
140         bool step()
141         {
142                 if(do_step()) {
143                         return skip_dots();
144                 }
145                 return false;
146         }
147
148         bool do_step()
149         {
150                 if(h_find_ != INVALID_HANDLE_VALUE) {
151                         bool result = FindNextFileW(h_find_, &find_data_) == TRUE;
152                         if(!result) {
153                                 FindClose(h_find_);
154                                 h_find_ = INVALID_HANDLE_VALUE;
155                         }
156                         return result;
157                 }
158                 return false;
159         }
160
161         bool skip_dots()
162         {
163                 while(wcscmp(find_data_.cFileName, L".") == 0 ||
164                       wcscmp(find_data_.cFileName, L"..") == 0)
165                 {
166                         if(!do_step()) {
167                                 return false;
168                         }
169                 }
170                 return true;
171         }
172
173         string path_;
174         path_info path_info_;
175         WIN32_FIND_DATAW find_data_;
176         HANDLE h_find_;
177 };
178 #else  /* _WIN32 */
179
180 class directory_iterator {
181 public:
182         class path_info {
183         public:
184                 explicit path_info(const string& path)
185                 : path_(path),
186                   entry_(NULL)
187                 {
188                 }
189
190                 string path() {
191                         return path_join(path_, entry_->d_name);
192                 }
193
194                 void current_entry_set(const struct dirent *entry)
195                 {
196                         entry_ = entry;
197                 }
198         protected:
199                 const string& path_;
200                 const struct dirent *entry_;
201         };
202
203         directory_iterator()
204         : path_info_(""),
205           name_list_(NULL),
206           num_entries_(-1),
207           cur_entry_(-1)
208         {
209         }
210
211         explicit directory_iterator(const string& path)
212         : path_(path),
213           path_info_(path_),
214           cur_entry_(0)
215         {
216                 num_entries_ = scandir(path.c_str(),
217                                        &name_list_,
218                                        NULL,
219                                        alphasort);
220                 if(num_entries_ < 0) {
221                         perror("scandir");
222                 }
223                 else {
224                         skip_dots();
225                 }
226         }
227
228         ~directory_iterator()
229         {
230                 destroy_name_list();
231         }
232
233         directory_iterator& operator++()
234         {
235                 step();
236                 return *this;
237         }
238
239         path_info* operator-> ()
240         {
241                 path_info_.current_entry_set(name_list_[cur_entry_]);
242                 return &path_info_;
243         }
244
245         bool operator!=(const directory_iterator& other)
246         {
247                 return name_list_ != other.name_list_;
248         }
249
250 protected:
251         bool step()
252         {
253                 if(do_step()) {
254                         return skip_dots();
255                 }
256                 return false;
257         }
258
259         bool do_step()
260         {
261                 ++cur_entry_;
262                 if(cur_entry_ >= num_entries_) {
263                         destroy_name_list();
264                         return false;
265                 }
266                 return true;
267         }
268
269         /* Skip . and .. folders. */
270         bool skip_dots()
271         {
272                 while(strcmp(name_list_[cur_entry_]->d_name, ".") == 0 ||
273                       strcmp(name_list_[cur_entry_]->d_name, "..") == 0)
274                 {
275                         if(!step()) {
276                                 return false;
277                         }
278                 }
279                 return true;
280         }
281
282         void destroy_name_list()
283         {
284                 if(name_list_ == NULL) {
285                         return;
286                 }
287                 for(int i = 0; i < num_entries_; ++i) {
288                         free(name_list_[i]);
289                 }
290                 free(name_list_);
291                 name_list_ = NULL;
292         }
293
294         string path_;
295         path_info path_info_;
296         struct dirent **name_list_;
297         int num_entries_, cur_entry_;
298 };
299
300 #endif  /* _WIN32 */
301
302 size_t find_last_slash(const string& path)
303 {
304         for(size_t i = 0; i < path.size(); ++i) {
305                 size_t index = path.size() - 1 - i;
306 #ifdef _WIN32
307                 if(path[index] == DIR_SEP || path[index] == DIR_SEP_ALT)
308 #else
309                 if(path[index] == DIR_SEP)
310 #endif
311                 {
312                         return index;
313                 }
314         }
315         return string::npos;
316 }
317
318 }  /* namespace */
319
320 static char *path_specials(const string& sub)
321 {
322         static bool env_init = false;
323         static char *env_shader_path;
324         static char *env_source_path;
325         if(!env_init) {
326                 env_shader_path = getenv("CYCLES_SHADER_PATH");
327                 /* NOTE: It is KERNEL in env variable for compatibility reasons. */
328                 env_source_path = getenv("CYCLES_KERNEL_PATH");
329                 env_init = true;
330         }
331         if(env_shader_path != NULL && sub == "shader") {
332                 return env_shader_path;
333         }
334         else if(env_shader_path != NULL && sub == "source") {
335                 return env_source_path;
336         }
337         return NULL;
338 }
339
340 #if defined(__linux__) || defined(__APPLE__)
341 static string path_xdg_cache_get()
342 {
343         const char *home = getenv("XDG_CACHE_HOME");
344         if(home) {
345                 return string(home);
346         }
347         else {
348                 home = getenv("HOME");
349                 if(home == NULL) {
350                         home = getpwuid(getuid())->pw_dir;
351                 }
352                 return path_join(string(home), ".cache");
353         }
354 }
355 #endif
356
357 void path_init(const string& path, const string& user_path)
358 {
359         cached_path = path;
360         cached_user_path = user_path;
361
362 #ifdef _MSC_VER
363         // workaround for https://svn.boost.org/trac/boost/ticket/6320
364         // indirectly init boost codec here since it's not thread safe, and can
365         // cause crashes when it happens in multithreaded image load
366         OIIO::Filesystem::exists(path);
367 #endif
368 }
369
370 string path_get(const string& sub)
371 {
372         char *special = path_specials(sub);
373         if(special != NULL)
374                 return special;
375
376         if(cached_path == "")
377                 cached_path = path_dirname(Sysutil::this_program_path());
378
379         return path_join(cached_path, sub);
380 }
381
382 string path_user_get(const string& sub)
383 {
384         if(cached_user_path == "")
385                 cached_user_path = path_dirname(Sysutil::this_program_path());
386
387         return path_join(cached_user_path, sub);
388 }
389
390 string path_cache_get(const string& sub)
391 {
392 #if defined(__linux__) || defined(__APPLE__)
393         if(cached_xdg_cache_path == "") {
394                 cached_xdg_cache_path = path_xdg_cache_get();
395         }
396         string result = path_join(cached_xdg_cache_path, "cycles");
397         return path_join(result, sub);
398 #else
399         /* TODO(sergey): What that should be on Windows? */
400         return path_user_get(path_join("cache", sub));
401 #endif
402 }
403
404 #if defined(__linux__) || defined(__APPLE__)
405 string path_xdg_home_get(const string& sub = "");
406 #endif
407
408 string path_filename(const string& path)
409 {
410         size_t index = find_last_slash(path);
411         if(index != string::npos) {
412                 /* Corner cases to match boost behavior. */
413 #ifndef _WIN32
414                 if(index == 0 && path.size() == 1) {
415                         return path;
416                 }
417 #endif
418                 if(index == path.size() - 1) {
419 #ifdef _WIN32
420                         if(index == 2) {
421                                 return string(1, DIR_SEP);
422                         }
423 #endif
424                         return ".";
425                 }
426                 return path.substr(index + 1, path.size() - index - 1);
427         }
428         return path;
429 }
430
431 string path_dirname(const string& path)
432 {
433         size_t index = find_last_slash(path);
434         if(index != string::npos) {
435 #ifndef _WIN32
436                 if(index == 0 && path.size() > 1) {
437                         return string(1, DIR_SEP);
438                 }
439 #endif
440                 return path.substr(0, index);
441         }
442         return "";
443 }
444
445 string path_join(const string& dir, const string& file)
446 {
447         if(dir.size() == 0) {
448                 return file;
449         }
450         if(file.size() == 0) {
451                 return dir;
452         }
453         string result = dir;
454 #ifndef _WIN32
455         if(result[result.size() - 1] != DIR_SEP &&
456            file[0] != DIR_SEP)
457 #else
458         if(result[result.size() - 1] != DIR_SEP &&
459            result[result.size() - 1] != DIR_SEP_ALT &&
460            file[0] != DIR_SEP &&
461            file[0] != DIR_SEP_ALT)
462 #endif
463         {
464                 result += DIR_SEP;
465         }
466         result += file;
467         return result;
468 }
469
470 string path_escape(const string& path)
471 {
472         string result = path;
473         string_replace(result, " ", "\\ ");
474         return result;
475 }
476
477 bool path_is_relative(const string& path)
478 {
479 #ifdef _WIN32
480 #  ifdef HAVE_SHLWAPI_H
481         return PathIsRelative(path.c_str());
482 #  else  /* HAVE_SHLWAPI_H */
483         if(path.size() >= 3) {
484                 return !(((path[0] >= 'a' && path[0] <= 'z') ||
485                          (path[0] >= 'A' && path[0] <= 'Z')) &&
486                          path[1] == ':' && path[2] == DIR_SEP);
487         }
488         return true;
489 #  endif  /* HAVE_SHLWAPI_H */
490 #else  /* _WIN32 */
491         if(path.size() == 0) {
492                 return 1;
493         }
494         return path[0] != DIR_SEP;
495 #endif  /* _WIN32 */
496 }
497
498 #ifdef _WIN32
499 /* Add a slash if the UNC path points to a share. */
500 static string path_unc_add_slash_to_share(const string& path)
501 {
502         size_t slash_after_server = path.find(DIR_SEP, 2);
503         if(slash_after_server != string::npos) {
504                 size_t slash_after_share = path.find(DIR_SEP,
505                                                      slash_after_server + 1);
506                 if(slash_after_share == string::npos) {
507                         return path + DIR_SEP;
508                 }
509         }
510         return path;
511 }
512
513 /* Convert:
514  *    \\?\UNC\server\share\folder\... to \\server\share\folder\...
515  *    \\?\C:\ to C:\ and \\?\C:\folder\... to C:\folder\...
516  */
517 static string path_unc_to_short(const string& path)
518 {
519         size_t len = path.size();
520         if((len > 3) &&
521            (path[0] ==  DIR_SEP) &&
522            (path[1] ==  DIR_SEP) &&
523            (path[2] ==  '?') &&
524            ((path[3] ==  DIR_SEP) || (path[3] ==  DIR_SEP_ALT)))
525         {
526                 if((len > 5) && (path[5] ==  ':')) {
527                         return path.substr(4, len - 4);
528                 }
529                 else if((len > 7) &&
530                         (path.substr(4, 3) == "UNC") &&
531                         ((path[7] ==  DIR_SEP) || (path[7] ==  DIR_SEP_ALT)))
532                 {
533                         return "\\\\" + path.substr(8, len - 8);
534                 }
535         }
536         return path;
537 }
538
539 static string path_cleanup_unc(const string& path)
540 {
541         string result = path_unc_to_short(path);
542         if(path.size() > 2) {
543                 /* It's possible path is now a non-UNC. */
544                 if(result[0] == DIR_SEP && result[1] == DIR_SEP) {
545                         return path_unc_add_slash_to_share(result);
546                 }
547         }
548         return result;
549 }
550
551 /* Make path compatible for stat() functions. */
552 static string path_make_compatible(const string& path)
553 {
554         string result = path;
555         /* In Windows stat() doesn't recognize dir ending on a slash. */
556         if(result.size() > 3 && result[result.size() - 1] == DIR_SEP) {
557                 result.resize(result.size() - 1);
558         }
559         /* Clean up UNC path. */
560         if((path.size() >= 3) && (path[0] == DIR_SEP) && (path[1] == DIR_SEP)) {
561                 result = path_cleanup_unc(result);
562         }
563         /* Make sure volume-only path ends up wit ha directory separator. */
564         if(result.size() == 2 && result[1] == ':') {
565                 result += DIR_SEP;
566         }
567         return result;
568 }
569
570 static int path_wstat(const wstring& path_wc, path_stat_t *st)
571 {
572 #if defined(_MSC_VER) || defined(__MINGW64__)
573         return _wstat64(path_wc.c_str(), st);
574 #elif defined(__MINGW32__)
575         return _wstati64(path_wc.c_str(), st);
576 #else
577         return _wstat(path_wc.c_str(), st);
578 #endif
579 }
580
581 static int path_stat(const string& path, path_stat_t *st)
582 {
583         wstring path_wc = string_to_wstring(path);
584         return path_wstat(path_wc, st);
585 }
586 #else  /* _WIN32 */
587 static int path_stat(const string& path, path_stat_t *st)
588 {
589         return stat(path.c_str(), st);
590 }
591 #endif  /* _WIN32 */
592
593 size_t path_file_size(const string& path)
594 {
595         path_stat_t st;
596         if(path_stat(path, &st) != 0) {
597                 return -1;
598         }
599         return st.st_size;
600 }
601
602 bool path_exists(const string& path)
603 {
604 #ifdef _WIN32
605         string fixed_path = path_make_compatible(path);
606         wstring path_wc = string_to_wstring(fixed_path);
607         path_stat_t st;
608         if(path_wstat(path_wc, &st) != 0) {
609                 return false;
610         }
611         return st.st_mode != 0;
612 #else  /* _WIN32 */
613         struct stat st;
614         if(stat(path.c_str(), &st) != 0) {
615                 return 0;
616         }
617         return st.st_mode != 0;
618 #endif /* _WIN32 */
619 }
620
621 bool path_is_directory(const string& path)
622 {
623         path_stat_t st;
624         if(path_stat(path, &st) != 0) {
625                 return false;
626         }
627         return S_ISDIR(st.st_mode);
628 }
629
630 static void path_files_md5_hash_recursive(MD5Hash& hash, const string& dir)
631 {
632         if(path_exists(dir)) {
633                 directory_iterator it(dir), it_end;
634
635                 for(; it != it_end; ++it) {
636                         if(path_is_directory(it->path())) {
637                                 path_files_md5_hash_recursive(hash, it->path());
638                         }
639                         else {
640                                 string filepath = it->path();
641
642                                 hash.append((const uint8_t*)filepath.c_str(), filepath.size());
643                                 hash.append_file(filepath);
644                         }
645                 }
646         }
647 }
648
649 string path_files_md5_hash(const string& dir)
650 {
651         /* computes md5 hash of all files in the directory */
652         MD5Hash hash;
653
654         path_files_md5_hash_recursive(hash, dir);
655
656         return hash.get_hex();
657 }
658
659 static bool create_directories_recursivey(const string& path)
660 {
661         if(path_is_directory(path)) {
662                 /* Directory already exists, nothing to do. */
663                 return true;
664         }
665         if(path_exists(path)) {
666                 /* File exists and it's not a directory. */
667                 return false;
668         }
669
670         string parent = path_dirname(path);
671         if(parent.size() > 0 && parent != path) {
672                 if(!create_directories_recursivey(parent)) {
673                         return false;
674                 }
675         }
676
677 #ifdef _WIN32
678         wstring path_wc = string_to_wstring(path);
679         return _wmkdir(path_wc.c_str()) == 0;
680 #else
681         return mkdir(path.c_str(), 0777) == 0;
682 #endif
683 }
684
685 void path_create_directories(const string& filepath)
686 {
687         string path = path_dirname(filepath);
688         create_directories_recursivey(path);
689 }
690
691 bool path_write_binary(const string& path, const vector<uint8_t>& binary)
692 {
693         path_create_directories(path);
694
695         /* write binary file from memory */
696         FILE *f = path_fopen(path, "wb");
697
698         if(!f)
699                 return false;
700
701         if(binary.size() > 0)
702                 fwrite(&binary[0], sizeof(uint8_t), binary.size(), f);
703
704         fclose(f);
705
706         return true;
707 }
708
709 bool path_write_text(const string& path, string& text)
710 {
711         vector<uint8_t> binary(text.length(), 0);
712         std::copy(text.begin(), text.end(), binary.begin());
713
714         return path_write_binary(path, binary);
715 }
716
717 bool path_read_binary(const string& path, vector<uint8_t>& binary)
718 {
719         /* read binary file into memory */
720         FILE *f = path_fopen(path, "rb");
721
722         if(!f) {
723                 binary.resize(0);
724                 return false;
725         }
726
727         binary.resize(path_file_size(path));
728
729         if(binary.size() == 0) {
730                 fclose(f);
731                 return false;
732         }
733
734         if(fread(&binary[0], sizeof(uint8_t), binary.size(), f) != binary.size()) {
735                 fclose(f);
736                 return false;
737         }
738
739         fclose(f);
740
741         return true;
742 }
743
744 bool path_read_text(const string& path, string& text)
745 {
746         vector<uint8_t> binary;
747
748         if(!path_exists(path) || !path_read_binary(path, binary))
749                 return false;
750
751         const char *str = (const char*)&binary[0];
752         size_t size = binary.size();
753         text = string(str, size);
754
755         return true;
756 }
757
758 uint64_t path_modified_time(const string& path)
759 {
760         path_stat_t st;
761         if(path_stat(path, &st) != 0) {
762                 return 0;
763         }
764         return st.st_mtime;
765 }
766
767 bool path_remove(const string& path)
768 {
769         return remove(path.c_str()) == 0;
770 }
771
772 struct SourceReplaceState {
773         typedef map<string, string> ProcessedMapping;
774         /* Base director for all relative include headers. */
775         string base;
776         /* Result of processed files. */
777         ProcessedMapping processed_files;
778         /* Set of files which are considered "precompiled" and which are replaced
779          * with and empty string on a subsequent occurrence in include statement.
780          */
781         set<string> precompiled_headers;
782 };
783
784 static string path_source_replace_includes_recursive(
785         const string& source,
786         const string& source_filepath,
787         SourceReplaceState *state);
788
789 static string line_directive(const SourceReplaceState& state,
790                              const string& path,
791                              const int line)
792 {
793         string unescaped_path = path;
794         /* First we make path relative. */
795         if(string_startswith(unescaped_path, state.base.c_str())) {
796                 const string base_file = path_filename(state.base);
797                 const size_t base_len = state.base.length();
798                 unescaped_path = base_file +
799                         unescaped_path.substr(base_len,
800                                             unescaped_path.length() - base_len);
801         }
802         /* Second, we replace all unsafe characters. */
803         const size_t length = unescaped_path.length();
804         string escaped_path = "";
805         for(size_t i = 0; i < length; ++i) {
806                 const char ch = unescaped_path[i];
807                 if(strchr("\"\'\?\\", ch) != NULL) {
808                         escaped_path += "\\";
809                 }
810                 escaped_path += ch;
811         }
812         /* TODO(sergey): Check whether using std::to_string combined with several
813          * concatenation operations is any faster.
814          */
815         return string_printf("#line %d \"%s\"", line, escaped_path.c_str());
816 }
817
818 static string path_source_handle_preprocessor(
819         const string& preprocessor_line,
820         const string& source_filepath,
821         const size_t line_number,
822         SourceReplaceState *state)
823 {
824         string result = preprocessor_line;
825         string token = string_strip(
826                 preprocessor_line.substr(1, preprocessor_line.size() - 1));
827         if(string_startswith(token, "include")) {
828                 token = string_strip(token.substr(7, token.size() - 7));
829                 if(token[0] == '"') {
830                         const size_t n_start = 1;
831                         const size_t n_end = token.find("\"", n_start);
832                         const string filename = token.substr(n_start, n_end - n_start);
833                         const bool is_precompiled = string_endswith(token, "// PRECOMPILED");
834                         string filepath = path_join(state->base, filename);
835                         if(!path_exists(filepath)) {
836                                 filepath = path_join(path_dirname(source_filepath),
837                                                      filename);
838                         }
839                         if(is_precompiled) {
840                                 state->precompiled_headers.insert(filepath);
841                         }
842                         string text;
843                         if(path_read_text(filepath, text)) {
844                                 text = path_source_replace_includes_recursive(
845                                         text, filepath, state);
846                                 /* Use line directives for better error messages. */
847                                 result = line_directive(*state, filepath, 1) + "\n"
848                                      + text + "\n"
849                                      + line_directive(*state, source_filepath, line_number + 1);
850                         }
851                 }
852         }
853         return result;
854 }
855
856 /* Our own little c preprocessor that replaces #includes with the file
857  * contents, to work around issue of OpenCL drivers not supporting
858  * include paths with spaces in them.
859  */
860 static string path_source_replace_includes_recursive(
861         const string& source,
862         const string& source_filepath,
863         SourceReplaceState *state)
864 {
865         /* Try to re-use processed file without spending time on replacing all
866          * include directives again.
867          */
868         SourceReplaceState::ProcessedMapping::iterator replaced_file =
869                 state->processed_files.find(source_filepath);
870         if(replaced_file != state->processed_files.end()) {
871                 if(state->precompiled_headers.find(source_filepath) !=
872                         state->precompiled_headers.end()) {
873                         return "";
874                 }
875                 return replaced_file->second;
876         }
877         /* Perform full file processing. */
878         string result = "";
879         const size_t source_length = source.length();
880         size_t index = 0;
881         /* Information about where we are in the source. */
882         size_t line_number = 0, column_number = 1;
883         /* Currently gathered non-preprocessor token.
884          * Store as start/length rather than token itself to avoid overhead of
885          * memory re-allocations on each character concatenation.
886          */
887         size_t token_start = 0, token_length = 0;
888         /* Denotes whether we're inside of preprocessor line, together with
889          * preprocessor line itself.
890          *
891          * TODO(sergey): Investigate whether using token start/end position
892          * gives measurable speedup.
893          */
894         bool inside_preprocessor = false;
895         string preprocessor_line = "";
896         /* Actual loop over the whole source. */
897         while(index < source_length) {
898                 const char ch = source[index];
899                 if(ch == '\n') {
900                         if(inside_preprocessor) {
901                                 result += path_source_handle_preprocessor(preprocessor_line,
902                                                                           source_filepath,
903                                                                           line_number,
904                                                                           state);
905                                 /* Start gathering net part of the token. */
906                                 token_start = index;
907                                 token_length = 0;
908                         }
909                         inside_preprocessor = false;
910                         preprocessor_line = "";
911                         column_number = 0;
912                         ++line_number;
913                 }
914                 else if(ch == '#' && column_number == 1 && !inside_preprocessor) {
915                         /* Append all possible non-preprocessor token to the result. */
916                         if(token_length != 0) {
917                                 result.append(source, token_start, token_length);
918                                 token_start = index;
919                                 token_length = 0;
920                         }
921                         inside_preprocessor = true;
922                 }
923                 if(inside_preprocessor) {
924                         preprocessor_line += ch;
925                 }
926                 else {
927                         ++token_length;
928                 }
929                 ++index;
930                 ++column_number;
931         }
932         /* Append possible tokens which happened before special events handled
933          * above.
934          */
935         if(token_length != 0) {
936                 result.append(source, token_start, token_length);
937         }
938         if(inside_preprocessor) {
939                 result += path_source_handle_preprocessor(preprocessor_line,
940                                                           source_filepath,
941                                                           line_number,
942                                                           state);
943         }
944         /* Store result for further reuse. */
945         state->processed_files[source_filepath] = result;
946         return result;
947 }
948
949 string path_source_replace_includes(const string& source,
950                                     const string& path,
951                                     const string& source_filename)
952 {
953         SourceReplaceState state;
954         state.base = path;
955         return path_source_replace_includes_recursive(
956                 source,
957                 path_join(path, source_filename),
958                 &state);
959 }
960
961 FILE *path_fopen(const string& path, const string& mode)
962 {
963 #ifdef _WIN32
964         wstring path_wc = string_to_wstring(path);
965         wstring mode_wc = string_to_wstring(mode);
966         return _wfopen(path_wc.c_str(), mode_wc.c_str());
967 #else
968         return fopen(path.c_str(), mode.c_str());
969 #endif
970 }
971
972 void path_cache_clear_except(const string& name, const set<string>& except)
973 {
974         string dir = path_user_get("cache");
975
976         if(path_exists(dir)) {
977                 directory_iterator it(dir), it_end;
978
979                 for(; it != it_end; ++it) {
980                         string filename = path_filename(it->path());
981
982                         if(string_startswith(filename, name.c_str()))
983                                 if(except.find(filename) == except.end())
984                                         path_remove(it->path());
985                 }
986         }
987
988 }
989
990 CCL_NAMESPACE_END
991