0509dea89d5b6c657fa60bd7677617b25fec24a6
[blender-staging.git] / source / blender / imbuf / intern / indexer.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * Peter Schlaile <peter [at] schlaile [dot] de> 2011
19  *
20  * Contributor(s): none yet.
21  *
22  * ***** END GPL LICENSE BLOCK *****
23  */
24
25 /** \file blender/imbuf/intern/indexer.c
26  *  \ingroup imbuf
27  */
28
29 #include <stdlib.h>
30
31 #include "MEM_guardedalloc.h"
32
33 #include "BLI_utildefines.h"
34 #include "BLI_endian_switch.h"
35 #include "BLI_path_util.h"
36 #include "BLI_string.h"
37 #include "BLI_fileops.h"
38 #include "BLI_ghash.h"
39
40 #include "IMB_indexer.h"
41 #include "IMB_anim.h"
42 #include "imbuf.h"
43
44 #include "MEM_guardedalloc.h"
45 #include "BKE_global.h"
46
47 #ifdef WITH_AVI
48 #  include "AVI_avi.h"
49 #endif
50
51 #ifdef WITH_FFMPEG
52 #  include "ffmpeg_compat.h"
53 #endif
54
55
56 static const char magic[] = "BlenMIdx";
57 static const char temp_ext[] = "_part";
58
59 static const int proxy_sizes[] = { IMB_PROXY_25, IMB_PROXY_50, IMB_PROXY_75,
60                                  IMB_PROXY_100 };
61 static const float proxy_fac[] = { 0.25, 0.50, 0.75, 1.00 };
62
63 #ifdef WITH_FFMPEG
64 static int tc_types[] = {IMB_TC_RECORD_RUN,
65                          IMB_TC_FREE_RUN,
66                          IMB_TC_INTERPOLATED_REC_DATE_FREE_RUN,
67                          IMB_TC_RECORD_RUN_NO_GAPS};
68 #endif
69
70 #define INDEX_FILE_VERSION 1
71
72 /* ----------------------------------------------------------------------
73  * - time code index functions
74  * ---------------------------------------------------------------------- */
75
76 anim_index_builder *IMB_index_builder_create(const char *name)
77 {
78
79         anim_index_builder *rv = MEM_callocN(sizeof(struct anim_index_builder),
80                                              "index builder");
81
82         fprintf(stderr, "Starting work on index: %s\n", name);
83
84         BLI_strncpy(rv->name, name, sizeof(rv->name));
85         BLI_strncpy(rv->temp_name, name, sizeof(rv->temp_name));
86
87         strcat(rv->temp_name, temp_ext);
88
89         BLI_make_existing_file(rv->temp_name);
90
91         rv->fp = BLI_fopen(rv->temp_name, "wb");
92
93         if (!rv->fp) {
94                 fprintf(stderr, "Couldn't open index target: %s! "
95                         "Index build broken!\n", rv->temp_name);
96                 MEM_freeN(rv);
97                 return NULL;
98         }
99
100         fprintf(rv->fp, "%s%c%.3d", magic, (ENDIAN_ORDER == B_ENDIAN) ? 'V' : 'v',
101                 INDEX_FILE_VERSION);
102
103         return rv;
104 }
105
106 void IMB_index_builder_add_entry(anim_index_builder *fp,
107                                  int frameno, unsigned long long seek_pos,
108                                  unsigned long long seek_pos_dts,
109                                  unsigned long long pts)
110 {
111         fwrite(&frameno, sizeof(int), 1, fp->fp);
112         fwrite(&seek_pos, sizeof(unsigned long long), 1, fp->fp);
113         fwrite(&seek_pos_dts, sizeof(unsigned long long), 1, fp->fp);
114         fwrite(&pts, sizeof(unsigned long long), 1, fp->fp);
115 }
116
117 void IMB_index_builder_proc_frame(anim_index_builder *fp,
118                                   unsigned char *buffer,
119                                   int data_size,
120                                   int frameno, unsigned long long seek_pos,
121                                   unsigned long long seek_pos_dts,
122                                   unsigned long long pts)
123 {
124         if (fp->proc_frame) {
125                 anim_index_entry e;
126                 e.frameno = frameno;
127                 e.seek_pos = seek_pos;
128                 e.seek_pos_dts = seek_pos_dts;
129                 e.pts = pts;
130
131                 fp->proc_frame(fp, buffer, data_size, &e);
132         }
133         else {
134                 IMB_index_builder_add_entry(fp, frameno, seek_pos,
135                                             seek_pos_dts, pts);
136         }
137 }
138
139 void IMB_index_builder_finish(anim_index_builder *fp, int rollback)
140 {
141         if (fp->delete_priv_data) {
142                 fp->delete_priv_data(fp);
143         }
144
145         fclose(fp->fp);
146         
147         if (rollback) {
148                 unlink(fp->temp_name);
149         }
150         else {
151                 unlink(fp->name);
152                 BLI_rename(fp->temp_name, fp->name);
153         }
154
155         MEM_freeN(fp);
156 }
157
158 struct anim_index *IMB_indexer_open(const char *name)
159 {
160         char header[13];
161         struct anim_index *idx;
162         FILE *fp = BLI_fopen(name, "rb");
163         int i;
164
165         if (!fp) {
166                 return NULL;
167         }
168
169         if (fread(header, 12, 1, fp) != 1) {
170                 fclose(fp);
171                 return NULL;
172         }
173
174         header[12] = 0;
175
176         if (memcmp(header, magic, 8) != 0) {
177                 fclose(fp);
178                 return NULL;
179         }
180
181         if (atoi(header + 9) != INDEX_FILE_VERSION) {
182                 fclose(fp);
183                 return NULL;
184         }
185
186         idx = MEM_callocN(sizeof(struct anim_index), "anim_index");
187
188         BLI_strncpy(idx->name, name, sizeof(idx->name));
189         
190         fseek(fp, 0, SEEK_END);
191
192         idx->num_entries = (ftell(fp) - 12) /
193                            (sizeof(int) +                /* framepos */
194                             sizeof(unsigned long long) + /* seek_pos */
195                             sizeof(unsigned long long) + /* seek_pos_dts */
196                             sizeof(unsigned long long)   /* pts */
197                            );
198
199         fseek(fp, 12, SEEK_SET);
200
201         idx->entries = MEM_callocN(sizeof(struct anim_index_entry) *
202                                    idx->num_entries, "anim_index_entries");
203
204         for (i = 0; i < idx->num_entries; i++) {
205                 fread(&idx->entries[i].frameno,
206                       sizeof(int), 1, fp);
207                 fread(&idx->entries[i].seek_pos,
208                       sizeof(unsigned long long), 1, fp);
209                 fread(&idx->entries[i].seek_pos_dts,
210                       sizeof(unsigned long long), 1, fp);
211                 fread(&idx->entries[i].pts,
212                       sizeof(unsigned long long), 1, fp);
213         }
214
215         if (((ENDIAN_ORDER == B_ENDIAN) != (header[8] == 'V'))) {
216                 for (i = 0; i < idx->num_entries; i++) {
217                         BLI_endian_switch_int32(&idx->entries[i].frameno);
218                         BLI_endian_switch_int64((int64_t *)&idx->entries[i].seek_pos);
219                         BLI_endian_switch_int64((int64_t *)&idx->entries[i].seek_pos_dts);
220                         BLI_endian_switch_int64((int64_t *)&idx->entries[i].pts);
221                 }
222         }
223
224         fclose(fp);
225
226         return idx;
227 }
228
229 unsigned long long IMB_indexer_get_seek_pos(
230         struct anim_index *idx, int frame_index)
231 {
232         if (frame_index < 0) {
233                 frame_index = 0;
234         }
235         if (frame_index >= idx->num_entries) {
236                 frame_index = idx->num_entries - 1;
237         }
238         return idx->entries[frame_index].seek_pos;
239 }
240
241 unsigned long long IMB_indexer_get_seek_pos_dts(
242         struct anim_index *idx, int frame_index)
243 {
244         if (frame_index < 0) {
245                 frame_index = 0;
246         }
247         if (frame_index >= idx->num_entries) {
248                 frame_index = idx->num_entries - 1;
249         }
250         return idx->entries[frame_index].seek_pos_dts;
251 }
252
253 int IMB_indexer_get_frame_index(struct anim_index *idx, int frameno)
254 {
255         int len = idx->num_entries;
256         int half;
257         int middle;
258         int first = 0;
259
260         /* bsearch (lower bound) the right index */
261         
262         while (len > 0) {
263                 half = len >> 1;
264                 middle = first;
265
266                 middle += half;
267
268                 if (idx->entries[middle].frameno < frameno) {
269                         first = middle;
270                         first++;
271                         len = len - half - 1;
272                 }
273                 else {
274                         len = half;
275                 }
276         }
277
278         if (first == idx->num_entries) {
279                 return idx->num_entries - 1;
280         }
281         else {
282                 return first;
283         }
284 }
285
286 unsigned long long IMB_indexer_get_pts(struct anim_index *idx,
287                                        int frame_index)
288 {
289         if (frame_index < 0) {
290                 frame_index = 0;
291         }
292         if (frame_index >= idx->num_entries) {
293                 frame_index = idx->num_entries - 1;
294         }
295         return idx->entries[frame_index].pts;
296 }
297
298 int IMB_indexer_get_duration(struct anim_index *idx)
299 {
300         if (idx->num_entries == 0) {
301                 return 0;
302         }
303         return idx->entries[idx->num_entries - 1].frameno + 1;
304 }
305
306 int IMB_indexer_can_scan(struct anim_index *idx,
307                          int old_frame_index, int new_frame_index)
308 {
309         /* makes only sense, if it is the same I-Frame and we are not
310          * trying to run backwards in time... */
311         return (IMB_indexer_get_seek_pos(idx, old_frame_index) == IMB_indexer_get_seek_pos(idx, new_frame_index) &&
312                 old_frame_index < new_frame_index);
313 }
314
315 void IMB_indexer_close(struct anim_index *idx)
316 {
317         MEM_freeN(idx->entries);
318         MEM_freeN(idx);
319 }
320
321 int IMB_proxy_size_to_array_index(IMB_Proxy_Size pr_size)
322 {
323         switch (pr_size) {
324                 case IMB_PROXY_NONE: /* if we got here, something is broken anyways,
325                                       * so sane defaults... */
326                         return 0;
327                 case IMB_PROXY_25:
328                         return 0;
329                 case IMB_PROXY_50:
330                         return 1;
331                 case IMB_PROXY_75:
332                         return 2;
333                 case IMB_PROXY_100:
334                         return 3;
335                 default:
336                         return 0;
337         }
338 }
339
340 int IMB_timecode_to_array_index(IMB_Timecode_Type tc)
341 {
342         switch (tc) {
343                 case IMB_TC_NONE: /* if we got here, something is broken anyways,
344                                    * so sane defaults... */
345                         return 0;
346                 case IMB_TC_RECORD_RUN:
347                         return 0;
348                 case IMB_TC_FREE_RUN:
349                         return 1;
350                 case IMB_TC_INTERPOLATED_REC_DATE_FREE_RUN:
351                         return 2;
352                 case IMB_TC_RECORD_RUN_NO_GAPS:
353                         return 3;
354                 default:
355                         return 0;
356         }
357 }
358
359
360 /* ----------------------------------------------------------------------
361  * - rebuild helper functions
362  * ---------------------------------------------------------------------- */
363
364 static void get_index_dir(struct anim *anim, char *index_dir, size_t index_dir_len)
365 {
366         if (!anim->index_dir[0]) {
367                 char fname[FILE_MAXFILE];
368                 BLI_split_dirfile(anim->name, index_dir, fname, index_dir_len, sizeof(fname));
369                 BLI_path_append(index_dir, index_dir_len, "BL_proxy");
370                 BLI_path_append(index_dir, index_dir_len, fname);
371         }
372         else {
373                 BLI_strncpy(index_dir, anim->index_dir, index_dir_len);
374         }
375 }
376
377 void IMB_anim_get_fname(struct anim *anim, char *file, int size)
378 {
379         char fname[FILE_MAXFILE];
380         BLI_split_dirfile(anim->name, file, fname, size, sizeof(fname));
381         BLI_strncpy(file, fname, size);
382 }
383
384 static void get_proxy_filename(struct anim *anim, IMB_Proxy_Size preview_size,
385                                char *fname, bool temp)
386 {
387         char index_dir[FILE_MAXDIR];
388         int i = IMB_proxy_size_to_array_index(preview_size);
389
390         char proxy_name[256];
391         char stream_suffix[20];
392         const char *name = (temp) ? "proxy_%d%s_part.avi" : "proxy_%d%s.avi";
393         
394         stream_suffix[0] = 0;
395
396         if (anim->streamindex > 0) {
397                 BLI_snprintf(stream_suffix, sizeof(stream_suffix), "_st%d", anim->streamindex);
398         }
399
400         BLI_snprintf(proxy_name, sizeof(proxy_name), name,
401                      (int) (proxy_fac[i] * 100), stream_suffix, anim->suffix);
402
403         get_index_dir(anim, index_dir, sizeof(index_dir));
404
405         BLI_join_dirfile(fname, FILE_MAXFILE + FILE_MAXDIR, index_dir, proxy_name);
406 }
407
408 static void get_tc_filename(struct anim *anim, IMB_Timecode_Type tc,
409                             char *fname)
410 {
411         char index_dir[FILE_MAXDIR];
412         int i = IMB_timecode_to_array_index(tc);
413         const char *index_names[] = {
414                 "record_run%s%s.blen_tc",
415                 "free_run%s%s.blen_tc",
416                 "interp_free_run%s%s.blen_tc",
417                 "record_run_no_gaps%s%s.blen_tc"
418         };
419
420         char stream_suffix[20];
421         char index_name[256];
422         
423         stream_suffix[0] = 0;
424
425         if (anim->streamindex > 0) {
426                 BLI_snprintf(stream_suffix, 20, "_st%d", anim->streamindex);
427         }
428         
429         BLI_snprintf(index_name, 256, index_names[i], stream_suffix, anim->suffix);
430
431         get_index_dir(anim, index_dir, sizeof(index_dir));
432         
433         BLI_join_dirfile(fname, FILE_MAXFILE + FILE_MAXDIR, 
434                          index_dir, index_name);
435 }
436
437 /* ----------------------------------------------------------------------
438  * - common rebuilder structures
439  * ---------------------------------------------------------------------- */
440
441 typedef struct IndexBuildContext {
442         int anim_type;
443 } IndexBuildContext;
444
445
446 /* ----------------------------------------------------------------------
447  * - ffmpeg rebuilder
448  * ---------------------------------------------------------------------- */
449
450 #ifdef WITH_FFMPEG
451
452 struct proxy_output_ctx {
453         AVFormatContext *of;
454         AVStream *st;
455         AVCodecContext *c;
456         AVCodec *codec;
457         struct SwsContext *sws_ctx;
458         AVFrame *frame;
459         int cfra;
460         int proxy_size;
461         int orig_height;
462         struct anim *anim;
463 };
464
465 // work around stupid swscaler 16 bytes alignment bug...
466
467 static int round_up(int x, int mod)
468 {
469         return x + ((mod - (x % mod)) % mod);
470 }
471
472 static struct proxy_output_ctx *alloc_proxy_output_ffmpeg(
473         struct anim *anim,
474         AVStream *st, int proxy_size, int width, int height,
475         int quality)
476 {
477         struct proxy_output_ctx *rv = MEM_callocN(
478                 sizeof(struct proxy_output_ctx), "alloc_proxy_output");
479         
480         char fname[FILE_MAX];
481         int ffmpeg_quality;
482
483         /* JPEG requires this */
484         width = round_up(width, 8);
485         height = round_up(height, 8);
486
487         rv->proxy_size = proxy_size;
488         rv->anim = anim;
489
490         get_proxy_filename(rv->anim, rv->proxy_size, fname, true);
491         BLI_make_existing_file(fname);
492
493         rv->of = avformat_alloc_context();
494         rv->of->oformat = av_guess_format("avi", NULL, NULL);
495         
496         BLI_strncpy(rv->of->filename, fname, sizeof(rv->of->filename));
497
498         fprintf(stderr, "Starting work on proxy: %s\n", rv->of->filename);
499
500         rv->st = avformat_new_stream(rv->of, NULL);
501         rv->st->id = 0;
502
503         rv->c = rv->st->codec;
504         rv->c->codec_type = AVMEDIA_TYPE_VIDEO;
505         rv->c->codec_id = AV_CODEC_ID_MJPEG;
506         rv->c->width = width;
507         rv->c->height = height;
508
509         rv->of->oformat->video_codec = rv->c->codec_id;
510         rv->codec = avcodec_find_encoder(rv->c->codec_id);
511
512         if (!rv->codec) {
513                 fprintf(stderr, "No ffmpeg MJPEG encoder available? "
514                         "Proxy not built!\n");
515                 av_free(rv->of);
516                 return NULL;
517         }
518
519         if (rv->codec->pix_fmts) {
520                 rv->c->pix_fmt = rv->codec->pix_fmts[0];
521         }
522         else {
523                 rv->c->pix_fmt = PIX_FMT_YUVJ420P;
524         }
525
526         rv->c->sample_aspect_ratio =
527             rv->st->sample_aspect_ratio =
528                 st->codec->sample_aspect_ratio;
529
530         rv->c->time_base.den = 25;
531         rv->c->time_base.num = 1;
532         rv->st->time_base = rv->c->time_base;
533
534         /* there's no  way to set JPEG quality in the same way as in AVI JPEG and image sequence,
535          * but this seems to be giving expected quality result */
536         ffmpeg_quality = (int)(1.0f + 30.0f * (1.0f - (float)quality / 100.0f) + 0.5f);
537         av_opt_set_int(rv->c, "qmin", ffmpeg_quality, 0);
538         av_opt_set_int(rv->c, "qmax", ffmpeg_quality, 0);
539
540         if (rv->of->flags & AVFMT_GLOBALHEADER) {
541                 rv->c->flags |= CODEC_FLAG_GLOBAL_HEADER;
542         }
543
544         if (avio_open(&rv->of->pb, fname, AVIO_FLAG_WRITE) < 0) {
545                 fprintf(stderr, "Couldn't open outputfile! "
546                         "Proxy not built!\n");
547                 av_free(rv->of);
548                 return 0;
549         }
550
551         avcodec_open2(rv->c, rv->codec, NULL);
552
553         rv->orig_height = av_get_cropped_height_from_codec(st->codec);
554
555         if (st->codec->width != width || st->codec->height != height ||
556             st->codec->pix_fmt != rv->c->pix_fmt)
557         {
558                 rv->frame = avcodec_alloc_frame();
559                 avpicture_fill((AVPicture *) rv->frame,
560                                MEM_mallocN(avpicture_get_size(
561                                                rv->c->pix_fmt,
562                                                round_up(width, 16), height),
563                                            "alloc proxy output frame"),
564                                rv->c->pix_fmt, round_up(width, 16), height);
565
566                 rv->sws_ctx = sws_getContext(
567                         st->codec->width,
568                         rv->orig_height,
569                         st->codec->pix_fmt,
570                         width, height,
571                         rv->c->pix_fmt,
572                         SWS_FAST_BILINEAR | SWS_PRINT_INFO,
573                         NULL, NULL, NULL);
574         }
575
576         if (avformat_write_header(rv->of, NULL) < 0) {
577                 fprintf(stderr, "Couldn't set output parameters? "
578                         "Proxy not built!\n");
579                 av_free(rv->of);
580                 return 0;
581         }
582
583         return rv;
584 }
585
586 static int add_to_proxy_output_ffmpeg(
587         struct proxy_output_ctx *ctx, AVFrame *frame)
588 {
589         AVPacket packet = { 0 };
590         int ret, got_output;
591
592         av_init_packet(&packet);
593
594         if (!ctx) {
595                 return 0;
596         }
597
598         if (ctx->sws_ctx && frame &&
599             (frame->data[0] || frame->data[1] ||
600              frame->data[2] || frame->data[3]))
601         {
602                 sws_scale(ctx->sws_ctx, (const uint8_t *const *) frame->data,
603                           frame->linesize, 0, ctx->orig_height,
604                           ctx->frame->data, ctx->frame->linesize);
605         }
606
607         frame = ctx->sws_ctx ? (frame ? ctx->frame : 0) : frame;
608
609         if (frame) {
610                 frame->pts = ctx->cfra++;
611         }
612
613         ret = avcodec_encode_video2(ctx->c, &packet, frame, &got_output);
614         if (ret < 0) {
615                 fprintf(stderr, "Error encoding proxy frame %d for '%s'\n", 
616                         ctx->cfra - 1, ctx->of->filename);
617                 return 0;
618         }
619
620         if (got_output) {
621                 if (packet.pts != AV_NOPTS_VALUE) {
622                         packet.pts = av_rescale_q(packet.pts,
623                                                   ctx->c->time_base,
624                                                   ctx->st->time_base);
625                 }
626                 if (packet.dts != AV_NOPTS_VALUE) {
627                         packet.dts = av_rescale_q(packet.dts,
628                                                   ctx->c->time_base,
629                                                   ctx->st->time_base);
630                 }
631
632                 packet.stream_index = ctx->st->index;
633
634                 if (av_interleaved_write_frame(ctx->of, &packet) != 0) {
635                         fprintf(stderr, "Error writing proxy frame %d "
636                                 "into '%s'\n", ctx->cfra - 1,
637                                 ctx->of->filename);
638                         return 0;
639                 }
640
641                 return 1;
642         }
643         else {
644                 return 0;
645         }
646 }
647
648 static void free_proxy_output_ffmpeg(struct proxy_output_ctx *ctx,
649                                      int rollback)
650 {
651         char fname[FILE_MAX];
652         char fname_tmp[FILE_MAX];
653
654         if (!ctx) {
655                 return;
656         }
657
658         if (!rollback) {
659                 while (add_to_proxy_output_ffmpeg(ctx, NULL)) {}
660         }
661
662         avcodec_flush_buffers(ctx->c);
663
664         av_write_trailer(ctx->of);
665         
666         avcodec_close(ctx->c);
667         
668         if (ctx->of->oformat) {
669                 if (!(ctx->of->oformat->flags & AVFMT_NOFILE)) {
670                         avio_close(ctx->of->pb);
671                 }
672         }
673         avformat_free_context(ctx->of);
674
675         if (ctx->sws_ctx) {
676                 sws_freeContext(ctx->sws_ctx);
677
678                 MEM_freeN(ctx->frame->data[0]);
679                 av_free(ctx->frame);
680         }
681
682         get_proxy_filename(ctx->anim, ctx->proxy_size, 
683                            fname_tmp, true);
684
685         if (rollback) {
686                 unlink(fname_tmp);
687         }
688         else {
689                 get_proxy_filename(ctx->anim, ctx->proxy_size, 
690                                    fname, false);
691                 unlink(fname);
692                 BLI_rename(fname_tmp, fname);
693         }
694         
695         MEM_freeN(ctx);
696 }
697
698 typedef struct FFmpegIndexBuilderContext {
699         int anim_type;
700
701         AVFormatContext *iFormatCtx;
702         AVCodecContext *iCodecCtx;
703         AVCodec *iCodec;
704         AVStream *iStream;
705         int videoStream;
706
707         int num_proxy_sizes;
708         int num_indexers;
709
710         struct proxy_output_ctx *proxy_ctx[IMB_PROXY_MAX_SLOT];
711         anim_index_builder *indexer[IMB_TC_MAX_SLOT];
712
713         IMB_Timecode_Type tcs_in_use;
714         IMB_Proxy_Size proxy_sizes_in_use;
715
716         unsigned long long seek_pos;
717         unsigned long long last_seek_pos;
718         unsigned long long seek_pos_dts;
719         unsigned long long seek_pos_pts;
720         unsigned long long last_seek_pos_dts;
721         unsigned long long start_pts;
722         double frame_rate;
723         double pts_time_base;
724         int frameno, frameno_gapless;
725         int start_pts_set;
726 } FFmpegIndexBuilderContext;
727
728 static IndexBuildContext *index_ffmpeg_create_context(struct anim *anim, IMB_Timecode_Type tcs_in_use,
729                                                       IMB_Proxy_Size proxy_sizes_in_use, int quality)
730 {
731         FFmpegIndexBuilderContext *context = MEM_callocN(sizeof(FFmpegIndexBuilderContext), "FFmpeg index builder context");
732         int num_proxy_sizes = IMB_PROXY_MAX_SLOT;
733         int num_indexers = IMB_TC_MAX_SLOT;
734         int i, streamcount;
735
736         context->tcs_in_use = tcs_in_use;
737         context->proxy_sizes_in_use = proxy_sizes_in_use;
738         context->num_proxy_sizes = IMB_PROXY_MAX_SLOT;
739         context->num_indexers = IMB_TC_MAX_SLOT;
740
741         memset(context->proxy_ctx, 0, sizeof(context->proxy_ctx));
742         memset(context->indexer, 0, sizeof(context->indexer));
743
744         if (avformat_open_input(&context->iFormatCtx, anim->name, NULL, NULL) != 0) {
745                 MEM_freeN(context);
746                 return NULL;
747         }
748
749         if (avformat_find_stream_info(context->iFormatCtx, NULL) < 0) {
750                 avformat_close_input(&context->iFormatCtx);
751                 MEM_freeN(context);
752                 return NULL;
753         }
754
755         streamcount = anim->streamindex;
756
757         /* Find the video stream */
758         context->videoStream = -1;
759         for (i = 0; i < context->iFormatCtx->nb_streams; i++)
760                 if (context->iFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
761                         if (streamcount > 0) {
762                                 streamcount--;
763                                 continue;
764                         }
765                         context->videoStream = i;
766                         break;
767                 }
768
769         if (context->videoStream == -1) {
770                 avformat_close_input(&context->iFormatCtx);
771                 MEM_freeN(context);
772                 return NULL;
773         }
774
775         context->iStream = context->iFormatCtx->streams[context->videoStream];
776         context->iCodecCtx = context->iStream->codec;
777
778         context->iCodec = avcodec_find_decoder(context->iCodecCtx->codec_id);
779
780         if (context->iCodec == NULL) {
781                 avformat_close_input(&context->iFormatCtx);
782                 MEM_freeN(context);
783                 return NULL;
784         }
785
786         context->iCodecCtx->workaround_bugs = 1;
787
788         if (avcodec_open2(context->iCodecCtx, context->iCodec, NULL) < 0) {
789                 avformat_close_input(&context->iFormatCtx);
790                 MEM_freeN(context);
791                 return NULL;
792         }
793
794         for (i = 0; i < num_proxy_sizes; i++) {
795                 if (proxy_sizes_in_use & proxy_sizes[i]) {
796                         context->proxy_ctx[i] = alloc_proxy_output_ffmpeg(
797                                 anim, context->iStream, proxy_sizes[i],
798                                 context->iCodecCtx->width * proxy_fac[i],
799                                 av_get_cropped_height_from_codec(
800                                 context->iCodecCtx) * proxy_fac[i],
801                                 quality);
802                         if (!context->proxy_ctx[i]) {
803                                 proxy_sizes_in_use &= ~proxy_sizes[i];
804                         }
805                 }
806         }
807
808         for (i = 0; i < num_indexers; i++) {
809                 if (tcs_in_use & tc_types[i]) {
810                         char fname[FILE_MAX];
811
812                         get_tc_filename(anim, tc_types[i], fname);
813
814                         context->indexer[i] = IMB_index_builder_create(fname);
815                         if (!context->indexer[i]) {
816                                 tcs_in_use &= ~tc_types[i];
817                         }
818                 }
819         }
820
821         return (IndexBuildContext *)context;
822 }
823
824 static void index_rebuild_ffmpeg_finish(FFmpegIndexBuilderContext *context, int stop)
825 {
826         int i;
827
828         for (i = 0; i < context->num_indexers; i++) {
829                 if (context->tcs_in_use & tc_types[i]) {
830                         IMB_index_builder_finish(context->indexer[i], stop);
831                 }
832         }
833
834         for (i = 0; i < context->num_proxy_sizes; i++) {
835                 if (context->proxy_sizes_in_use & proxy_sizes[i]) {
836                         free_proxy_output_ffmpeg(context->proxy_ctx[i], stop);
837                 }
838         }
839
840         avcodec_close(context->iCodecCtx);
841         avformat_close_input(&context->iFormatCtx);
842
843         MEM_freeN(context);
844 }
845
846 static void index_rebuild_ffmpeg_proc_decoded_frame(
847         FFmpegIndexBuilderContext *context, 
848         AVPacket * curr_packet,
849         AVFrame *in_frame)
850 {
851         int i;
852         unsigned long long s_pos = context->seek_pos;
853         unsigned long long s_dts = context->seek_pos_dts;
854         unsigned long long pts = av_get_pts_from_frame(context->iFormatCtx, in_frame);
855
856         for (i = 0; i < context->num_proxy_sizes; i++) {
857                 add_to_proxy_output_ffmpeg(context->proxy_ctx[i], in_frame);
858         }
859
860         if (!context->start_pts_set) {
861                 context->start_pts = pts;
862                 context->start_pts_set = true;
863         }
864
865         context->frameno = floor((pts - context->start_pts) *
866                                  context->pts_time_base  *
867                                  context->frame_rate + 0.5);
868
869         /* decoding starts *always* on I-Frames,
870          * so: P-Frames won't work, even if all the
871          * information is in place, when we seek
872          * to the I-Frame presented *after* the P-Frame,
873          * but located before the P-Frame within
874          * the stream */
875
876         if (pts < context->seek_pos_pts) {
877                 s_pos = context->last_seek_pos;
878                 s_dts = context->last_seek_pos_dts;
879         }
880
881         for (i = 0; i < context->num_indexers; i++) {
882                 if (context->tcs_in_use & tc_types[i]) {
883                         int tc_frameno = context->frameno;
884
885                         if (tc_types[i] == IMB_TC_RECORD_RUN_NO_GAPS)
886                                 tc_frameno = context->frameno_gapless;
887                         
888                         IMB_index_builder_proc_frame(
889                                 context->indexer[i],
890                                 curr_packet->data,
891                                 curr_packet->size,
892                                 tc_frameno,
893                                 s_pos, s_dts, pts);
894                 }
895         }
896         
897         context->frameno_gapless++;
898 }
899
900 static int index_rebuild_ffmpeg(FFmpegIndexBuilderContext *context,
901                                 short *stop, short *do_update, float *progress)
902 {
903         AVFrame *in_frame = 0;
904         AVPacket next_packet;
905         uint64_t stream_size;
906
907         memset(&next_packet, 0, sizeof(AVPacket));
908
909         in_frame = avcodec_alloc_frame();
910
911         stream_size = avio_size(context->iFormatCtx->pb);
912
913         context->frame_rate = av_q2d(av_get_r_frame_rate_compat(context->iStream));
914         context->pts_time_base = av_q2d(context->iStream->time_base);
915
916         while (av_read_frame(context->iFormatCtx, &next_packet) >= 0) {
917                 int frame_finished = 0;
918                 float next_progress =  (float)((int)floor(((double) next_packet.pos) * 100 /
919                                                           ((double) stream_size) + 0.5)) / 100;
920
921                 if (*progress != next_progress) {
922                         *progress = next_progress;
923                         *do_update = true;
924                 }
925
926                 if (*stop) {
927                         av_free_packet(&next_packet);
928                         break;
929                 }
930
931                 if (next_packet.stream_index == context->videoStream) {
932                         if (next_packet.flags & AV_PKT_FLAG_KEY) {
933                                 context->last_seek_pos = context->seek_pos;
934                                 context->last_seek_pos_dts = context->seek_pos_dts;
935                                 context->seek_pos = next_packet.pos;
936                                 context->seek_pos_dts = next_packet.dts;
937                                 context->seek_pos_pts = next_packet.pts;
938                         }
939
940                         avcodec_decode_video2(
941                                 context->iCodecCtx, in_frame, &frame_finished,
942                                 &next_packet);
943                 }
944
945                 if (frame_finished) {
946                         index_rebuild_ffmpeg_proc_decoded_frame(
947                                 context, &next_packet, in_frame);
948                 }
949                 av_free_packet(&next_packet);
950         }
951
952         /* process pictures still stuck in decoder engine after EOF
953          * according to ffmpeg docs using 0-size packets.
954          *
955          * At least, if we haven't already stopped... */
956
957         /* this creates the 0-size packet and prevents a memory leak. */
958         av_free_packet(&next_packet);
959
960         if (!*stop) {
961                 int frame_finished;
962
963                 do {
964                         frame_finished = 0;
965
966                         avcodec_decode_video2(
967                                 context->iCodecCtx, in_frame, &frame_finished,
968                                 &next_packet);
969
970                         if (frame_finished) {
971                                 index_rebuild_ffmpeg_proc_decoded_frame(
972                                         context, &next_packet, in_frame);
973                         }
974                 } while (frame_finished);
975         }
976
977         av_free(in_frame);
978
979         return 1;
980 }
981
982 #endif
983
984 /* ----------------------------------------------------------------------
985  * - internal AVI (fallback) rebuilder
986  * ---------------------------------------------------------------------- */
987
988 #ifdef WITH_AVI
989 typedef struct FallbackIndexBuilderContext {
990         int anim_type;
991
992         struct anim *anim;
993         AviMovie *proxy_ctx[IMB_PROXY_MAX_SLOT];
994         IMB_Proxy_Size proxy_sizes_in_use;
995 } FallbackIndexBuilderContext;
996
997 static AviMovie *alloc_proxy_output_avi(
998         struct anim *anim, char *filename, int width, int height,
999         int quality)
1000 {
1001         int x, y;
1002         AviFormat format;
1003         double framerate;
1004         AviMovie *avi;
1005         short frs_sec = 25;      /* it doesn't really matter for proxies,
1006                                   * but sane defaults help anyways...*/
1007         float frs_sec_base = 1.0;
1008
1009         IMB_anim_get_fps(anim, &frs_sec, &frs_sec_base, false);
1010         
1011         x = width;
1012         y = height;
1013
1014         framerate = (double) frs_sec / (double) frs_sec_base;
1015         
1016         avi = MEM_mallocN(sizeof(AviMovie), "avimovie");
1017
1018         format = AVI_FORMAT_MJPEG;
1019
1020         if (AVI_open_compress(filename, avi, 1, format) != AVI_ERROR_NONE) {
1021                 MEM_freeN(avi);
1022                 return NULL;
1023         }
1024                         
1025         AVI_set_compress_option(avi, AVI_OPTION_TYPE_MAIN, 0, AVI_OPTION_WIDTH, &x);
1026         AVI_set_compress_option(avi, AVI_OPTION_TYPE_MAIN, 0, AVI_OPTION_HEIGHT, &y);
1027         AVI_set_compress_option(avi, AVI_OPTION_TYPE_MAIN, 0, AVI_OPTION_QUALITY, &quality);
1028         AVI_set_compress_option(avi, AVI_OPTION_TYPE_MAIN, 0, AVI_OPTION_FRAMERATE, &framerate);
1029
1030         avi->interlace = 0;
1031         avi->odd_fields = 0;
1032
1033         return avi;
1034 }
1035
1036 static IndexBuildContext *index_fallback_create_context(struct anim *anim, IMB_Timecode_Type UNUSED(tcs_in_use),
1037                                                         IMB_Proxy_Size proxy_sizes_in_use, int quality)
1038 {
1039         FallbackIndexBuilderContext *context;
1040         int i;
1041
1042         /* since timecode indices only work with ffmpeg right now,
1043          * don't know a sensible fallback here...
1044          *
1045          * so no proxies, no game to play...
1046          */
1047         if (proxy_sizes_in_use == IMB_PROXY_NONE) {
1048                 return NULL;
1049         }
1050
1051         context = MEM_callocN(sizeof(FallbackIndexBuilderContext), "fallback index builder context");
1052
1053         context->anim = anim;
1054         context->proxy_sizes_in_use = proxy_sizes_in_use;
1055
1056         memset(context->proxy_ctx, 0, sizeof(context->proxy_ctx));
1057
1058         for (i = 0; i < IMB_PROXY_MAX_SLOT; i++) {
1059                 if (context->proxy_sizes_in_use & proxy_sizes[i]) {
1060                         char fname[FILE_MAX];
1061
1062                         get_proxy_filename(anim, proxy_sizes[i], fname, true);
1063                         BLI_make_existing_file(fname);
1064
1065                         context->proxy_ctx[i] = alloc_proxy_output_avi(anim, fname,
1066                                                                        anim->x * proxy_fac[i], anim->y * proxy_fac[i], quality);
1067                 }
1068         }
1069
1070         return (IndexBuildContext *)context;
1071 }
1072
1073 static void index_rebuild_fallback_finish(FallbackIndexBuilderContext *context, int stop)
1074 {
1075         struct anim *anim = context->anim;
1076         char fname[FILE_MAX];
1077         char fname_tmp[FILE_MAX];
1078         int i;
1079
1080         for (i = 0; i < IMB_PROXY_MAX_SLOT; i++) {
1081                 if (context->proxy_sizes_in_use & proxy_sizes[i]) {
1082                         AVI_close_compress(context->proxy_ctx[i]);
1083                         MEM_freeN(context->proxy_ctx[i]);
1084
1085                         get_proxy_filename(anim, proxy_sizes[i], fname_tmp, true);
1086                         get_proxy_filename(anim, proxy_sizes[i], fname, false);
1087
1088                         if (stop) {
1089                                 unlink(fname_tmp);
1090                         }
1091                         else {
1092                                 unlink(fname);
1093                                 rename(fname_tmp, fname);
1094                         }
1095                 }
1096         }
1097 }
1098
1099 static void index_rebuild_fallback(FallbackIndexBuilderContext *context,
1100                                    short *stop, short *do_update, float *progress)
1101 {
1102         int cnt = IMB_anim_get_duration(context->anim, IMB_TC_NONE);
1103         int i, pos;
1104         struct anim *anim = context->anim;
1105
1106         for (pos = 0; pos < cnt; pos++) {
1107                 struct ImBuf *ibuf = IMB_anim_absolute(anim, pos, IMB_TC_NONE, IMB_PROXY_NONE);
1108                 struct ImBuf *tmp_ibuf = IMB_dupImBuf(ibuf);
1109                 float next_progress = (float) pos / (float) cnt;
1110
1111                 if (*progress != next_progress) {
1112                         *progress = next_progress;
1113                         *do_update = true;
1114                 }
1115                 
1116                 if (*stop) {
1117                         break;
1118                 }
1119
1120                 IMB_flipy(tmp_ibuf);
1121
1122                 for (i = 0; i < IMB_PROXY_MAX_SLOT; i++) {
1123                         if (context->proxy_sizes_in_use & proxy_sizes[i]) {
1124                                 int x = anim->x * proxy_fac[i];
1125                                 int y = anim->y * proxy_fac[i];
1126
1127                                 struct ImBuf *s_ibuf = IMB_dupImBuf(tmp_ibuf);
1128
1129                                 IMB_scalefastImBuf(s_ibuf, x, y);
1130
1131                                 IMB_convert_rgba_to_abgr(s_ibuf);
1132         
1133                                 AVI_write_frame(context->proxy_ctx[i], pos,
1134                                                 AVI_FORMAT_RGB32,
1135                                                 s_ibuf->rect, x * y * 4);
1136
1137                                 /* note that libavi free's the buffer... */
1138                                 s_ibuf->rect = NULL;
1139
1140                                 IMB_freeImBuf(s_ibuf);
1141                         }
1142                 }
1143
1144                 IMB_freeImBuf(tmp_ibuf);
1145                 IMB_freeImBuf(ibuf);
1146         }
1147 }
1148
1149 #endif  /* WITH_AVI */
1150
1151 /* ----------------------------------------------------------------------
1152  * - public API
1153  * ---------------------------------------------------------------------- */
1154
1155 IndexBuildContext *IMB_anim_index_rebuild_context(struct anim *anim, IMB_Timecode_Type tcs_in_use,
1156                                                   IMB_Proxy_Size proxy_sizes_in_use, int quality,
1157                                                   const bool overwrite, GSet *file_list)
1158 {
1159         IndexBuildContext *context = NULL;
1160         IMB_Proxy_Size proxy_sizes_to_build = proxy_sizes_in_use;
1161         int i;
1162
1163         /* Don't generate the same file twice! */
1164         if (file_list) {
1165                 for (i = 0; i < IMB_PROXY_MAX_SLOT; ++i) {
1166                         IMB_Proxy_Size proxy_size = proxy_sizes[i];
1167                         if (proxy_size & proxy_sizes_to_build) {
1168                                 char filename[FILE_MAX];
1169                                 get_proxy_filename(anim, proxy_size, filename, false);
1170
1171                                 if (BLI_gset_haskey(file_list, filename)) {
1172                                         proxy_sizes_to_build &= ~proxy_size;
1173                                         printf("Proxy: %s already registered for generation, skipping\n", filename);
1174                                 }
1175                                 else {
1176                                         BLI_gset_insert(file_list, BLI_strdup(filename));
1177                                 }
1178                         }
1179                 }
1180         }
1181         
1182         if (!overwrite) {
1183                 IMB_Proxy_Size built_proxies = IMB_anim_proxy_get_existing(anim);
1184                 if (built_proxies != 0) {
1185
1186                         for (i = 0; i < IMB_PROXY_MAX_SLOT; ++i) {
1187                                 IMB_Proxy_Size proxy_size = proxy_sizes[i];
1188                                 if (proxy_size & built_proxies) {
1189                                         char filename[FILE_MAX];
1190                                         get_proxy_filename(anim, proxy_size, filename, false);
1191                                         printf("Skipping proxy: %s\n", filename);
1192                                 }
1193                         }
1194                 }
1195                 proxy_sizes_to_build &= ~built_proxies;
1196         }
1197         
1198         fflush(stdout);
1199
1200         if (proxy_sizes_to_build == 0) {
1201                 return NULL;
1202         }
1203         
1204
1205         switch (anim->curtype) {
1206 #ifdef WITH_FFMPEG
1207                 case ANIM_FFMPEG:
1208                         context = index_ffmpeg_create_context(anim, tcs_in_use, proxy_sizes_to_build, quality);
1209                         break;
1210 #endif
1211 #ifdef WITH_AVI
1212                 default:
1213                         context = index_fallback_create_context(anim, tcs_in_use, proxy_sizes_to_build, quality);
1214                         break;
1215 #endif
1216         }
1217
1218         if (context)
1219                 context->anim_type = anim->curtype;
1220
1221         return context;
1222
1223         UNUSED_VARS(tcs_in_use, proxy_sizes_in_use, quality);
1224 }
1225
1226 void IMB_anim_index_rebuild(struct IndexBuildContext *context,
1227                             short *stop, short *do_update, float *progress)
1228 {
1229         switch (context->anim_type) {
1230 #ifdef WITH_FFMPEG
1231                 case ANIM_FFMPEG:
1232                         index_rebuild_ffmpeg((FFmpegIndexBuilderContext *)context, stop, do_update, progress);
1233                         break;
1234 #endif
1235 #ifdef WITH_AVI
1236                 default:
1237                         index_rebuild_fallback((FallbackIndexBuilderContext *)context, stop, do_update, progress);
1238                         break;
1239 #endif
1240         }
1241
1242         UNUSED_VARS(stop, do_update, progress);
1243 }
1244
1245 void IMB_anim_index_rebuild_finish(IndexBuildContext *context, short stop)
1246 {
1247         switch (context->anim_type) {
1248 #ifdef WITH_FFMPEG
1249                 case ANIM_FFMPEG:
1250                         index_rebuild_ffmpeg_finish((FFmpegIndexBuilderContext *)context, stop);
1251                         break;
1252 #endif
1253 #ifdef WITH_AVI
1254                 default:
1255                         index_rebuild_fallback_finish((FallbackIndexBuilderContext *)context, stop);
1256                         break;
1257 #endif
1258         }
1259
1260         /* static defined at top of the file */
1261         UNUSED_VARS(stop, proxy_sizes);
1262 }
1263
1264
1265 void IMB_free_indices(struct anim *anim)
1266 {
1267         int i;
1268
1269         for (i = 0; i < IMB_PROXY_MAX_SLOT; i++) {
1270                 if (anim->proxy_anim[i]) {
1271                         IMB_close_anim(anim->proxy_anim[i]);
1272                         anim->proxy_anim[i] = NULL;
1273                 }
1274         }
1275
1276         for (i = 0; i < IMB_TC_MAX_SLOT; i++) {
1277                 if (anim->curr_idx[i]) {
1278                         IMB_indexer_close(anim->curr_idx[i]);
1279                         anim->curr_idx[i] = NULL;
1280                 }
1281         }
1282
1283
1284         anim->proxies_tried = 0;
1285         anim->indices_tried = 0;
1286 }
1287
1288 void IMB_anim_set_index_dir(struct anim *anim, const char *dir)
1289 {
1290         if (STREQ(anim->index_dir, dir)) {
1291                 return;
1292         }
1293         BLI_strncpy(anim->index_dir, dir, sizeof(anim->index_dir));
1294
1295         IMB_free_indices(anim);
1296 }
1297
1298 struct anim *IMB_anim_open_proxy(
1299         struct anim *anim, IMB_Proxy_Size preview_size)
1300 {
1301         char fname[FILE_MAX];
1302         int i = IMB_proxy_size_to_array_index(preview_size);
1303
1304         if (anim->proxy_anim[i]) {
1305                 return anim->proxy_anim[i];
1306         }
1307
1308         if (anim->proxies_tried & preview_size) {
1309                 return NULL;
1310         }
1311
1312         get_proxy_filename(anim, preview_size, fname, false);
1313
1314         /* proxies are generated in the same color space as animation itself */
1315         anim->proxy_anim[i] = IMB_open_anim(fname, 0, 0, anim->colorspace);
1316         
1317         anim->proxies_tried |= preview_size;
1318
1319         return anim->proxy_anim[i];
1320 }
1321
1322 struct anim_index *IMB_anim_open_index(
1323         struct anim *anim, IMB_Timecode_Type tc)
1324 {
1325         char fname[FILE_MAX];
1326         int i = IMB_timecode_to_array_index(tc);
1327
1328         if (anim->curr_idx[i]) {
1329                 return anim->curr_idx[i];
1330         }
1331
1332         if (anim->indices_tried & tc) {
1333                 return NULL;
1334         }
1335
1336         get_tc_filename(anim, tc, fname);
1337
1338         anim->curr_idx[i] = IMB_indexer_open(fname);
1339         
1340         anim->indices_tried |= tc;
1341
1342         return anim->curr_idx[i];
1343 }
1344
1345 int IMB_anim_index_get_frame_index(struct anim *anim, IMB_Timecode_Type tc,
1346                                    int position)
1347 {
1348         struct anim_index *idx = IMB_anim_open_index(anim, tc);
1349
1350         if (!idx) {
1351                 return position;
1352         }
1353
1354         return IMB_indexer_get_frame_index(idx, position);
1355 }
1356
1357 IMB_Proxy_Size IMB_anim_proxy_get_existing(struct anim *anim)
1358 {
1359         const int num_proxy_sizes = IMB_PROXY_MAX_SLOT;
1360         IMB_Proxy_Size existing = 0;
1361         int i;
1362         for (i = 0; i < num_proxy_sizes; ++i) {
1363                 IMB_Proxy_Size proxy_size = proxy_sizes[i];
1364                 char filename[FILE_MAX];
1365                 get_proxy_filename(anim, proxy_size, filename, false);
1366                 if (BLI_exists(filename)) {
1367                         existing |= proxy_size;
1368                 }
1369         }
1370         return existing;
1371 }