Merge branch 'blender2.7'
[blender.git] / source / blender / blenkernel / intern / tracking_util.c
1 /*
2  * This program is free software; you can redistribute it and/or
3  * modify it under the terms of the GNU General Public License
4  * as published by the Free Software Foundation; either version 2
5  * of the License, or (at your option) any later version.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * along with this program; if not, write to the Free Software Foundation,
14  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15  *
16  * The Original Code is Copyright (C) 2011 Blender Foundation.
17  * All rights reserved.
18  */
19
20 /** \file \ingroup bke
21  *
22  * This file contains implementation of function which are used
23  * by multiple tracking files but which should not be public.
24  */
25
26 #include <stddef.h>
27
28 #include "MEM_guardedalloc.h"
29
30 #include "DNA_movieclip_types.h"
31
32 #include "BLI_utildefines.h"
33 #include "BLI_math.h"
34 #include "BLI_listbase.h"
35 #include "BLI_ghash.h"
36 #include "BLI_string.h"
37 #include "BLI_string_utils.h"
38
39 #include "BLT_translation.h"
40
41 #include "BKE_movieclip.h"
42 #include "BKE_tracking.h"
43
44 #include "IMB_imbuf_types.h"
45 #include "IMB_imbuf.h"
46 #include "IMB_moviecache.h"
47
48 #include "tracking_private.h"
49
50 #include "libmv-capi.h"
51
52 /* Uncomment this to have caching-specific debug prints. */
53 // #define DEBUG_CACHE
54
55 #ifdef DEBUG_CACHE
56 #  define CACHE_PRINTF(...) printf(__VA_ARGS__)
57 #else
58 #  define CACHE_PRINTF(...)
59 #endif
60
61 /*********************** Tracks map *************************/
62
63 TracksMap *tracks_map_new(const char *object_name, bool is_camera, int num_tracks, int customdata_size)
64 {
65         TracksMap *map = MEM_callocN(sizeof(TracksMap), "TrackingsMap");
66
67         BLI_strncpy(map->object_name, object_name, sizeof(map->object_name));
68         map->is_camera = is_camera;
69
70         map->num_tracks = num_tracks;
71         map->customdata_size = customdata_size;
72
73         map->tracks = MEM_callocN(sizeof(MovieTrackingTrack) * num_tracks, "TrackingsMap tracks");
74
75         if (customdata_size)
76                 map->customdata = MEM_callocN(customdata_size * num_tracks, "TracksMap customdata");
77
78         map->hash = BLI_ghash_ptr_new("TracksMap hash");
79
80         BLI_spin_init(&map->spin_lock);
81
82         return map;
83 }
84
85 int tracks_map_get_size(TracksMap *map)
86 {
87         return map->num_tracks;
88 }
89
90 void tracks_map_get_indexed_element(TracksMap *map, int index, MovieTrackingTrack **track, void **customdata)
91 {
92         *track = &map->tracks[index];
93
94         if (map->customdata)
95                 *customdata = &map->customdata[index * map->customdata_size];
96 }
97
98 void tracks_map_insert(TracksMap *map, MovieTrackingTrack *track, void *customdata)
99 {
100         MovieTrackingTrack new_track = *track;
101
102         new_track.markers = MEM_dupallocN(new_track.markers);
103
104         map->tracks[map->ptr] = new_track;
105
106         if (customdata)
107                 memcpy(&map->customdata[map->ptr * map->customdata_size], customdata, map->customdata_size);
108
109         BLI_ghash_insert(map->hash, &map->tracks[map->ptr], track);
110
111         map->ptr++;
112 }
113
114 void tracks_map_merge(TracksMap *map, MovieTracking *tracking)
115 {
116         MovieTrackingTrack *track;
117         ListBase tracks = {NULL, NULL}, new_tracks = {NULL, NULL};
118         ListBase *old_tracks;
119         int a;
120
121         if (map->is_camera) {
122                 old_tracks = &tracking->tracks;
123         }
124         else {
125                 MovieTrackingObject *object = BKE_tracking_object_get_named(tracking, map->object_name);
126
127                 if (!object) {
128                         /* object was deleted by user, create new one */
129                         object = BKE_tracking_object_add(tracking, map->object_name);
130                 }
131
132                 old_tracks = &object->tracks;
133         }
134
135         /* duplicate currently operating tracks to temporary list.
136          * this is needed to keep names in unique state and it's faster to change names
137          * of currently operating tracks (if needed)
138          */
139         for (a = 0; a < map->num_tracks; a++) {
140                 MovieTrackingTrack *old_track;
141                 bool mapped_to_old = false;
142
143                 track = &map->tracks[a];
144
145                 /* find original of operating track in list of previously displayed tracks */
146                 old_track = BLI_ghash_lookup(map->hash, track);
147                 if (old_track) {
148                         if (BLI_findindex(old_tracks, old_track) != -1) {
149                                 BLI_remlink(old_tracks, old_track);
150
151                                 BLI_spin_lock(&map->spin_lock);
152
153                                 /* Copy flags like selection back to the track map. */
154                                 track->flag = old_track->flag;
155                                 track->pat_flag = old_track->pat_flag;
156                                 track->search_flag = old_track->search_flag;
157
158                                 /* Copy all the rest settings back from the map to the actual tracks. */
159                                 MEM_freeN(old_track->markers);
160                                 *old_track = *track;
161                                 old_track->markers = MEM_dupallocN(old_track->markers);
162
163                                 BLI_spin_unlock(&map->spin_lock);
164
165                                 BLI_addtail(&tracks, old_track);
166
167                                 mapped_to_old = true;
168                         }
169                 }
170
171                 if (mapped_to_old == false) {
172                         MovieTrackingTrack *new_track = BKE_tracking_track_duplicate(track);
173
174                         /* Update old-new track mapping */
175                         BLI_ghash_reinsert(map->hash, track, new_track, NULL, NULL);
176
177                         BLI_addtail(&tracks, new_track);
178                 }
179         }
180
181         /* move all tracks, which aren't operating */
182         track = old_tracks->first;
183         while (track) {
184                 MovieTrackingTrack *next = track->next;
185                 BLI_addtail(&new_tracks, track);
186                 track = next;
187         }
188
189         /* now move all tracks which are currently operating and keep their names unique */
190         track = tracks.first;
191         while (track) {
192                 MovieTrackingTrack *next = track->next;
193
194                 BLI_remlink(&tracks, track);
195
196                 track->next = track->prev = NULL;
197                 BLI_addtail(&new_tracks, track);
198
199                 BLI_uniquename(&new_tracks, track, CTX_DATA_(BLT_I18NCONTEXT_ID_MOVIECLIP, "Track"), '.',
200                                offsetof(MovieTrackingTrack, name), sizeof(track->name));
201
202                 track = next;
203         }
204
205         *old_tracks = new_tracks;
206 }
207
208 void tracks_map_free(TracksMap *map, void (*customdata_free)(void *customdata))
209 {
210         int i = 0;
211
212         BLI_ghash_free(map->hash, NULL, NULL);
213
214         for (i = 0; i < map->num_tracks; i++) {
215                 if (map->customdata && customdata_free)
216                         customdata_free(&map->customdata[i * map->customdata_size]);
217
218                 BKE_tracking_track_free(&map->tracks[i]);
219         }
220
221         if (map->customdata)
222                 MEM_freeN(map->customdata);
223
224         MEM_freeN(map->tracks);
225
226         BLI_spin_end(&map->spin_lock);
227
228         MEM_freeN(map);
229 }
230
231 /*********************** Space transformation functions *************************/
232
233 /* Three coordinate frames: Frame, Search, and Marker
234  * Two units: Pixels, Unified
235  * Notation: {coordinate frame}_{unit}; for example, "search_pixel" are search
236  * window relative coordinates in pixels, and "frame_unified" are unified 0..1
237  * coordinates relative to the entire frame.
238  */
239 static void unified_to_pixel(int frame_width, int frame_height,
240                              const float unified_coords[2], float pixel_coords[2])
241 {
242         pixel_coords[0] = unified_coords[0] * frame_width;
243         pixel_coords[1] = unified_coords[1] * frame_height;
244 }
245
246 static void marker_to_frame_unified(const MovieTrackingMarker *marker, const float marker_unified_coords[2],
247                                     float frame_unified_coords[2])
248 {
249         frame_unified_coords[0] = marker_unified_coords[0] + marker->pos[0];
250         frame_unified_coords[1] = marker_unified_coords[1] + marker->pos[1];
251 }
252
253 static void marker_unified_to_frame_pixel_coordinates(int frame_width, int frame_height,
254                                                       const MovieTrackingMarker *marker,
255                                                       const float marker_unified_coords[2],
256                                                       float frame_pixel_coords[2])
257 {
258         marker_to_frame_unified(marker, marker_unified_coords, frame_pixel_coords);
259         unified_to_pixel(frame_width, frame_height, frame_pixel_coords, frame_pixel_coords);
260 }
261
262 void tracking_get_search_origin_frame_pixel(int frame_width, int frame_height,
263                                             const MovieTrackingMarker *marker,
264                                             float frame_pixel[2])
265 {
266         /* Get the lower left coordinate of the search window and snap to pixel coordinates */
267         marker_unified_to_frame_pixel_coordinates(frame_width, frame_height, marker, marker->search_min, frame_pixel);
268         frame_pixel[0] = (int)frame_pixel[0];
269         frame_pixel[1] = (int)frame_pixel[1];
270 }
271
272 static void pixel_to_unified(int frame_width, int frame_height, const float pixel_coords[2], float unified_coords[2])
273 {
274         unified_coords[0] = pixel_coords[0] / frame_width;
275         unified_coords[1] = pixel_coords[1] / frame_height;
276 }
277
278 static void marker_unified_to_search_pixel(int frame_width, int frame_height,
279                                            const MovieTrackingMarker *marker,
280                                            const float marker_unified[2], float search_pixel[2])
281 {
282         float frame_pixel[2];
283         float search_origin_frame_pixel[2];
284
285         marker_unified_to_frame_pixel_coordinates(frame_width, frame_height, marker, marker_unified, frame_pixel);
286         tracking_get_search_origin_frame_pixel(frame_width, frame_height, marker, search_origin_frame_pixel);
287         sub_v2_v2v2(search_pixel, frame_pixel, search_origin_frame_pixel);
288 }
289
290 static void search_pixel_to_marker_unified(int frame_width, int frame_height,
291                                            const MovieTrackingMarker *marker,
292                                            const float search_pixel[2], float marker_unified[2])
293 {
294         float frame_unified[2];
295         float search_origin_frame_pixel[2];
296
297         tracking_get_search_origin_frame_pixel(frame_width, frame_height, marker, search_origin_frame_pixel);
298         add_v2_v2v2(frame_unified, search_pixel, search_origin_frame_pixel);
299         pixel_to_unified(frame_width, frame_height, frame_unified, frame_unified);
300
301         /* marker pos is in frame unified */
302         sub_v2_v2v2(marker_unified, frame_unified, marker->pos);
303 }
304
305 /* Each marker has 5 coordinates associated with it that get warped with
306  * tracking: the four corners ("pattern_corners"), and the center ("pos").
307  * This function puts those 5 points into the appropriate frame for tracking
308  * (the "search" coordinate frame).
309  */
310 void tracking_get_marker_coords_for_tracking(int frame_width, int frame_height,
311                                              const MovieTrackingMarker *marker,
312                                              double search_pixel_x[5], double search_pixel_y[5])
313 {
314         int i;
315         float unified_coords[2];
316         float pixel_coords[2];
317
318         /* Convert the corners into search space coordinates. */
319         for (i = 0; i < 4; i++) {
320                 marker_unified_to_search_pixel(frame_width, frame_height, marker, marker->pattern_corners[i], pixel_coords);
321                 search_pixel_x[i] = pixel_coords[0] - 0.5f;
322                 search_pixel_y[i] = pixel_coords[1] - 0.5f;
323         }
324
325         /* Convert the center position (aka "pos"); this is the origin */
326         unified_coords[0] = 0.0f;
327         unified_coords[1] = 0.0f;
328         marker_unified_to_search_pixel(frame_width, frame_height, marker, unified_coords, pixel_coords);
329
330         search_pixel_x[4] = pixel_coords[0] - 0.5f;
331         search_pixel_y[4] = pixel_coords[1] - 0.5f;
332 }
333
334 /* Inverse of above. */
335 void tracking_set_marker_coords_from_tracking(int frame_width, int frame_height, MovieTrackingMarker *marker,
336                                               const double search_pixel_x[5], const double search_pixel_y[5])
337 {
338         int i;
339         float marker_unified[2];
340         float search_pixel[2];
341
342         /* Convert the corners into search space coordinates. */
343         for (i = 0; i < 4; i++) {
344                 search_pixel[0] = search_pixel_x[i] + 0.5;
345                 search_pixel[1] = search_pixel_y[i] + 0.5;
346                 search_pixel_to_marker_unified(frame_width, frame_height, marker, search_pixel, marker->pattern_corners[i]);
347         }
348
349         /* Convert the center position (aka "pos"); this is the origin */
350         search_pixel[0] = search_pixel_x[4] + 0.5;
351         search_pixel[1] = search_pixel_y[4] + 0.5;
352         search_pixel_to_marker_unified(frame_width, frame_height, marker, search_pixel, marker_unified);
353
354         /* If the tracker tracked nothing, then "marker_unified" would be zero.
355          * Otherwise, the entire patch shifted, and that delta should be applied to
356          * all the coordinates.
357          */
358         for (i = 0; i < 4; i++) {
359                 marker->pattern_corners[i][0] -= marker_unified[0];
360                 marker->pattern_corners[i][1] -= marker_unified[1];
361         }
362
363         marker->pos[0] += marker_unified[0];
364         marker->pos[1] += marker_unified[1];
365 }
366
367 /*********************** General purpose utility functions *************************/
368
369 /* Place a disabled marker before or after specified ref_marker.
370  *
371  * If before is truth, disabled marker is placed before reference
372  * one, and it's placed after it otherwise.
373  *
374  * If there's already a marker at the frame where disabled one
375  * is expected to be placed, nothing will happen if overwrite
376  * is false.
377  */
378 void tracking_marker_insert_disabled(MovieTrackingTrack *track, const MovieTrackingMarker *ref_marker,
379                                      bool before, bool overwrite)
380 {
381         MovieTrackingMarker marker_new;
382
383         marker_new = *ref_marker;
384         marker_new.flag &= ~MARKER_TRACKED;
385         marker_new.flag |= MARKER_DISABLED;
386
387         if (before)
388                 marker_new.framenr--;
389         else
390                 marker_new.framenr++;
391
392         if (overwrite || !BKE_tracking_track_has_marker_at_frame(track, marker_new.framenr))
393                 BKE_tracking_marker_insert(track, &marker_new);
394 }
395
396
397 /* Fill in Libmv C-API camera intrinsics options from tracking structure. */
398 void tracking_cameraIntrinscisOptionsFromTracking(MovieTracking *tracking,
399                                                   int calibration_width, int calibration_height,
400                                                   libmv_CameraIntrinsicsOptions *camera_intrinsics_options)
401 {
402         MovieTrackingCamera *camera = &tracking->camera;
403         float aspy = 1.0f / tracking->camera.pixel_aspect;
404
405         camera_intrinsics_options->focal_length = camera->focal;
406
407         camera_intrinsics_options->principal_point_x = camera->principal[0];
408         camera_intrinsics_options->principal_point_y = camera->principal[1] * aspy;
409
410         switch (camera->distortion_model) {
411                 case TRACKING_DISTORTION_MODEL_POLYNOMIAL:
412                         camera_intrinsics_options->distortion_model = LIBMV_DISTORTION_MODEL_POLYNOMIAL;
413                         camera_intrinsics_options->polynomial_k1 = camera->k1;
414                         camera_intrinsics_options->polynomial_k2 = camera->k2;
415                         camera_intrinsics_options->polynomial_k3 = camera->k3;
416                         camera_intrinsics_options->polynomial_p1 = 0.0;
417                         camera_intrinsics_options->polynomial_p2 = 0.0;
418                         break;
419                 case TRACKING_DISTORTION_MODEL_DIVISION:
420                         camera_intrinsics_options->distortion_model = LIBMV_DISTORTION_MODEL_DIVISION;
421                         camera_intrinsics_options->division_k1 = camera->division_k1;
422                         camera_intrinsics_options->division_k2 = camera->division_k2;
423                         break;
424                 default:
425                         BLI_assert(!"Unknown distortion model");
426                         break;
427         }
428
429         camera_intrinsics_options->image_width = calibration_width;
430         camera_intrinsics_options->image_height = (int) (calibration_height * aspy);
431 }
432
433 void tracking_trackingCameraFromIntrinscisOptions(MovieTracking *tracking,
434                                                   const libmv_CameraIntrinsicsOptions *camera_intrinsics_options)
435 {
436         float aspy = 1.0f / tracking->camera.pixel_aspect;
437         MovieTrackingCamera *camera = &tracking->camera;
438
439         camera->focal = camera_intrinsics_options->focal_length;
440
441         camera->principal[0] = camera_intrinsics_options->principal_point_x;
442         camera->principal[1] = camera_intrinsics_options->principal_point_y / (double) aspy;
443
444         switch (camera_intrinsics_options->distortion_model) {
445                 case LIBMV_DISTORTION_MODEL_POLYNOMIAL:
446                         camera->distortion_model = TRACKING_DISTORTION_MODEL_POLYNOMIAL;
447                         camera->k1 = camera_intrinsics_options->polynomial_k1;
448                         camera->k2 = camera_intrinsics_options->polynomial_k2;
449                         camera->k3 = camera_intrinsics_options->polynomial_k3;
450                         break;
451                 case LIBMV_DISTORTION_MODEL_DIVISION:
452                         camera->distortion_model = TRACKING_DISTORTION_MODEL_DIVISION;
453                         camera->division_k1 = camera_intrinsics_options->division_k1;
454                         camera->division_k2 = camera_intrinsics_options->division_k2;
455                         break;
456                 default:
457                         BLI_assert(!"Unknown distortion model");
458                         break;
459         }
460 }
461
462 /* Get previous keyframed marker. */
463 MovieTrackingMarker *tracking_get_keyframed_marker(MovieTrackingTrack *track,
464                                                    int current_frame,
465                                                    bool backwards)
466 {
467         MovieTrackingMarker *marker_keyed = NULL;
468         MovieTrackingMarker *marker_keyed_fallback = NULL;
469         int a = BKE_tracking_marker_get(track, current_frame) - track->markers;
470
471         while (a >= 0 && a < track->markersnr) {
472                 int next = backwards ? a + 1 : a - 1;
473                 bool is_keyframed = false;
474                 MovieTrackingMarker *cur_marker = &track->markers[a];
475                 MovieTrackingMarker *next_marker = NULL;
476
477                 if (next >= 0 && next < track->markersnr)
478                         next_marker = &track->markers[next];
479
480                 if ((cur_marker->flag & MARKER_DISABLED) == 0) {
481                         /* If it'll happen so we didn't find a real keyframe marker,
482                          * fallback to the first marker in current tracked segment
483                          * as a keyframe.
484                          */
485                         if (next_marker == NULL) {
486                                 /* Could happen when trying to get reference marker for the fist
487                                  * one on the segment which isn't surrounded by disabled markers.
488                                  *
489                                  * There's no really good choice here, just use the reference
490                                  * marker which looks correct..
491                                  */
492                                 if (marker_keyed_fallback == NULL) {
493                                         marker_keyed_fallback = cur_marker;
494                                 }
495                         }
496                         else if (next_marker->flag & MARKER_DISABLED) {
497                                 if (marker_keyed_fallback == NULL)
498                                         marker_keyed_fallback = cur_marker;
499                         }
500
501                         is_keyframed |= (cur_marker->flag & MARKER_TRACKED) == 0;
502                 }
503
504                 if (is_keyframed) {
505                         marker_keyed = cur_marker;
506
507                         break;
508                 }
509
510                 a = next;
511         }
512
513         if (marker_keyed == NULL)
514                 marker_keyed = marker_keyed_fallback;
515
516         return marker_keyed;
517 }
518
519 /*********************** Frame accessr *************************/
520
521 typedef struct AccessCacheKey {
522         int clip_index;
523         int frame;
524         int downscale;
525         libmv_InputMode input_mode;
526         bool has_region;
527         float region_min[2], region_max[2];
528         int64_t transform_key;
529 } AccessCacheKey;
530
531 static unsigned int accesscache_hashhash(const void *key_v)
532 {
533         const AccessCacheKey *key = (const AccessCacheKey *) key_v;
534         /* TODP(sergey): Need better hashing here for faster frame access. */
535         return key->clip_index << 16 | key->frame;
536 }
537
538 static bool accesscache_hashcmp(const void *a_v, const void *b_v)
539 {
540         const AccessCacheKey *a = (const AccessCacheKey *) a_v;
541         const AccessCacheKey *b = (const AccessCacheKey *) b_v;
542         if (a->clip_index != b->clip_index ||
543             a->frame != b->frame ||
544             a->downscale != b->downscale ||
545             a->input_mode != b->input_mode ||
546             a->has_region != b->has_region ||
547             a->transform_key != b->transform_key)
548         {
549                 return true;
550         }
551         /* If there is region applied, compare it. */
552         if (a->has_region) {
553                 if (!equals_v2v2(a->region_min, b->region_min) ||
554                     !equals_v2v2(a->region_max, b->region_max))
555                 {
556                         return true;
557                 }
558         }
559         return false;
560 }
561
562 static void accesscache_construct_key(AccessCacheKey *key,
563                                       int clip_index,
564                                       int frame,
565                                       libmv_InputMode input_mode,
566                                       int downscale,
567                                       const libmv_Region *region,
568                                       int64_t transform_key)
569 {
570         key->clip_index = clip_index;
571         key->frame = frame;
572         key->input_mode = input_mode;
573         key->downscale = downscale;
574         key->has_region = (region != NULL);
575         if (key->has_region) {
576                 copy_v2_v2(key->region_min, region->min);
577                 copy_v2_v2(key->region_max, region->max);
578         }
579         key->transform_key = transform_key;
580 }
581
582 static void accesscache_put(TrackingImageAccessor *accessor,
583                             int clip_index,
584                             int frame,
585                             libmv_InputMode input_mode,
586                             int downscale,
587                             const libmv_Region *region,
588                             int64_t transform_key,
589                             ImBuf *ibuf)
590 {
591         AccessCacheKey key;
592         accesscache_construct_key(&key, clip_index, frame, input_mode, downscale,
593                                   region, transform_key);
594         IMB_moviecache_put(accessor->cache, &key, ibuf);
595 }
596
597 static ImBuf *accesscache_get(TrackingImageAccessor *accessor,
598                               int clip_index,
599                               int frame,
600                               libmv_InputMode input_mode,
601                               int downscale,
602                               const libmv_Region *region,
603                               int64_t transform_key)
604 {
605         AccessCacheKey key;
606         accesscache_construct_key(&key, clip_index, frame, input_mode, downscale,
607                                   region, transform_key);
608         return IMB_moviecache_get(accessor->cache, &key);
609 }
610
611 static ImBuf *accessor_get_preprocessed_ibuf(TrackingImageAccessor *accessor,
612                                              int clip_index,
613                                              int frame)
614 {
615         MovieClip *clip;
616         MovieClipUser user;
617         ImBuf *ibuf;
618         int scene_frame;
619
620         BLI_assert(clip_index < accessor->num_clips);
621
622         clip = accessor->clips[clip_index];
623         scene_frame = BKE_movieclip_remap_clip_to_scene_frame(clip, frame);
624         BKE_movieclip_user_set_frame(&user, scene_frame);
625         user.render_size = MCLIP_PROXY_RENDER_SIZE_FULL;
626         user.render_flag = 0;
627         ibuf = BKE_movieclip_get_ibuf(clip, &user);
628
629         return ibuf;
630 }
631
632 static ImBuf *make_grayscale_ibuf_copy(ImBuf *ibuf)
633 {
634         ImBuf *grayscale = IMB_allocImBuf(ibuf->x, ibuf->y, 32, 0);
635         size_t size;
636         int i;
637
638         BLI_assert(ibuf->channels == 3 || ibuf->channels == 4);
639
640         /* TODO(sergey): Bummer, currently IMB API only allows to create 4 channels
641          * float buffer, so we do it manually here.
642          *
643          * Will generalize it later.
644          */
645         size = (size_t)grayscale->x * (size_t)grayscale->y * sizeof(float);
646         grayscale->channels = 1;
647         if ((grayscale->rect_float = MEM_mapallocN(size, "tracking grayscale image")) != NULL) {
648                 grayscale->mall |= IB_rectfloat;
649                 grayscale->flags |= IB_rectfloat;
650
651                 for (i = 0; i < grayscale->x * grayscale->y; ++i) {
652                         const float *pixel = ibuf->rect_float + ibuf->channels * i;
653
654                         grayscale->rect_float[i] = 0.2126f * pixel[0] +
655                                                    0.7152f * pixel[1] +
656                                                    0.0722f * pixel[2];
657                 }
658         }
659
660         return grayscale;
661 }
662
663 static void ibuf_to_float_image(const ImBuf *ibuf, libmv_FloatImage *float_image)
664 {
665         BLI_assert(ibuf->rect_float != NULL);
666         float_image->buffer = ibuf->rect_float;
667         float_image->width = ibuf->x;
668         float_image->height = ibuf->y;
669         float_image->channels = ibuf->channels;
670 }
671
672 static ImBuf *float_image_to_ibuf(libmv_FloatImage *float_image)
673 {
674         ImBuf *ibuf = IMB_allocImBuf(float_image->width, float_image->height, 32, 0);
675         size_t size = (size_t)ibuf->x * (size_t)ibuf->y * float_image->channels * sizeof(float);
676         ibuf->channels = float_image->channels;
677         if ((ibuf->rect_float = MEM_mapallocN(size, "tracking grayscale image")) != NULL) {
678                 ibuf->mall |= IB_rectfloat;
679                 ibuf->flags |= IB_rectfloat;
680
681                 memcpy(ibuf->rect_float, float_image->buffer, size);
682         }
683         return ibuf;
684 }
685
686 static ImBuf *accessor_get_ibuf(TrackingImageAccessor *accessor,
687                                 int clip_index,
688                                 int frame,
689                                 libmv_InputMode input_mode,
690                                 int downscale,
691                                 const libmv_Region *region,
692                                 const libmv_FrameTransform *transform)
693 {
694         ImBuf *ibuf, *orig_ibuf, *final_ibuf;
695         int64_t transform_key = 0;
696         if (transform != NULL) {
697                 transform_key = libmv_frameAccessorgetTransformKey(transform);
698         }
699         /* First try to get fully processed image from the cache. */
700         BLI_spin_lock(&accessor->cache_lock);
701         ibuf = accesscache_get(accessor,
702                                clip_index,
703                                frame,
704                                input_mode,
705                                downscale,
706                                region,
707                                transform_key);
708         BLI_spin_unlock(&accessor->cache_lock);
709         if (ibuf != NULL) {
710                 CACHE_PRINTF("Used cached buffer for frame %d\n", frame);
711                 /* This is a little heuristic here: if we re-used image once, this is
712                  * a high probability of the image to be related to a keyframe matched
713                  * reference image. Those images we don't want to be thrown away because
714                  * if we toss them out we'll be re-calculating them at the next
715                  * iteration.
716                  */
717                 ibuf->userflags |= IB_PERSISTENT;
718                 return ibuf;
719         }
720         CACHE_PRINTF("Calculate new buffer for frame %d\n", frame);
721         /* And now we do postprocessing of the original frame. */
722         orig_ibuf = accessor_get_preprocessed_ibuf(accessor, clip_index, frame);
723         if (orig_ibuf == NULL) {
724                 return NULL;
725         }
726         /* Cut a region if requested. */
727         if (region != NULL) {
728                 int width = region->max[0] - region->min[0],
729                     height = region->max[1] - region->min[1];
730
731                 /* If the requested region goes outside of the actual frame we still
732                  * return the requested region size, but only fill it's partially with
733                  * the data we can.
734                  */
735                 int clamped_origin_x = max_ii((int)region->min[0], 0),
736                     clamped_origin_y = max_ii((int)region->min[1], 0);
737                 int dst_offset_x = clamped_origin_x - (int)region->min[0],
738                     dst_offset_y = clamped_origin_y - (int)region->min[1];
739                 int clamped_width = width - dst_offset_x,
740                     clamped_height = height - dst_offset_y;
741                 clamped_width = min_ii(clamped_width, orig_ibuf->x - clamped_origin_x);
742                 clamped_height = min_ii(clamped_height, orig_ibuf->y - clamped_origin_y);
743
744                 final_ibuf = IMB_allocImBuf(width, height, 32, IB_rectfloat);
745
746                 if (orig_ibuf->rect_float != NULL) {
747                         IMB_rectcpy(final_ibuf, orig_ibuf,
748                                     dst_offset_x, dst_offset_y,
749                                     clamped_origin_x, clamped_origin_y,
750                                     clamped_width, clamped_height);
751                 }
752                 else {
753                         int y;
754                         /* TODO(sergey): We don't do any color space or alpha conversion
755                          * here. Probably Libmv is better to work in the linear space,
756                          * but keep sRGB space here for compatibility for now.
757                          */
758                         for (y = 0; y < clamped_height; ++y) {
759                                 int x;
760                                 for (x = 0; x < clamped_width; ++x) {
761                                         int src_x = x + clamped_origin_x,
762                                             src_y = y + clamped_origin_y;
763                                         int dst_x = x + dst_offset_x,
764                                             dst_y = y + dst_offset_y;
765                                         int dst_index = (dst_y * width + dst_x) * 4,
766                                             src_index = (src_y * orig_ibuf->x + src_x) * 4;
767                                         rgba_uchar_to_float(final_ibuf->rect_float + dst_index,
768                                                             (unsigned char *)orig_ibuf->rect +
769                                                                              src_index);
770                                 }
771                         }
772                 }
773         }
774         else {
775                 /* Libmv only works with float images,
776                  *
777                  * This would likely make it so loads of float buffers are being stored
778                  * in the cache which is nice on the one hand (faster re-use of the
779                  * frames) but on the other hand it bumps the memory usage up.
780                  */
781                 BLI_thread_lock(LOCK_MOVIECLIP);
782                 IMB_float_from_rect(orig_ibuf);
783                 BLI_thread_unlock(LOCK_MOVIECLIP);
784                 final_ibuf = orig_ibuf;
785         }
786         /* Downscale if needed. */
787         if (downscale > 0) {
788                 if (final_ibuf == orig_ibuf) {
789                         final_ibuf = IMB_dupImBuf(orig_ibuf);
790                 }
791                 IMB_scaleImBuf(final_ibuf,
792                                orig_ibuf->x / (1 << downscale),
793                                orig_ibuf->y / (1 << downscale));
794         }
795         /* Apply possible transformation. */
796         if (transform != NULL) {
797                 libmv_FloatImage input_image, output_image;
798                 ibuf_to_float_image(final_ibuf, &input_image);
799                 libmv_frameAccessorgetTransformRun(transform,
800                                                    &input_image,
801                                                    &output_image);
802                 if (final_ibuf != orig_ibuf) {
803                         IMB_freeImBuf(final_ibuf);
804                 }
805                 final_ibuf = float_image_to_ibuf(&output_image);
806                 libmv_floatImageDestroy(&output_image);
807         }
808         /* Transform number of channels. */
809         if (input_mode == LIBMV_IMAGE_MODE_RGBA) {
810                 BLI_assert(orig_ibuf->channels == 3 || orig_ibuf->channels == 4);
811                 /* pass */
812         }
813         else /* if (input_mode == LIBMV_IMAGE_MODE_MONO) */ {
814                 BLI_assert(input_mode == LIBMV_IMAGE_MODE_MONO);
815                 if (final_ibuf->channels != 1) {
816                         ImBuf *grayscale_ibuf = make_grayscale_ibuf_copy(final_ibuf);
817                         if (final_ibuf != orig_ibuf) {
818                                 /* We dereference original frame later. */
819                                 IMB_freeImBuf(final_ibuf);
820                         }
821                         final_ibuf = grayscale_ibuf;
822                 }
823         }
824         /* It's possible processing still didn't happen at this point,
825          * but we really need a copy of the buffer to be transformed
826          * and to be put to the cache.
827          */
828         if (final_ibuf == orig_ibuf) {
829                 final_ibuf = IMB_dupImBuf(orig_ibuf);
830         }
831         IMB_freeImBuf(orig_ibuf);
832         BLI_spin_lock(&accessor->cache_lock);
833         /* Put final buffer to cache. */
834         accesscache_put(accessor,
835                         clip_index,
836                         frame,
837                         input_mode,
838                         downscale,
839                         region,
840                         transform_key,
841                         final_ibuf);
842         BLI_spin_unlock(&accessor->cache_lock);
843         return final_ibuf;
844 }
845
846 static libmv_CacheKey accessor_get_image_callback(
847         struct libmv_FrameAccessorUserData *user_data,
848         int clip_index,
849         int frame,
850         libmv_InputMode input_mode,
851         int downscale,
852         const libmv_Region *region,
853         const libmv_FrameTransform *transform,
854         float **destination,
855         int *width,
856         int *height,
857         int *channels)
858 {
859         TrackingImageAccessor *accessor = (TrackingImageAccessor *) user_data;
860         ImBuf *ibuf;
861
862         BLI_assert(clip_index >= 0 && clip_index < accessor->num_clips);
863
864         ibuf = accessor_get_ibuf(accessor,
865                                  clip_index,
866                                  frame,
867                                  input_mode,
868                                  downscale,
869                                  region,
870                                  transform);
871
872         if (ibuf) {
873                 *destination = ibuf->rect_float;
874                 *width = ibuf->x;
875                 *height = ibuf->y;
876                 *channels = ibuf->channels;
877         }
878         else {
879                 *destination = NULL;
880                 *width = 0;
881                 *height = 0;
882                 *channels = 0;
883         }
884
885         return ibuf;
886 }
887
888 static void accessor_release_image_callback(libmv_CacheKey cache_key)
889 {
890         ImBuf *ibuf = (ImBuf *) cache_key;
891         IMB_freeImBuf(ibuf);
892 }
893
894 static libmv_CacheKey accessor_get_mask_for_track_callback(
895         libmv_FrameAccessorUserData *user_data,
896         int clip_index,
897         int frame,
898         int track_index,
899         const libmv_Region *region,
900         float **r_destination,
901         int *r_width,
902         int *r_height)
903 {
904         /* Perform sanity checks first. */
905         TrackingImageAccessor *accessor = (TrackingImageAccessor *) user_data;
906         BLI_assert(clip_index < accessor->num_clips);
907         BLI_assert(track_index < accessor->num_tracks);
908         MovieTrackingTrack *track = accessor->tracks[track_index];
909         /* Early output, track does not use mask. */
910         if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) == 0) {
911                 return NULL;
912         }
913         MovieClip *clip = accessor->clips[clip_index];
914         /* Construct fake user so we can access movie clip. */
915         MovieClipUser user;
916         int scene_frame = BKE_movieclip_remap_clip_to_scene_frame(clip, frame);
917         BKE_movieclip_user_set_frame(&user, scene_frame);
918         user.render_size = MCLIP_PROXY_RENDER_SIZE_FULL;
919         user.render_flag = 0;
920         /* Get frame width and height so we can convert stroke coordinates
921          * and other things from normalized to pixel space.
922          */
923         int frame_width, frame_height;
924         BKE_movieclip_get_size(clip, &user, &frame_width, &frame_height);
925         /* Actual mask sampling. */
926         MovieTrackingMarker *marker = BKE_tracking_marker_get_exact(track, frame);
927         const float region_min[2] = {region->min[0] - marker->pos[0] * frame_width,
928                                      region->min[1] - marker->pos[1] * frame_height};
929         const float region_max[2] = {region->max[0] - marker->pos[0] * frame_width,
930                                      region->max[1] - marker->pos[1] * frame_height};
931         *r_destination = tracking_track_get_mask_for_region(frame_width, frame_height,
932                                                             region_min,
933                                                             region_max,
934                                                             track);
935         *r_width = region->max[0] - region->min[0];
936         *r_height = region->max[1] - region->min[1];
937         return *r_destination;
938 }
939
940 static void accessor_release_mask_callback(libmv_CacheKey cache_key)
941 {
942         if (cache_key != NULL) {
943                 float *mask = (float *)cache_key;
944                 MEM_freeN(mask);
945         }
946 }
947
948 TrackingImageAccessor *tracking_image_accessor_new(MovieClip *clips[MAX_ACCESSOR_CLIP],
949                                                    int num_clips,
950                                                    MovieTrackingTrack **tracks,
951                                                    int num_tracks,
952                                                    int start_frame)
953 {
954         TrackingImageAccessor *accessor =
955                 MEM_callocN(sizeof(TrackingImageAccessor), "tracking image accessor");
956
957         BLI_assert(num_clips <= MAX_ACCESSOR_CLIP);
958
959         accessor->cache = IMB_moviecache_create("frame access cache",
960                                                 sizeof(AccessCacheKey),
961                                                 accesscache_hashhash,
962                                                 accesscache_hashcmp);
963
964         memcpy(accessor->clips, clips, num_clips * sizeof(MovieClip *));
965         accessor->num_clips = num_clips;
966         accessor->tracks = tracks;
967         accessor->num_tracks = num_tracks;
968         accessor->start_frame = start_frame;
969
970         accessor->libmv_accessor =
971                 libmv_FrameAccessorNew((libmv_FrameAccessorUserData *) accessor,
972                                        accessor_get_image_callback,
973                                        accessor_release_image_callback,
974                                        accessor_get_mask_for_track_callback,
975                                        accessor_release_mask_callback);
976
977         BLI_spin_init(&accessor->cache_lock);
978
979         return accessor;
980 }
981
982 void tracking_image_accessor_destroy(TrackingImageAccessor *accessor)
983 {
984         IMB_moviecache_free(accessor->cache);
985         libmv_FrameAccessorDestroy(accessor->libmv_accessor);
986         BLI_spin_end(&accessor->cache_lock);
987         MEM_freeN(accessor);
988 }