8b2e45fa09819aeff028bd26a44a7183d76b5bc7
[blender.git] / source / blender / blenkernel / intern / tracking_util.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2011 Blender Foundation.
19  * All rights reserved.
20  *
21  * Contributor(s): Blender Foundation,
22  *                 Sergey Sharybin
23  *                 Keir Mierle
24  *
25  * ***** END GPL LICENSE BLOCK *****
26  */
27
28 /** \file blender/blenkernel/intern/tracking_util.c
29  *  \ingroup bke
30  *
31  * This file contains implementation of function which are used
32  * by multiple tracking files but which should not be public.
33  */
34
35 #include <stddef.h>
36
37 #include "MEM_guardedalloc.h"
38
39 #include "DNA_movieclip_types.h"
40
41 #include "BLI_utildefines.h"
42 #include "BLI_math.h"
43 #include "BLI_listbase.h"
44 #include "BLI_ghash.h"
45 #include "BLI_path_util.h"
46 #include "BLI_string.h"
47
48 #include "BLF_translation.h"
49
50 #include "BKE_movieclip.h"
51 #include "BKE_tracking.h"
52
53 #include "IMB_imbuf_types.h"
54 #include "IMB_imbuf.h"
55 #include "IMB_moviecache.h"
56
57 #include "tracking_private.h"
58
59 #include "libmv-capi.h"
60
61 /*********************** Tracks map *************************/
62
63 TracksMap *tracks_map_new(const char *object_name, bool is_camera, int num_tracks, int customdata_size)
64 {
65         TracksMap *map = MEM_callocN(sizeof(TracksMap), "TrackingsMap");
66
67         BLI_strncpy(map->object_name, object_name, sizeof(map->object_name));
68         map->is_camera = is_camera;
69
70         map->num_tracks = num_tracks;
71         map->customdata_size = customdata_size;
72
73         map->tracks = MEM_callocN(sizeof(MovieTrackingTrack) * num_tracks, "TrackingsMap tracks");
74
75         if (customdata_size)
76                 map->customdata = MEM_callocN(customdata_size * num_tracks, "TracksMap customdata");
77
78         map->hash = BLI_ghash_ptr_new("TracksMap hash");
79
80         BLI_spin_init(&map->spin_lock);
81
82         return map;
83 }
84
85 int tracks_map_get_size(TracksMap *map)
86 {
87         return map->num_tracks;
88 }
89
90 void tracks_map_get_indexed_element(TracksMap *map, int index, MovieTrackingTrack **track, void **customdata)
91 {
92         *track = &map->tracks[index];
93
94         if (map->customdata)
95                 *customdata = &map->customdata[index * map->customdata_size];
96 }
97
98 void tracks_map_insert(TracksMap *map, MovieTrackingTrack *track, void *customdata)
99 {
100         MovieTrackingTrack new_track = *track;
101
102         new_track.markers = MEM_dupallocN(new_track.markers);
103
104         map->tracks[map->ptr] = new_track;
105
106         if (customdata)
107                 memcpy(&map->customdata[map->ptr * map->customdata_size], customdata, map->customdata_size);
108
109         BLI_ghash_insert(map->hash, &map->tracks[map->ptr], track);
110
111         map->ptr++;
112 }
113
114 void tracks_map_merge(TracksMap *map, MovieTracking *tracking)
115 {
116         MovieTrackingTrack *track;
117         ListBase tracks = {NULL, NULL}, new_tracks = {NULL, NULL};
118         ListBase *old_tracks;
119         int a;
120
121         if (map->is_camera) {
122                 old_tracks = &tracking->tracks;
123         }
124         else {
125                 MovieTrackingObject *object = BKE_tracking_object_get_named(tracking, map->object_name);
126
127                 if (!object) {
128                         /* object was deleted by user, create new one */
129                         object = BKE_tracking_object_add(tracking, map->object_name);
130                 }
131
132                 old_tracks = &object->tracks;
133         }
134
135         /* duplicate currently operating tracks to temporary list.
136          * this is needed to keep names in unique state and it's faster to change names
137          * of currently operating tracks (if needed)
138          */
139         for (a = 0; a < map->num_tracks; a++) {
140                 MovieTrackingTrack *old_track;
141                 bool mapped_to_old = false;
142
143                 track = &map->tracks[a];
144
145                 /* find original of operating track in list of previously displayed tracks */
146                 old_track = BLI_ghash_lookup(map->hash, track);
147                 if (old_track) {
148                         if (BLI_findindex(old_tracks, old_track) != -1) {
149                                 BLI_remlink(old_tracks, old_track);
150
151                                 BLI_spin_lock(&map->spin_lock);
152
153                                 /* Copy flags like selection back to the track map. */
154                                 track->flag = old_track->flag;
155                                 track->pat_flag = old_track->pat_flag;
156                                 track->search_flag = old_track->search_flag;
157
158                                 /* Copy all the rest settings back from the map to the actual tracks. */
159                                 MEM_freeN(old_track->markers);
160                                 *old_track = *track;
161                                 old_track->markers = MEM_dupallocN(old_track->markers);
162
163                                 BLI_spin_unlock(&map->spin_lock);
164
165                                 BLI_addtail(&tracks, old_track);
166
167                                 mapped_to_old = true;
168                         }
169                 }
170
171                 if (mapped_to_old == false) {
172                         MovieTrackingTrack *new_track = BKE_tracking_track_duplicate(track);
173
174                         /* Update old-new track mapping */
175                         BLI_ghash_remove(map->hash, track, NULL, NULL);
176                         BLI_ghash_insert(map->hash, track, new_track);
177
178                         BLI_addtail(&tracks, new_track);
179                 }
180         }
181
182         /* move all tracks, which aren't operating */
183         track = old_tracks->first;
184         while (track) {
185                 MovieTrackingTrack *next = track->next;
186                 BLI_addtail(&new_tracks, track);
187                 track = next;
188         }
189
190         /* now move all tracks which are currently operating and keep their names unique */
191         track = tracks.first;
192         while (track) {
193                 MovieTrackingTrack *next = track->next;
194
195                 BLI_remlink(&tracks, track);
196
197                 track->next = track->prev = NULL;
198                 BLI_addtail(&new_tracks, track);
199
200                 BLI_uniquename(&new_tracks, track, CTX_DATA_(BLF_I18NCONTEXT_ID_MOVIECLIP, "Track"), '.',
201                                offsetof(MovieTrackingTrack, name), sizeof(track->name));
202
203                 track = next;
204         }
205
206         *old_tracks = new_tracks;
207 }
208
209 void tracks_map_free(TracksMap *map, void (*customdata_free)(void *customdata))
210 {
211         int i = 0;
212
213         BLI_ghash_free(map->hash, NULL, NULL);
214
215         for (i = 0; i < map->num_tracks; i++) {
216                 if (map->customdata && customdata_free)
217                         customdata_free(&map->customdata[i * map->customdata_size]);
218
219                 BKE_tracking_track_free(&map->tracks[i]);
220         }
221
222         if (map->customdata)
223                 MEM_freeN(map->customdata);
224
225         MEM_freeN(map->tracks);
226
227         BLI_spin_end(&map->spin_lock);
228
229         MEM_freeN(map);
230 }
231
232 /*********************** Space transformation functions *************************/
233
234 /* Three coordinate frames: Frame, Search, and Marker
235  * Two units: Pixels, Unified
236  * Notation: {coordinate frame}_{unit}; for example, "search_pixel" are search
237  * window relative coordinates in pixels, and "frame_unified" are unified 0..1
238  * coordinates relative to the entire frame.
239  */
240 static void unified_to_pixel(int frame_width, int frame_height,
241                              const float unified_coords[2], float pixel_coords[2])
242 {
243         pixel_coords[0] = unified_coords[0] * frame_width;
244         pixel_coords[1] = unified_coords[1] * frame_height;
245 }
246
247 static void marker_to_frame_unified(const MovieTrackingMarker *marker, const float marker_unified_coords[2],
248                                     float frame_unified_coords[2])
249 {
250         frame_unified_coords[0] = marker_unified_coords[0] + marker->pos[0];
251         frame_unified_coords[1] = marker_unified_coords[1] + marker->pos[1];
252 }
253
254 static void marker_unified_to_frame_pixel_coordinates(int frame_width, int frame_height,
255                                                       const MovieTrackingMarker *marker,
256                                                       const float marker_unified_coords[2],
257                                                       float frame_pixel_coords[2])
258 {
259         marker_to_frame_unified(marker, marker_unified_coords, frame_pixel_coords);
260         unified_to_pixel(frame_width, frame_height, frame_pixel_coords, frame_pixel_coords);
261 }
262
263 void tracking_get_search_origin_frame_pixel(int frame_width, int frame_height,
264                                             const MovieTrackingMarker *marker,
265                                             float frame_pixel[2])
266 {
267         /* Get the lower left coordinate of the search window and snap to pixel coordinates */
268         marker_unified_to_frame_pixel_coordinates(frame_width, frame_height, marker, marker->search_min, frame_pixel);
269         frame_pixel[0] = (int)frame_pixel[0];
270         frame_pixel[1] = (int)frame_pixel[1];
271 }
272
273 static void pixel_to_unified(int frame_width, int frame_height, const float pixel_coords[2], float unified_coords[2])
274 {
275         unified_coords[0] = pixel_coords[0] / frame_width;
276         unified_coords[1] = pixel_coords[1] / frame_height;
277 }
278
279 static void marker_unified_to_search_pixel(int frame_width, int frame_height,
280                                            const MovieTrackingMarker *marker,
281                                            const float marker_unified[2], float search_pixel[2])
282 {
283         float frame_pixel[2];
284         float search_origin_frame_pixel[2];
285
286         marker_unified_to_frame_pixel_coordinates(frame_width, frame_height, marker, marker_unified, frame_pixel);
287         tracking_get_search_origin_frame_pixel(frame_width, frame_height, marker, search_origin_frame_pixel);
288         sub_v2_v2v2(search_pixel, frame_pixel, search_origin_frame_pixel);
289 }
290
291 static void search_pixel_to_marker_unified(int frame_width, int frame_height,
292                                            const MovieTrackingMarker *marker,
293                                            const float search_pixel[2], float marker_unified[2])
294 {
295         float frame_unified[2];
296         float search_origin_frame_pixel[2];
297
298         tracking_get_search_origin_frame_pixel(frame_width, frame_height, marker, search_origin_frame_pixel);
299         add_v2_v2v2(frame_unified, search_pixel, search_origin_frame_pixel);
300         pixel_to_unified(frame_width, frame_height, frame_unified, frame_unified);
301
302         /* marker pos is in frame unified */
303         sub_v2_v2v2(marker_unified, frame_unified, marker->pos);
304 }
305
306 /* Each marker has 5 coordinates associated with it that get warped with
307  * tracking: the four corners ("pattern_corners"), and the center ("pos").
308  * This function puts those 5 points into the appropriate frame for tracking
309  * (the "search" coordinate frame).
310  */
311 void tracking_get_marker_coords_for_tracking(int frame_width, int frame_height,
312                                              const MovieTrackingMarker *marker,
313                                              double search_pixel_x[5], double search_pixel_y[5])
314 {
315         int i;
316         float unified_coords[2];
317         float pixel_coords[2];
318
319         /* Convert the corners into search space coordinates. */
320         for (i = 0; i < 4; i++) {
321                 marker_unified_to_search_pixel(frame_width, frame_height, marker, marker->pattern_corners[i], pixel_coords);
322                 search_pixel_x[i] = pixel_coords[0] - 0.5f;
323                 search_pixel_y[i] = pixel_coords[1] - 0.5f;
324         }
325
326         /* Convert the center position (aka "pos"); this is the origin */
327         unified_coords[0] = 0.0f;
328         unified_coords[1] = 0.0f;
329         marker_unified_to_search_pixel(frame_width, frame_height, marker, unified_coords, pixel_coords);
330
331         search_pixel_x[4] = pixel_coords[0] - 0.5f;
332         search_pixel_y[4] = pixel_coords[1] - 0.5f;
333 }
334
335 /* Inverse of above. */
336 void tracking_set_marker_coords_from_tracking(int frame_width, int frame_height, MovieTrackingMarker *marker,
337                                               const double search_pixel_x[5], const double search_pixel_y[5])
338 {
339         int i;
340         float marker_unified[2];
341         float search_pixel[2];
342
343         /* Convert the corners into search space coordinates. */
344         for (i = 0; i < 4; i++) {
345                 search_pixel[0] = search_pixel_x[i] + 0.5;
346                 search_pixel[1] = search_pixel_y[i] + 0.5;
347                 search_pixel_to_marker_unified(frame_width, frame_height, marker, search_pixel, marker->pattern_corners[i]);
348         }
349
350         /* Convert the center position (aka "pos"); this is the origin */
351         search_pixel[0] = search_pixel_x[4] + 0.5;
352         search_pixel[1] = search_pixel_y[4] + 0.5;
353         search_pixel_to_marker_unified(frame_width, frame_height, marker, search_pixel, marker_unified);
354
355         /* If the tracker tracked nothing, then "marker_unified" would be zero.
356          * Otherwise, the entire patch shifted, and that delta should be applied to
357          * all the coordinates.
358          */
359         for (i = 0; i < 4; i++) {
360                 marker->pattern_corners[i][0] -= marker_unified[0];
361                 marker->pattern_corners[i][1] -= marker_unified[1];
362         }
363
364         marker->pos[0] += marker_unified[0];
365         marker->pos[1] += marker_unified[1];
366 }
367
368 /*********************** General purpose utility functions *************************/
369
370 /* Place a disabled marker before or after specified ref_marker.
371  *
372  * If before is truth, disabled marker is placed before reference
373  * one, and it's placed after it otherwise.
374  *
375  * If there's already a marker at the frame where disabled one
376  * is expected to be placed, nothing will happen if overwrite
377  * is false.
378  */
379 void tracking_marker_insert_disabled(MovieTrackingTrack *track, const MovieTrackingMarker *ref_marker,
380                                      bool before, bool overwrite)
381 {
382         MovieTrackingMarker marker_new;
383
384         marker_new = *ref_marker;
385         marker_new.flag &= ~MARKER_TRACKED;
386         marker_new.flag |= MARKER_DISABLED;
387
388         if (before)
389                 marker_new.framenr--;
390         else
391                 marker_new.framenr++;
392
393         if (overwrite || !BKE_tracking_track_has_marker_at_frame(track, marker_new.framenr))
394                 BKE_tracking_marker_insert(track, &marker_new);
395 }
396
397
398 /* Fill in Libmv C-API camera intrinsics options from tracking structure. */
399 void tracking_cameraIntrinscisOptionsFromTracking(MovieTracking *tracking,
400                                                   int calibration_width, int calibration_height,
401                                                   libmv_CameraIntrinsicsOptions *camera_intrinsics_options)
402 {
403         MovieTrackingCamera *camera = &tracking->camera;
404         float aspy = 1.0f / tracking->camera.pixel_aspect;
405
406         camera_intrinsics_options->focal_length = camera->focal;
407
408         camera_intrinsics_options->principal_point_x = camera->principal[0];
409         camera_intrinsics_options->principal_point_y = camera->principal[1] * aspy;
410
411         switch (camera->distortion_model) {
412                 case TRACKING_DISTORTION_MODEL_POLYNOMIAL:
413                         camera_intrinsics_options->distortion_model = LIBMV_DISTORTION_MODEL_POLYNOMIAL;
414                         camera_intrinsics_options->polynomial_k1 = camera->k1;
415                         camera_intrinsics_options->polynomial_k2 = camera->k2;
416                         camera_intrinsics_options->polynomial_k3 = camera->k3;
417                         camera_intrinsics_options->polynomial_p1 = 0.0;
418                         camera_intrinsics_options->polynomial_p2 = 0.0;
419                         break;
420                 case TRACKING_DISTORTION_MODEL_DIVISION:
421                         camera_intrinsics_options->distortion_model = LIBMV_DISTORTION_MODEL_DIVISION;
422                         camera_intrinsics_options->division_k1 = camera->division_k1;
423                         camera_intrinsics_options->division_k2 = camera->division_k2;
424                         break;
425                 default:
426                         BLI_assert(!"Unknown distortion model");
427         }
428
429         camera_intrinsics_options->image_width = calibration_width;
430         camera_intrinsics_options->image_height = (int) (calibration_height * aspy);
431 }
432
433 void tracking_trackingCameraFromIntrinscisOptions(MovieTracking *tracking,
434                                                   const libmv_CameraIntrinsicsOptions *camera_intrinsics_options)
435 {
436         float aspy = 1.0f / tracking->camera.pixel_aspect;
437         MovieTrackingCamera *camera = &tracking->camera;
438
439         camera->focal = camera_intrinsics_options->focal_length;
440
441         camera->principal[0] = camera_intrinsics_options->principal_point_x;
442         camera->principal[1] = camera_intrinsics_options->principal_point_y / (double) aspy;
443
444         switch (camera_intrinsics_options->distortion_model) {
445                 case LIBMV_DISTORTION_MODEL_POLYNOMIAL:
446                         camera->distortion_model = TRACKING_DISTORTION_MODEL_POLYNOMIAL;
447                         camera->k1 = camera_intrinsics_options->polynomial_k1;
448                         camera->k2 = camera_intrinsics_options->polynomial_k2;
449                         camera->k3 = camera_intrinsics_options->polynomial_k3;
450                         break;
451                 case LIBMV_DISTORTION_MODEL_DIVISION:
452                         camera->distortion_model = TRACKING_DISTORTION_MODEL_DIVISION;
453                         camera->division_k1 = camera_intrinsics_options->division_k1;
454                         camera->division_k2 = camera_intrinsics_options->division_k2;
455                         break;
456                 default:
457                         BLI_assert(!"Unknown distortion model");
458         }
459 }
460
461 /* Get previous keyframed marker. */
462 MovieTrackingMarker *tracking_get_keyframed_marker(MovieTrackingTrack *track,
463                                                    int current_frame,
464                                                    bool backwards)
465 {
466         MovieTrackingMarker *marker_keyed = NULL;
467         MovieTrackingMarker *marker_keyed_fallback = NULL;
468         int a = BKE_tracking_marker_get(track, current_frame) - track->markers;
469
470         while (a >= 0 && a < track->markersnr) {
471                 int next = backwards ? a + 1 : a - 1;
472                 bool is_keyframed = false;
473                 MovieTrackingMarker *cur_marker = &track->markers[a];
474                 MovieTrackingMarker *next_marker = NULL;
475
476                 if (next >= 0 && next < track->markersnr)
477                         next_marker = &track->markers[next];
478
479                 if ((cur_marker->flag & MARKER_DISABLED) == 0) {
480                         /* If it'll happen so we didn't find a real keyframe marker,
481                          * fallback to the first marker in current tracked segment
482                          * as a keyframe.
483                          */
484                         if (next_marker && next_marker->flag & MARKER_DISABLED) {
485                                 if (marker_keyed_fallback == NULL)
486                                         marker_keyed_fallback = cur_marker;
487                         }
488
489                         is_keyframed |= (cur_marker->flag & MARKER_TRACKED) == 0;
490                 }
491
492                 if (is_keyframed) {
493                         marker_keyed = cur_marker;
494
495                         break;
496                 }
497
498                 a = next;
499         }
500
501         if (marker_keyed == NULL)
502                 marker_keyed = marker_keyed_fallback;
503
504         return marker_keyed;
505 }
506
507 /*********************** Frame accessr *************************/
508
509 typedef struct AccessCacheKey {
510         int clip_index;
511         int frame;
512         int downscale;
513         libmv_InputMode input_mode;
514         int64_t transform_key;
515 } AccessCacheKey;
516
517 static unsigned int accesscache_hashhash(const void *key_v)
518 {
519         const AccessCacheKey *key = (const AccessCacheKey *) key_v;
520         /* TODP(sergey): Need better hasing here for faster frame access. */
521         return key->clip_index << 16 | key->frame;
522 }
523
524 static bool accesscache_hashcmp(const void *a_v, const void *b_v)
525 {
526         const AccessCacheKey *a = (const AccessCacheKey *) a_v;
527         const AccessCacheKey *b = (const AccessCacheKey *) b_v;
528
529 #define COMPARE_FIELD(field)
530         { \
531                 if (a->clip_index != b->clip_index) { \
532                         return false; \
533                 } \
534         } (void) 0
535
536         COMPARE_FIELD(clip_index);
537         COMPARE_FIELD(frame);
538         COMPARE_FIELD(downscale);
539         COMPARE_FIELD(input_mode);
540         COMPARE_FIELD(transform_key);
541
542 #undef COMPARE_FIELD
543
544         return true;
545 }
546
547 static void accesscache_put(TrackingImageAccessor *accessor,
548                             int clip_index,
549                             int frame,
550                             libmv_InputMode input_mode,
551                             int downscale,
552                             int64_t transform_key,
553                             ImBuf *ibuf)
554 {
555         AccessCacheKey key;
556         key.clip_index = clip_index;
557         key.frame = frame;
558         key.input_mode = input_mode;
559         key.downscale = downscale;
560         key.transform_key = transform_key;
561         IMB_moviecache_put(accessor->cache, &key, ibuf);
562 }
563
564 static ImBuf *accesscache_get(TrackingImageAccessor *accessor,
565                               int clip_index,
566                               int frame,
567                               libmv_InputMode input_mode,
568                               int downscale,
569                               int64_t transform_key)
570 {
571         AccessCacheKey key;
572         key.clip_index = clip_index;
573         key.frame = frame;
574         key.input_mode = input_mode;
575         key.downscale = downscale;
576         key.transform_key = transform_key;
577         return IMB_moviecache_get(accessor->cache, &key);
578 }
579
580 static ImBuf *accessor_get_preprocessed_ibuf(TrackingImageAccessor *accessor,
581                                              int clip_index,
582                                              int frame)
583 {
584         MovieClip *clip;
585         MovieClipUser user;
586         ImBuf *ibuf;
587         int scene_frame;
588
589         BLI_assert(clip_index < accessor->num_clips);
590
591         clip = accessor->clips[clip_index];
592         scene_frame = BKE_movieclip_remap_clip_to_scene_frame(clip, frame);
593         BKE_movieclip_user_set_frame(&user, scene_frame);
594         user.render_size = MCLIP_PROXY_RENDER_SIZE_FULL;
595         user.render_flag = 0;
596         ibuf = BKE_movieclip_get_ibuf(clip, &user);
597
598         return ibuf;
599 }
600
601 static ImBuf *make_grayscale_ibuf_copy(ImBuf *ibuf)
602 {
603         ImBuf *grayscale = IMB_allocImBuf(ibuf->x, ibuf->y, 32, 0);
604         size_t size;
605         int i;
606
607         BLI_assert(ibuf->channels == 3 || ibuf->channels == 4);
608
609         /* TODO(sergey): Bummer, currently IMB API only allows to create 4 channels
610          * float buffer, so we do it manually here.
611          *
612          * Will generalize it later.
613          */
614         size = (size_t)grayscale->x * (size_t)grayscale->y * sizeof(float);
615         grayscale->channels = 1;
616         if ((grayscale->rect_float = MEM_mapallocN(size, "tracking grayscale image"))) {
617                 grayscale->mall |= IB_rectfloat;
618                 grayscale->flags |= IB_rectfloat;
619         }
620
621         for (i = 0; i < grayscale->x * grayscale->y; ++i) {
622                 const float *pixel = ibuf->rect_float + ibuf->channels * i;
623
624                 grayscale->rect_float[i] = 0.2126f * pixel[0] +
625                                            0.7152f * pixel[1] +
626                                            0.0722f * pixel[2];
627         }
628
629         return grayscale;
630 }
631
632 static void ibuf_to_float_image(const ImBuf *ibuf, libmv_FloatImage *float_image)
633 {
634         BLI_assert(ibuf->rect_float != NULL);
635         float_image->buffer = ibuf->rect_float;
636         float_image->width = ibuf->x;
637         float_image->height = ibuf->y;
638         float_image->channels = ibuf->channels;
639 }
640
641 static ImBuf *float_image_to_ibuf(libmv_FloatImage *float_image)
642 {
643         ImBuf *ibuf = IMB_allocImBuf(float_image->width, float_image->height, 32, 0);
644         size_t size = (size_t)ibuf->x * (size_t)ibuf->y *
645                       float_image->channels * sizeof(float);
646         ibuf->channels = float_image->channels;
647         if ((ibuf->rect_float = MEM_mapallocN(size, "tracking grayscale image"))) {
648                 ibuf->mall |= IB_rectfloat;
649                 ibuf->flags |= IB_rectfloat;
650         }
651         memcpy(ibuf->rect_float, float_image->buffer, size);
652         return ibuf;
653 }
654
655 static ImBuf *accessor_get_ibuf(TrackingImageAccessor *accessor,
656                                 int clip_index,
657                                 int frame,
658                                 libmv_InputMode input_mode,
659                                 int downscale,
660                                 const libmv_Region *region,
661                                 const libmv_FrameTransform *transform)
662 {
663         ImBuf *ibuf, *orig_ibuf, *final_ibuf;
664         int64_t transform_key = 0;
665
666         if (transform != NULL) {
667                 transform_key = libmv_frameAccessorgetTransformKey(transform);
668         }
669
670         /* First try to get fully processed image from the cache. */
671         ibuf = accesscache_get(accessor,
672                                clip_index,
673                                frame,
674                                input_mode,
675                                downscale,
676                                transform_key);
677         if (ibuf != NULL) {
678                 return ibuf;
679         }
680
681         /* And now we do postprocessing of the original frame. */
682         orig_ibuf = accessor_get_preprocessed_ibuf(accessor, clip_index, frame);
683
684         if (orig_ibuf == NULL) {
685                 return NULL;
686         }
687
688         if (region != NULL) {
689                 int width = region->max[0] - region->min[0],
690                     height = region->max[1] - region->min[1];
691
692                 /* If the requested region goes outside of the actual frame we still
693                  * return the requested region size, but only fill it's partially with
694                  * the data we can.
695                  */
696                 int clamped_origin_x = max_ii((int)region->min[0], 0),
697                         clamped_origin_y = max_ii((int)region->min[1], 0);
698                 int clamped_width = min_ii(width, orig_ibuf->x - region->min[0] - 1),
699                         clamped_height = min_ii(height, orig_ibuf->y - region->min[1] - 1);
700
701                 final_ibuf = IMB_allocImBuf(width, height, 32, IB_rectfloat);
702
703                 if (orig_ibuf->rect_float != NULL) {
704                         IMB_rectcpy(final_ibuf, orig_ibuf,
705                                     0, 0,
706                                     region->min[0], region->min[1],
707                                     clamped_width, clamped_height);
708                 }
709                 else {
710                         int y;
711                         /* TODO(sergey): We don't do any color space or alpha conversion
712                          * here. Probably Libmv is better to work in the linear space,
713                          * but keep sRGB space here for compatibility for now.
714                          */
715                         for (y = 0; y < clamped_height; ++y) {
716                                 int x;
717                                 for (x = 0; x < clamped_width; ++x) {
718                                         int src_x = x + clamped_origin_x,
719                                             src_y = y + clamped_origin_y;
720                                         int dst_index = (y * width + x) * 4,
721                                                 src_index = (src_y * orig_ibuf->x + src_x) * 4;
722                                         rgba_uchar_to_float(final_ibuf->rect_float + dst_index,
723                                                             (unsigned char *)orig_ibuf->rect +
724                                                                              src_index);
725                                 }
726                         }
727                 }
728         }
729         else {
730                 /* Libmv only works with float images,
731                  *
732                  * This would likely make it so loads of float buffers are being stored
733                  * in the cache which is nice on the one hand (faster re-use of the
734                  * frames) but on the other hand it bumps the memory usage up.
735                  */
736                 BLI_lock_thread(LOCK_MOVIECLIP);
737                 IMB_float_from_rect(orig_ibuf);
738                 BLI_unlock_thread(LOCK_MOVIECLIP);
739                 final_ibuf = orig_ibuf;
740         }
741
742         if (downscale > 0) {
743                 if (final_ibuf == orig_ibuf) {
744                         final_ibuf = IMB_dupImBuf(orig_ibuf);
745                 }
746                 IMB_scaleImBuf(final_ibuf,
747                                ibuf->x / (1 << downscale),
748                                ibuf->y / (1 << downscale));
749         }
750
751         if (input_mode == LIBMV_IMAGE_MODE_RGBA) {
752                 BLI_assert(ibuf->channels == 3 || ibuf->channels == 4);
753                 /* pass */
754         }
755         else /* if (input_mode == LIBMV_IMAGE_MODE_MONO) */ {
756                 ImBuf *grayscale_ibuf = make_grayscale_ibuf_copy(final_ibuf);
757                 if (final_ibuf != orig_ibuf) {
758                         /* We dereference original frame later. */
759                         IMB_freeImBuf(final_ibuf);
760                 }
761                 final_ibuf = grayscale_ibuf;
762         }
763
764         if (transform != NULL) {
765                 libmv_FloatImage input_image, output_image;
766                 ibuf_to_float_image(final_ibuf, &input_image);
767                 libmv_frameAccessorgetTransformRun(transform,
768                                                    &input_image,
769                                                    &output_image);
770                 if (final_ibuf != orig_ibuf) {
771                         IMB_freeImBuf(final_ibuf);
772                 }
773                 final_ibuf = float_image_to_ibuf(&output_image);
774                 libmv_floatImaheDestroy(&output_image);
775         }
776
777         /* it's possible processing stil didn't happen at this point,
778          * but we really need a copy of the buffer to be transformed
779          * and to be put to the cache.
780          */
781         if (final_ibuf == orig_ibuf) {
782                 final_ibuf = IMB_dupImBuf(orig_ibuf);
783         }
784
785         IMB_freeImBuf(orig_ibuf);
786
787         /* We put postprocessed frame to the cache always for now,
788          * not the smartest thing in the world, but who cares at this point.
789          */
790
791         /* TODO(sergey): Disable cache for now, because we don't store region
792          * in the cache key and can't check whether cached version is usable for
793          * us or not.
794          *
795          * Need to think better about what to cache and when.
796          */
797         if (false) {
798                 accesscache_put(accessor,
799                                 clip_index,
800                                 frame,
801                                 input_mode,
802                                 downscale,
803                                 transform_key,
804                                 final_ibuf);
805         }
806
807         return final_ibuf;
808 }
809
810 static libmv_CacheKey accessor_get_image_callback(
811                 struct libmv_FrameAccessorUserData *user_data,
812                 int clip_index,
813                 int frame,
814                 libmv_InputMode input_mode,
815                 int downscale,
816                 const libmv_Region *region,
817                 const libmv_FrameTransform *transform,
818                 float **destination,
819                 int *width,
820                 int *height,
821                 int *channels)
822 {
823         TrackingImageAccessor *accessor = (TrackingImageAccessor *) user_data;
824         ImBuf *ibuf;
825
826         BLI_assert(clip_index >= 0 && clip_index < accessor->num_clips);
827
828         ibuf = accessor_get_ibuf(accessor,
829                                  clip_index,
830                                  frame,
831                                  input_mode,
832                                  downscale,
833                                  region,
834                                  transform);
835
836         if (ibuf) {
837                 *destination = ibuf->rect_float;
838                 *width = ibuf->x;
839                 *height = ibuf->y;
840                 *channels = ibuf->channels;
841         }
842         else {
843                 *destination = NULL;
844                 *width = 0;
845                 *height = 0;
846                 *channels = 0;
847         }
848
849         return ibuf;
850 }
851
852 static void accessor_release_image_callback(libmv_CacheKey cache_key)
853 {
854         ImBuf *ibuf = (ImBuf *) cache_key;
855         IMB_freeImBuf(ibuf);
856 }
857
858 TrackingImageAccessor *tracking_image_accessor_new(MovieClip *clips[MAX_ACCESSOR_CLIP],
859                                                    int num_clips,
860                                                    int start_frame)
861 {
862         TrackingImageAccessor *accessor =
863                 MEM_callocN(sizeof(TrackingImageAccessor), "tracking image accessor");
864
865         BLI_assert(num_clips <= MAX_ACCESSOR_CLIP);
866
867         accessor->cache = IMB_moviecache_create("frame access cache",
868                                                 sizeof(AccessCacheKey),
869                                                 accesscache_hashhash,
870                                                 accesscache_hashcmp);
871
872         memcpy(accessor->clips, clips, num_clips * sizeof(MovieClip*));
873         accessor->num_clips = num_clips;
874         accessor->start_frame = start_frame;
875
876         accessor->libmv_accessor =
877                 libmv_FrameAccessorNew((libmv_FrameAccessorUserData *) accessor,
878                                        accessor_get_image_callback,
879                                        accessor_release_image_callback);
880
881         return accessor;
882 }
883
884 void tracking_image_accessor_destroy(TrackingImageAccessor *accessor)
885 {
886         IMB_moviecache_free(accessor->cache);
887         libmv_FrameAccessorDestroy(accessor->libmv_accessor);
888         MEM_freeN(accessor);
889 }