doxygen: add newline after \file
[blender.git] / source / blender / blenkernel / intern / tracking_region_tracker.c
1 /*
2  * This program is free software; you can redistribute it and/or
3  * modify it under the terms of the GNU General Public License
4  * as published by the Free Software Foundation; either version 2
5  * of the License, or (at your option) any later version.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * along with this program; if not, write to the Free Software Foundation,
14  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15  *
16  * The Original Code is Copyright (C) 2011 Blender Foundation.
17  * All rights reserved.
18  */
19
20 /** \file
21  * \ingroup bke
22  *
23  * This file contains implementation of blender-side region tracker
24  * which is used for 2D feature tracking.
25  */
26
27 #include "MEM_guardedalloc.h"
28
29 #include "DNA_movieclip_types.h"
30
31 #include "BLI_utildefines.h"
32 #include "BLI_threads.h"
33
34 #include "BKE_tracking.h"
35 #include "BKE_movieclip.h"
36
37 #include "IMB_imbuf_types.h"
38 #include "IMB_imbuf.h"
39
40 #include "libmv-capi.h"
41 #include "tracking_private.h"
42
43 /* **** utility functions for tracking **** */
44
45 /* convert from float and byte RGBA to grayscale. Supports different coefficients for RGB. */
46 static void float_rgba_to_gray(const float *rgba, float *gray, int num_pixels,
47                                float weight_red, float weight_green, float weight_blue)
48 {
49         int i;
50
51         for (i = 0; i < num_pixels; i++) {
52                 const float *pixel = rgba + 4 * i;
53
54                 gray[i] = weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2];
55         }
56 }
57
58 static void uint8_rgba_to_float_gray(const unsigned char *rgba, float *gray, int num_pixels,
59                                      float weight_red, float weight_green, float weight_blue)
60 {
61         int i;
62
63         for (i = 0; i < num_pixels; i++) {
64                 const unsigned char *pixel = rgba + i * 4;
65
66                 gray[i] = (weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2]) / 255.0f;
67         }
68 }
69
70 /* Get grayscale float search buffer for given marker and frame. */
71 static float *track_get_search_floatbuf(ImBuf *ibuf, MovieTrackingTrack *track, MovieTrackingMarker *marker,
72                                         int *width_r, int *height_r)
73 {
74         ImBuf *searchibuf;
75         float *gray_pixels;
76         int width, height;
77
78         searchibuf = BKE_tracking_get_search_imbuf(ibuf, track, marker, false, true);
79
80         if (!searchibuf) {
81                 *width_r = 0;
82                 *height_r = 0;
83                 return NULL;
84         }
85
86         width = searchibuf->x;
87         height = searchibuf->y;
88
89         gray_pixels = MEM_callocN(width * height * sizeof(float), "tracking floatBuf");
90
91         if (searchibuf->rect_float) {
92                 float_rgba_to_gray(searchibuf->rect_float, gray_pixels, width * height,
93                                    0.2126f, 0.7152f, 0.0722f);
94         }
95         else {
96                 uint8_rgba_to_float_gray((unsigned char *)searchibuf->rect, gray_pixels, width * height,
97                                          0.2126f, 0.7152f, 0.0722f);
98         }
99
100         IMB_freeImBuf(searchibuf);
101
102         *width_r = width;
103         *height_r = height;
104
105         return gray_pixels;
106 }
107
108 /* Get image buffer for a given frame
109  *
110  * Frame is in clip space.
111  */
112 static ImBuf *tracking_context_get_frame_ibuf(MovieClip *clip, MovieClipUser *user, int clip_flag, int framenr)
113 {
114         ImBuf *ibuf;
115         MovieClipUser new_user = *user;
116
117         new_user.framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, framenr);
118
119         ibuf = BKE_movieclip_get_ibuf_flag(clip, &new_user, clip_flag, MOVIECLIP_CACHE_SKIP);
120
121         return ibuf;
122 }
123
124 /* Get image buffer for previous marker's keyframe. */
125 static ImBuf *tracking_context_get_keyframed_ibuf(MovieClip *clip, MovieClipUser *user, int clip_flag,
126                                                   MovieTrackingTrack *track, int curfra, bool backwards,
127                                                   MovieTrackingMarker **marker_keyed_r)
128 {
129         MovieTrackingMarker *marker_keyed;
130         int keyed_framenr;
131
132         marker_keyed = tracking_get_keyframed_marker(track, curfra, backwards);
133         if (marker_keyed == NULL) {
134                 return NULL;
135         }
136
137         keyed_framenr = marker_keyed->framenr;
138
139         *marker_keyed_r = marker_keyed;
140
141         return tracking_context_get_frame_ibuf(clip, user, clip_flag, keyed_framenr);
142 }
143
144 /* Get image buffer which si used as reference for track. */
145 static ImBuf *tracking_context_get_reference_ibuf(MovieClip *clip, MovieClipUser *user, int clip_flag,
146                                                   MovieTrackingTrack *track, int curfra, bool backwards,
147                                                   MovieTrackingMarker **reference_marker)
148 {
149         ImBuf *ibuf = NULL;
150
151         if (track->pattern_match == TRACK_MATCH_KEYFRAME) {
152                 ibuf = tracking_context_get_keyframed_ibuf(clip, user, clip_flag, track, curfra, backwards, reference_marker);
153         }
154         else {
155                 ibuf = tracking_context_get_frame_ibuf(clip, user, clip_flag, curfra);
156
157                 /* use current marker as keyframed position */
158                 *reference_marker = BKE_tracking_marker_get(track, curfra);
159         }
160
161         return ibuf;
162 }
163
164 /* Fill in libmv tracker options structure with settings need to be used to perform track. */
165 void tracking_configure_tracker(const MovieTrackingTrack *track, float *mask,
166                                 libmv_TrackRegionOptions *options)
167 {
168         options->motion_model = track->motion_model;
169
170         options->use_brute = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_BRUTE) != 0);
171
172         options->use_normalization = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_NORMALIZATION) != 0);
173
174         options->num_iterations = 50;
175         options->minimum_correlation = track->minimum_correlation;
176         options->sigma = 0.9;
177
178         if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) != 0)
179                 options->image1_mask = mask;
180         else
181                 options->image1_mask = NULL;
182 }
183
184 /* Perform tracking from a reference_marker to destination_ibuf.
185  * Uses marker as an initial position guess.
186  *
187  * Returns truth if tracker returned success, puts result
188  * to dst_pixel_x and dst_pixel_y.
189  */
190 static bool configure_and_run_tracker(ImBuf *destination_ibuf, MovieTrackingTrack *track,
191                                       MovieTrackingMarker *reference_marker, MovieTrackingMarker *marker,
192                                       float *reference_search_area, int reference_search_area_width,
193                                       int reference_search_area_height, float *mask,
194                                       double dst_pixel_x[5], double dst_pixel_y[5])
195 {
196         /* To convert to the x/y split array format for libmv. */
197         double src_pixel_x[5], src_pixel_y[5];
198
199         /* Settings for the tracker */
200         libmv_TrackRegionOptions options = {0};
201         libmv_TrackRegionResult result;
202
203         float *patch_new;
204
205         int new_search_area_width, new_search_area_height;
206         int frame_width, frame_height;
207
208         bool tracked;
209
210         frame_width = destination_ibuf->x;
211         frame_height = destination_ibuf->y;
212
213         /* for now track to the same search area dimension as marker has got for current frame
214          * will make all tracked markers in currently tracked segment have the same search area
215          * size, but it's quite close to what is actually needed
216          */
217         patch_new = track_get_search_floatbuf(destination_ibuf, track, marker,
218                                               &new_search_area_width, &new_search_area_height);
219
220         /* configure the tracker */
221         tracking_configure_tracker(track, mask, &options);
222
223         /* convert the marker corners and center into pixel coordinates in the search/destination images. */
224         tracking_get_marker_coords_for_tracking(frame_width, frame_height, reference_marker, src_pixel_x, src_pixel_y);
225         tracking_get_marker_coords_for_tracking(frame_width, frame_height, marker, dst_pixel_x, dst_pixel_y);
226
227         if (patch_new == NULL || reference_search_area == NULL)
228                 return false;
229
230         /* run the tracker! */
231         tracked = libmv_trackRegion(&options,
232                                     reference_search_area,
233                                     reference_search_area_width,
234                                     reference_search_area_height,
235                                     patch_new,
236                                     new_search_area_width,
237                                     new_search_area_height,
238                                     src_pixel_x, src_pixel_y,
239                                     &result,
240                                     dst_pixel_x, dst_pixel_y);
241
242         MEM_freeN(patch_new);
243
244         return tracked;
245 }
246
247 static bool refine_marker_reference_frame_get(MovieTrackingTrack *track,
248                                               MovieTrackingMarker *marker,
249                                               bool backwards,
250                                               int *reference_framenr)
251 {
252         const MovieTrackingMarker *first_marker = track->markers;
253         const MovieTrackingMarker *last_marker = track->markers + track->markersnr - 1;
254         MovieTrackingMarker *reference = backwards ? marker + 1 : marker - 1;
255
256         while (reference >= first_marker &&
257                reference <= last_marker &&
258                (reference->flag & MARKER_DISABLED) != 0)
259         {
260                 if (backwards)
261                         reference++;
262                 else
263                         reference--;
264         }
265
266         if (reference < first_marker ||
267             reference > last_marker)
268         {
269                 return false;
270         }
271
272         *reference_framenr = reference->framenr;
273         return (reference->flag & MARKER_DISABLED) == 0;
274 }
275
276 /* Refine marker's position using previously known keyframe.
277  * Direction of searching for a keyframe depends on backwards flag,
278  * which means if backwards is false, previous keyframe will be as
279  * reference.
280  */
281 void BKE_tracking_refine_marker(MovieClip *clip, MovieTrackingTrack *track, MovieTrackingMarker *marker, bool backwards)
282 {
283         MovieTrackingMarker *reference_marker = NULL;
284         ImBuf *reference_ibuf, *destination_ibuf;
285         float *search_area, *mask = NULL;
286         int frame_width, frame_height;
287         int search_area_height, search_area_width;
288         int clip_flag = clip->flag & MCLIP_TIMECODE_FLAGS;
289         int reference_framenr;
290         MovieClipUser user = {0};
291         double dst_pixel_x[5], dst_pixel_y[5];
292         bool tracked;
293
294         /* Construct a temporary clip used, used to acquire image buffers. */
295         user.framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
296
297         BKE_movieclip_get_size(clip, &user, &frame_width, &frame_height);
298
299         /* Get an image buffer for reference frame, also gets reference marker. */
300         if (!refine_marker_reference_frame_get(track,
301                                                marker,
302                                                backwards,
303                                                &reference_framenr))
304         {
305                 return;
306         }
307
308         reference_ibuf = tracking_context_get_reference_ibuf(clip, &user, clip_flag, track, reference_framenr,
309                                                              backwards, &reference_marker);
310         if (reference_ibuf == NULL) {
311                 return;
312         }
313
314         /* Could not refine with self. */
315         if (reference_marker == marker) {
316                 return;
317         }
318
319         /* Destination image buffer has got frame number corresponding to refining marker. */
320         destination_ibuf = BKE_movieclip_get_ibuf_flag(clip, &user, clip_flag, MOVIECLIP_CACHE_SKIP);
321         if (destination_ibuf == NULL) {
322                 IMB_freeImBuf(reference_ibuf);
323                 return;
324         }
325
326         /* Get search area from reference image. */
327         search_area = track_get_search_floatbuf(reference_ibuf, track, reference_marker,
328                                                 &search_area_width, &search_area_height);
329
330         /* If needed, compute track's mask. */
331         if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) != 0)
332                 mask = BKE_tracking_track_get_mask(frame_width, frame_height, track, marker);
333
334         /* Run the tracker from reference frame to current one. */
335         tracked = configure_and_run_tracker(destination_ibuf, track, reference_marker, marker,
336                                             search_area, search_area_width, search_area_height,
337                                             mask, dst_pixel_x, dst_pixel_y);
338
339         /* Refine current marker's position if track was successful. */
340         if (tracked) {
341                 tracking_set_marker_coords_from_tracking(frame_width, frame_height, marker, dst_pixel_x, dst_pixel_y);
342                 marker->flag |= MARKER_TRACKED;
343         }
344
345         /* Free memory used for refining */
346         MEM_freeN(search_area);
347         if (mask)
348                 MEM_freeN(mask);
349         IMB_freeImBuf(reference_ibuf);
350         IMB_freeImBuf(destination_ibuf);
351 }