Cleanup: style, use braces for blenkernel
[blender.git] / source / blender / blenkernel / intern / tracking_region_tracker.c
1 /*
2  * This program is free software; you can redistribute it and/or
3  * modify it under the terms of the GNU General Public License
4  * as published by the Free Software Foundation; either version 2
5  * of the License, or (at your option) any later version.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * along with this program; if not, write to the Free Software Foundation,
14  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15  *
16  * The Original Code is Copyright (C) 2011 Blender Foundation.
17  * All rights reserved.
18  */
19
20 /** \file
21  * \ingroup bke
22  *
23  * This file contains implementation of blender-side region tracker
24  * which is used for 2D feature tracking.
25  */
26
27 #include "MEM_guardedalloc.h"
28
29 #include "DNA_movieclip_types.h"
30
31 #include "BLI_utildefines.h"
32 #include "BLI_threads.h"
33
34 #include "BKE_tracking.h"
35 #include "BKE_movieclip.h"
36
37 #include "IMB_imbuf_types.h"
38 #include "IMB_imbuf.h"
39
40 #include "libmv-capi.h"
41 #include "tracking_private.h"
42
43 /* **** utility functions for tracking **** */
44
45 /* convert from float and byte RGBA to grayscale. Supports different coefficients for RGB. */
46 static void float_rgba_to_gray(const float *rgba,
47                                float *gray,
48                                int num_pixels,
49                                float weight_red,
50                                float weight_green,
51                                float weight_blue)
52 {
53   int i;
54
55   for (i = 0; i < num_pixels; i++) {
56     const float *pixel = rgba + 4 * i;
57
58     gray[i] = weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2];
59   }
60 }
61
62 static void uint8_rgba_to_float_gray(const unsigned char *rgba,
63                                      float *gray,
64                                      int num_pixels,
65                                      float weight_red,
66                                      float weight_green,
67                                      float weight_blue)
68 {
69   int i;
70
71   for (i = 0; i < num_pixels; i++) {
72     const unsigned char *pixel = rgba + i * 4;
73
74     gray[i] = (weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2]) / 255.0f;
75   }
76 }
77
78 /* Get grayscale float search buffer for given marker and frame. */
79 static float *track_get_search_floatbuf(ImBuf *ibuf,
80                                         MovieTrackingTrack *track,
81                                         MovieTrackingMarker *marker,
82                                         int *width_r,
83                                         int *height_r)
84 {
85   ImBuf *searchibuf;
86   float *gray_pixels;
87   int width, height;
88
89   searchibuf = BKE_tracking_get_search_imbuf(ibuf, track, marker, false, true);
90
91   if (!searchibuf) {
92     *width_r = 0;
93     *height_r = 0;
94     return NULL;
95   }
96
97   width = searchibuf->x;
98   height = searchibuf->y;
99
100   gray_pixels = MEM_callocN(width * height * sizeof(float), "tracking floatBuf");
101
102   if (searchibuf->rect_float) {
103     float_rgba_to_gray(
104         searchibuf->rect_float, gray_pixels, width * height, 0.2126f, 0.7152f, 0.0722f);
105   }
106   else {
107     uint8_rgba_to_float_gray(
108         (unsigned char *)searchibuf->rect, gray_pixels, width * height, 0.2126f, 0.7152f, 0.0722f);
109   }
110
111   IMB_freeImBuf(searchibuf);
112
113   *width_r = width;
114   *height_r = height;
115
116   return gray_pixels;
117 }
118
119 /* Get image buffer for a given frame
120  *
121  * Frame is in clip space.
122  */
123 static ImBuf *tracking_context_get_frame_ibuf(MovieClip *clip,
124                                               MovieClipUser *user,
125                                               int clip_flag,
126                                               int framenr)
127 {
128   ImBuf *ibuf;
129   MovieClipUser new_user = *user;
130
131   new_user.framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, framenr);
132
133   ibuf = BKE_movieclip_get_ibuf_flag(clip, &new_user, clip_flag, MOVIECLIP_CACHE_SKIP);
134
135   return ibuf;
136 }
137
138 /* Get image buffer for previous marker's keyframe. */
139 static ImBuf *tracking_context_get_keyframed_ibuf(MovieClip *clip,
140                                                   MovieClipUser *user,
141                                                   int clip_flag,
142                                                   MovieTrackingTrack *track,
143                                                   int curfra,
144                                                   bool backwards,
145                                                   MovieTrackingMarker **marker_keyed_r)
146 {
147   MovieTrackingMarker *marker_keyed;
148   int keyed_framenr;
149
150   marker_keyed = tracking_get_keyframed_marker(track, curfra, backwards);
151   if (marker_keyed == NULL) {
152     return NULL;
153   }
154
155   keyed_framenr = marker_keyed->framenr;
156
157   *marker_keyed_r = marker_keyed;
158
159   return tracking_context_get_frame_ibuf(clip, user, clip_flag, keyed_framenr);
160 }
161
162 /* Get image buffer which si used as reference for track. */
163 static ImBuf *tracking_context_get_reference_ibuf(MovieClip *clip,
164                                                   MovieClipUser *user,
165                                                   int clip_flag,
166                                                   MovieTrackingTrack *track,
167                                                   int curfra,
168                                                   bool backwards,
169                                                   MovieTrackingMarker **reference_marker)
170 {
171   ImBuf *ibuf = NULL;
172
173   if (track->pattern_match == TRACK_MATCH_KEYFRAME) {
174     ibuf = tracking_context_get_keyframed_ibuf(
175         clip, user, clip_flag, track, curfra, backwards, reference_marker);
176   }
177   else {
178     ibuf = tracking_context_get_frame_ibuf(clip, user, clip_flag, curfra);
179
180     /* use current marker as keyframed position */
181     *reference_marker = BKE_tracking_marker_get(track, curfra);
182   }
183
184   return ibuf;
185 }
186
187 /* Fill in libmv tracker options structure with settings need to be used to perform track. */
188 void tracking_configure_tracker(const MovieTrackingTrack *track,
189                                 float *mask,
190                                 libmv_TrackRegionOptions *options)
191 {
192   options->motion_model = track->motion_model;
193
194   options->use_brute = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_BRUTE) != 0);
195
196   options->use_normalization = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_NORMALIZATION) !=
197                                 0);
198
199   options->num_iterations = 50;
200   options->minimum_correlation = track->minimum_correlation;
201   options->sigma = 0.9;
202
203   if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) != 0) {
204     options->image1_mask = mask;
205   }
206   else {
207     options->image1_mask = NULL;
208   }
209 }
210
211 /* Perform tracking from a reference_marker to destination_ibuf.
212  * Uses marker as an initial position guess.
213  *
214  * Returns truth if tracker returned success, puts result
215  * to dst_pixel_x and dst_pixel_y.
216  */
217 static bool configure_and_run_tracker(ImBuf *destination_ibuf,
218                                       MovieTrackingTrack *track,
219                                       MovieTrackingMarker *reference_marker,
220                                       MovieTrackingMarker *marker,
221                                       float *reference_search_area,
222                                       int reference_search_area_width,
223                                       int reference_search_area_height,
224                                       float *mask,
225                                       double dst_pixel_x[5],
226                                       double dst_pixel_y[5])
227 {
228   /* To convert to the x/y split array format for libmv. */
229   double src_pixel_x[5], src_pixel_y[5];
230
231   /* Settings for the tracker */
232   libmv_TrackRegionOptions options = {0};
233   libmv_TrackRegionResult result;
234
235   float *patch_new;
236
237   int new_search_area_width, new_search_area_height;
238   int frame_width, frame_height;
239
240   bool tracked;
241
242   frame_width = destination_ibuf->x;
243   frame_height = destination_ibuf->y;
244
245   /* for now track to the same search area dimension as marker has got for current frame
246    * will make all tracked markers in currently tracked segment have the same search area
247    * size, but it's quite close to what is actually needed
248    */
249   patch_new = track_get_search_floatbuf(
250       destination_ibuf, track, marker, &new_search_area_width, &new_search_area_height);
251
252   /* configure the tracker */
253   tracking_configure_tracker(track, mask, &options);
254
255   /* convert the marker corners and center into pixel coordinates in the search/destination images. */
256   tracking_get_marker_coords_for_tracking(
257       frame_width, frame_height, reference_marker, src_pixel_x, src_pixel_y);
258   tracking_get_marker_coords_for_tracking(
259       frame_width, frame_height, marker, dst_pixel_x, dst_pixel_y);
260
261   if (patch_new == NULL || reference_search_area == NULL) {
262     return false;
263   }
264
265   /* run the tracker! */
266   tracked = libmv_trackRegion(&options,
267                               reference_search_area,
268                               reference_search_area_width,
269                               reference_search_area_height,
270                               patch_new,
271                               new_search_area_width,
272                               new_search_area_height,
273                               src_pixel_x,
274                               src_pixel_y,
275                               &result,
276                               dst_pixel_x,
277                               dst_pixel_y);
278
279   MEM_freeN(patch_new);
280
281   return tracked;
282 }
283
284 static bool refine_marker_reference_frame_get(MovieTrackingTrack *track,
285                                               MovieTrackingMarker *marker,
286                                               bool backwards,
287                                               int *reference_framenr)
288 {
289   const MovieTrackingMarker *first_marker = track->markers;
290   const MovieTrackingMarker *last_marker = track->markers + track->markersnr - 1;
291   MovieTrackingMarker *reference = backwards ? marker + 1 : marker - 1;
292
293   while (reference >= first_marker && reference <= last_marker &&
294          (reference->flag & MARKER_DISABLED) != 0) {
295     if (backwards) {
296       reference++;
297     }
298     else {
299       reference--;
300     }
301   }
302
303   if (reference < first_marker || reference > last_marker) {
304     return false;
305   }
306
307   *reference_framenr = reference->framenr;
308   return (reference->flag & MARKER_DISABLED) == 0;
309 }
310
311 /* Refine marker's position using previously known keyframe.
312  * Direction of searching for a keyframe depends on backwards flag,
313  * which means if backwards is false, previous keyframe will be as
314  * reference.
315  */
316 void BKE_tracking_refine_marker(MovieClip *clip,
317                                 MovieTrackingTrack *track,
318                                 MovieTrackingMarker *marker,
319                                 bool backwards)
320 {
321   MovieTrackingMarker *reference_marker = NULL;
322   ImBuf *reference_ibuf, *destination_ibuf;
323   float *search_area, *mask = NULL;
324   int frame_width, frame_height;
325   int search_area_height, search_area_width;
326   int clip_flag = clip->flag & MCLIP_TIMECODE_FLAGS;
327   int reference_framenr;
328   MovieClipUser user = {0};
329   double dst_pixel_x[5], dst_pixel_y[5];
330   bool tracked;
331
332   /* Construct a temporary clip used, used to acquire image buffers. */
333   user.framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
334
335   BKE_movieclip_get_size(clip, &user, &frame_width, &frame_height);
336
337   /* Get an image buffer for reference frame, also gets reference marker. */
338   if (!refine_marker_reference_frame_get(track, marker, backwards, &reference_framenr)) {
339     return;
340   }
341
342   reference_ibuf = tracking_context_get_reference_ibuf(
343       clip, &user, clip_flag, track, reference_framenr, backwards, &reference_marker);
344   if (reference_ibuf == NULL) {
345     return;
346   }
347
348   /* Could not refine with self. */
349   if (reference_marker == marker) {
350     return;
351   }
352
353   /* Destination image buffer has got frame number corresponding to refining marker. */
354   destination_ibuf = BKE_movieclip_get_ibuf_flag(clip, &user, clip_flag, MOVIECLIP_CACHE_SKIP);
355   if (destination_ibuf == NULL) {
356     IMB_freeImBuf(reference_ibuf);
357     return;
358   }
359
360   /* Get search area from reference image. */
361   search_area = track_get_search_floatbuf(
362       reference_ibuf, track, reference_marker, &search_area_width, &search_area_height);
363
364   /* If needed, compute track's mask. */
365   if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) != 0) {
366     mask = BKE_tracking_track_get_mask(frame_width, frame_height, track, marker);
367   }
368
369   /* Run the tracker from reference frame to current one. */
370   tracked = configure_and_run_tracker(destination_ibuf,
371                                       track,
372                                       reference_marker,
373                                       marker,
374                                       search_area,
375                                       search_area_width,
376                                       search_area_height,
377                                       mask,
378                                       dst_pixel_x,
379                                       dst_pixel_y);
380
381   /* Refine current marker's position if track was successful. */
382   if (tracked) {
383     tracking_set_marker_coords_from_tracking(
384         frame_width, frame_height, marker, dst_pixel_x, dst_pixel_y);
385     marker->flag |= MARKER_TRACKED;
386   }
387
388   /* Free memory used for refining */
389   MEM_freeN(search_area);
390   if (mask) {
391     MEM_freeN(mask);
392   }
393   IMB_freeImBuf(reference_ibuf);
394   IMB_freeImBuf(destination_ibuf);
395 }