source/blender/blenkernel/intern/tracking_region_tracker.c

   1 /*
   2  * ***** BEGIN GPL LICENSE BLOCK *****
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License
   6  * as published by the Free Software Foundation; either version 2
   7  * of the License, or (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  17  *
  18  * The Original Code is Copyright (C) 2011 Blender Foundation.
  19  * All rights reserved.
  20  *
  21  * Contributor(s): Blender Foundation,
  22  *                 Sergey Sharybin
  23  *                 Keir Mierle
  24  *
  25  * ***** END GPL LICENSE BLOCK *****
  26  */
  27
  28 /** \file blender/blenkernel/intern/tracking_region_tracker.c
  29  *  \ingroup bke
  30  *
  31  * This file contains implementation of blender-side region tracker
  32  * which is used for 2D feature tracking.
  33  */
  34
  35 #include "MEM_guardedalloc.h"
  36
  37 #include "DNA_movieclip_types.h"
  38
  39 #include "BLI_utildefines.h"
  40 #include "BLI_threads.h"
  41
  42 #include "BKE_tracking.h"
  43 #include "BKE_movieclip.h"
  44
  45 #include "IMB_imbuf_types.h"
  46 #include "IMB_imbuf.h"
  47
  48 #include "libmv-capi.h"
  49 #include "tracking_private.h"
  50
  51 /* **** utility functions for tracking **** */
  52
  53 /* convert from float and byte RGBA to grayscale. Supports different coefficients for RGB. */
  54 static void float_rgba_to_gray(const float *rgba, float *gray, int num_pixels,
  55                                float weight_red, float weight_green, float weight_blue)
  56 {
  57         int i;
  58
  59         for (i = 0; i < num_pixels; i++) {
  60                 const float *pixel = rgba + 4 * i;
  61
  62                 gray[i] = weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2];
  63         }
  64 }
  65
  66 static void uint8_rgba_to_float_gray(const unsigned char *rgba, float *gray, int num_pixels,
  67                                      float weight_red, float weight_green, float weight_blue)
  68 {
  69         int i;
  70
  71         for (i = 0; i < num_pixels; i++) {
  72                 const unsigned char *pixel = rgba + i * 4;
  73
  74                 gray[i] = (weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2]) / 255.0f;
  75         }
  76 }
  77
  78 /* Get grayscale float search buffer for given marker and frame. */
  79 static float *track_get_search_floatbuf(ImBuf *ibuf, MovieTrackingTrack *track, MovieTrackingMarker *marker,
  80                                         int *width_r, int *height_r)
  81 {
  82         ImBuf *searchibuf;
  83         float *gray_pixels;
  84         int width, height;
  85
  86         searchibuf = BKE_tracking_get_search_imbuf(ibuf, track, marker, false, true);
  87
  88         if (!searchibuf) {
  89                 *width_r = 0;
  90                 *height_r = 0;
  91                 return NULL;
  92         }
  93
  94         width = searchibuf->x;
  95         height = searchibuf->y;
  96
  97         gray_pixels = MEM_callocN(width * height * sizeof(float), "tracking floatBuf");
  98
  99         if (searchibuf->rect_float) {
 100                 float_rgba_to_gray(searchibuf->rect_float, gray_pixels, width * height,
 101                                    0.2126f, 0.7152f, 0.0722f);
 102         }
 103         else {
 104                 uint8_rgba_to_float_gray((unsigned char *)searchibuf->rect, gray_pixels, width * height,
 105                                          0.2126f, 0.7152f, 0.0722f);
 106         }
 107
 108         IMB_freeImBuf(searchibuf);
 109
 110         *width_r = width;
 111         *height_r = height;
 112
 113         return gray_pixels;
 114 }
 115
 116 /* Get image buffer for a given frame
 117  *
 118  * Frame is in clip space.
 119  */
 120 static ImBuf *tracking_context_get_frame_ibuf(MovieClip *clip, MovieClipUser *user, int clip_flag, int framenr)
 121 {
 122         ImBuf *ibuf;
 123         MovieClipUser new_user = *user;
 124
 125         new_user.framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, framenr);
 126
 127         ibuf = BKE_movieclip_get_ibuf_flag(clip, &new_user, clip_flag, MOVIECLIP_CACHE_SKIP);
 128
 129         return ibuf;
 130 }
 131
 132 /* Get image buffer for previous marker's keyframe. */
 133 static ImBuf *tracking_context_get_keyframed_ibuf(MovieClip *clip, MovieClipUser *user, int clip_flag,
 134                                                   MovieTrackingTrack *track, int curfra, bool backwards,
 135                                                   MovieTrackingMarker **marker_keyed_r)
 136 {
 137         MovieTrackingMarker *marker_keyed;
 138         int keyed_framenr;
 139
 140         marker_keyed = tracking_get_keyframed_marker(track, curfra, backwards);
 141         if (marker_keyed == NULL) {
 142                 return NULL;
 143         }
 144
 145         keyed_framenr = marker_keyed->framenr;
 146
 147         *marker_keyed_r = marker_keyed;
 148
 149         return tracking_context_get_frame_ibuf(clip, user, clip_flag, keyed_framenr);
 150 }
 151
 152 /* Get image buffer which si used as referece for track. */
 153 static ImBuf *tracking_context_get_reference_ibuf(MovieClip *clip, MovieClipUser *user, int clip_flag,
 154                                                   MovieTrackingTrack *track, int curfra, bool backwards,
 155                                                   MovieTrackingMarker **reference_marker)
 156 {
 157         ImBuf *ibuf = NULL;
 158
 159         if (track->pattern_match == TRACK_MATCH_KEYFRAME) {
 160                 ibuf = tracking_context_get_keyframed_ibuf(clip, user, clip_flag, track, curfra, backwards, reference_marker);
 161         }
 162         else {
 163                 ibuf = tracking_context_get_frame_ibuf(clip, user, clip_flag, curfra);
 164
 165                 /* use current marker as keyframed position */
 166                 *reference_marker = BKE_tracking_marker_get(track, curfra);
 167         }
 168
 169         return ibuf;
 170 }
 171
 172 /* Fill in libmv tracker options structure with settings need to be used to perform track. */
 173 void tracking_configure_tracker(const MovieTrackingTrack *track, float *mask,
 174                                 libmv_TrackRegionOptions *options)
 175 {
 176         options->motion_model = track->motion_model;
 177
 178         options->use_brute = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_BRUTE) != 0);
 179
 180         options->use_normalization = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_NORMALIZATION) != 0);
 181
 182         options->num_iterations = 50;
 183         options->minimum_correlation = track->minimum_correlation;
 184         options->sigma = 0.9;
 185
 186         if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) != 0)
 187                 options->image1_mask = mask;
 188         else
 189                 options->image1_mask = NULL;
 190 }
 191
 192 /* Peform tracking from a reference_marker to destination_ibuf.
 193  * Uses marker as an initial position guess.
 194  *
 195  * Returns truth if tracker returned success, puts result
 196  * to dst_pixel_x and dst_pixel_y.
 197  */
 198 static bool configure_and_run_tracker(ImBuf *destination_ibuf, MovieTrackingTrack *track,
 199                                       MovieTrackingMarker *reference_marker, MovieTrackingMarker *marker,
 200                                       float *reference_search_area, int reference_search_area_width,
 201                                       int reference_search_area_height, float *mask,
 202                                       double dst_pixel_x[5], double dst_pixel_y[5])
 203 {
 204         /* To convert to the x/y split array format for libmv. */
 205         double src_pixel_x[5], src_pixel_y[5];
 206
 207         /* Settings for the tracker */
 208         libmv_TrackRegionOptions options = {0};
 209         libmv_TrackRegionResult result;
 210
 211         float *patch_new;
 212
 213         int new_search_area_width, new_search_area_height;
 214         int frame_width, frame_height;
 215
 216         bool tracked;
 217
 218         frame_width = destination_ibuf->x;
 219         frame_height = destination_ibuf->y;
 220
 221         /* for now track to the same search area dimension as marker has got for current frame
 222          * will make all tracked markers in currently tracked segment have the same search area
 223          * size, but it's quite close to what is actually needed
 224          */
 225         patch_new = track_get_search_floatbuf(destination_ibuf, track, marker,
 226                                               &new_search_area_width, &new_search_area_height);
 227
 228         /* configure the tracker */
 229         tracking_configure_tracker(track, mask, &options);
 230
 231         /* convert the marker corners and center into pixel coordinates in the search/destination images. */
 232         tracking_get_marker_coords_for_tracking(frame_width, frame_height, reference_marker, src_pixel_x, src_pixel_y);
 233         tracking_get_marker_coords_for_tracking(frame_width, frame_height, marker, dst_pixel_x, dst_pixel_y);
 234
 235         if (patch_new == NULL || reference_search_area == NULL)
 236                 return false;
 237
 238         /* run the tracker! */
 239         tracked = libmv_trackRegion(&options,
 240                                     reference_search_area,
 241                                     reference_search_area_width,
 242                                     reference_search_area_height,
 243                                     patch_new,
 244                                     new_search_area_width,
 245                                     new_search_area_height,
 246                                     src_pixel_x, src_pixel_y,
 247                                     &result,
 248                                     dst_pixel_x, dst_pixel_y);
 249
 250         MEM_freeN(patch_new);
 251
 252         return tracked;
 253 }
 254
 255 static bool refine_marker_reference_frame_get(MovieTrackingTrack *track,
 256                                               MovieTrackingMarker *marker,
 257                                               bool backwards,
 258                                               int *reference_framenr)
 259 {
 260         const MovieTrackingMarker *first_marker = track->markers;
 261         const MovieTrackingMarker *last_marker = track->markers + track->markersnr - 1;
 262         MovieTrackingMarker *reference = backwards ? marker + 1 : marker - 1;
 263
 264         while (reference >= first_marker &&
 265                reference <= last_marker &&
 266                (reference->flag & MARKER_DISABLED) != 0)
 267         {
 268                 if (backwards)
 269                         reference++;
 270                 else
 271                         reference--;
 272         }
 273
 274         if (reference < first_marker ||
 275             reference > last_marker)
 276         {
 277                 return false;
 278         }
 279
 280         *reference_framenr = reference->framenr;
 281         return (reference->flag & MARKER_DISABLED) == 0;
 282 }
 283
 284 /* Refine marker's position using previously known keyframe.
 285  * Direction of searching for a keyframe depends on backwards flag,
 286  * which means if backwards is false, previous keyframe will be as
 287  * reference.
 288  */
 289 void BKE_tracking_refine_marker(MovieClip *clip, MovieTrackingTrack *track, MovieTrackingMarker *marker, bool backwards)
 290 {
 291         MovieTrackingMarker *reference_marker = NULL;
 292         ImBuf *reference_ibuf, *destination_ibuf;
 293         float *search_area, *mask = NULL;
 294         int frame_width, frame_height;
 295         int search_area_height, search_area_width;
 296         int clip_flag = clip->flag & MCLIP_TIMECODE_FLAGS;
 297         int reference_framenr;
 298         MovieClipUser user = {0};
 299         double dst_pixel_x[5], dst_pixel_y[5];
 300         bool tracked;
 301
 302         /* Construct a temporary clip used, used to acquire image buffers. */
 303         user.framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
 304
 305         BKE_movieclip_get_size(clip, &user, &frame_width, &frame_height);
 306
 307         /* Get an image buffer for reference frame, also gets reference marker. */
 308         if (!refine_marker_reference_frame_get(track,
 309                                                marker,
 310                                                backwards,
 311                                                &reference_framenr))
 312         {
 313                 return;
 314         }
 315
 316         reference_ibuf = tracking_context_get_reference_ibuf(clip, &user, clip_flag, track, reference_framenr,
 317                                                              backwards, &reference_marker);
 318         if (reference_ibuf == NULL) {
 319                 return;
 320         }
 321
 322         /* Could not refine with self. */
 323         if (reference_marker == marker) {
 324                 return;
 325         }
 326
 327         /* Destination image buffer has got frame number corresponding to refining marker. */
 328         destination_ibuf = BKE_movieclip_get_ibuf_flag(clip, &user, clip_flag, MOVIECLIP_CACHE_SKIP);
 329         if (destination_ibuf == NULL) {
 330                 IMB_freeImBuf(reference_ibuf);
 331                 return;
 332         }
 333
 334         /* Get search area from reference image. */
 335         search_area = track_get_search_floatbuf(reference_ibuf, track, reference_marker,
 336                                                 &search_area_width, &search_area_height);
 337
 338         /* If needed, compute track's mask. */
 339         if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) != 0)
 340                 mask = BKE_tracking_track_get_mask(frame_width, frame_height, track, marker);
 341
 342         /* Run the tracker from reference frame to current one. */
 343         tracked = configure_and_run_tracker(destination_ibuf, track, reference_marker, marker,
 344                                             search_area, search_area_width, search_area_height,
 345                                             mask, dst_pixel_x, dst_pixel_y);
 346
 347         /* Refine current marker's position if track was successful. */
 348         if (tracked) {
 349                 tracking_set_marker_coords_from_tracking(frame_width, frame_height, marker, dst_pixel_x, dst_pixel_y);
 350                 marker->flag |= MARKER_TRACKED;
 351         }
 352
 353         /* Free memory used for refining */
 354         MEM_freeN(search_area);
 355         if (mask)
 356                 MEM_freeN(mask);
 357         IMB_freeImBuf(reference_ibuf);
 358         IMB_freeImBuf(destination_ibuf);
 359 }