Libmv: Support disabled color channels in trackign settings
[blender.git] / source / blender / blenkernel / intern / tracking_auto.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2011 Blender Foundation.
19  * All rights reserved.
20  *
21  * Contributor(s): Blender Foundation,
22  *                 Sergey Sharybin
23  *                 Keir Mierle
24  *
25  * ***** END GPL LICENSE BLOCK *****
26  */
27
28 /** \file blender/blenkernel/intern/tracking_auto.c
29  *  \ingroup bke
30  */
31
32 #include <stdlib.h>
33
34 #include "MEM_guardedalloc.h"
35
36 #include "DNA_movieclip_types.h"
37 #include "DNA_object_types.h"   /* SELECT */
38
39 #include "BLI_threads.h"
40 #include "BLI_utildefines.h"
41 #include "BLI_math.h"
42
43 #include "BKE_movieclip.h"
44 #include "BKE_tracking.h"
45
46 #include "libmv-capi.h"
47 #include "tracking_private.h"
48
49 typedef struct AutoTrackOptions {
50         int clip_index;  /** Index of the clip this track belogs to. */
51         int track_index;  /* Index of the track in AutoTrack tracks structure. */
52         MovieTrackingTrack *track;  /* Pointer to an original track/ */
53         libmv_TrackRegionOptions track_region_options;  /* Options for the region
54                                                            tracker. */
55         bool use_keyframe_match;  /* Keyframe pattern matching. */
56
57         /* TODO(sergey): A bit awkward to keep it in here, only used to
58          * place a disabled marker once the trackign fails,
59          * Wither find a more clear way to do it or call it track context
60          * or state, not options.
61          */
62         bool is_failed;
63         int failed_frame;
64 } AutoTrackOptions;
65
66 typedef struct AutoTrackContext {
67         MovieClip *clips[MAX_ACCESSOR_CLIP];
68         int num_clips;
69
70         MovieClipUser user;
71         int frame_width, frame_height;
72
73         struct libmv_AutoTrack *autotrack;
74         TrackingImageAccessor *image_accessor;
75
76         int num_tracks;  /* Number of tracks being tracked. */
77         AutoTrackOptions *options;  /* Per-tracking track options. */
78
79         bool backwards;
80         bool sequence;
81         int first_frame;
82         int sync_frame;
83         bool first_sync;
84         SpinLock spin_lock;
85 } AutoTrackContext;
86
87 static void normalized_to_libmv_frame(const float normalized[2],
88                                       const int frame_dimensions[2],
89                                       float result[2])
90 {
91         result[0] = normalized[0] * frame_dimensions[0] - 0.5f;
92         result[1] = normalized[1] * frame_dimensions[1] - 0.5f;
93 }
94
95 static void normalized_relative_to_libmv_frame(const float normalized[2],
96                                                const float origin[2],
97                                                const int frame_dimensions[2],
98                                                float result[2])
99 {
100         result[0] = (normalized[0] + origin[0]) * frame_dimensions[0] - 0.5f;
101         result[1] = (normalized[1] + origin[1]) * frame_dimensions[1] - 0.5f;
102 }
103
104 static void libmv_frame_to_normalized(const float frame_coord[2],
105                                       const int frame_dimensions[2],
106                                       float result[2])
107 {
108         result[0] = (frame_coord[0] + 0.5f) / frame_dimensions[0];
109         result[1] = (frame_coord[1] + 0.5f) / frame_dimensions[1];
110 }
111
112 static void libmv_frame_to_normalized_relative(const float frame_coord[2],
113                                                const float origin[2],
114                                                const int frame_dimensions[2],
115                                                float result[2])
116 {
117         result[0] = (frame_coord[0] - origin[0]) / frame_dimensions[0];
118         result[1] = (frame_coord[1] - origin[1]) / frame_dimensions[1];
119 }
120
121 static void dna_marker_to_libmv_marker(/*const*/ MovieTrackingTrack *track,
122                                        /*const*/ MovieTrackingMarker *marker,
123                                        int clip,
124                                        int track_index,
125                                        int frame_width,
126                                        int frame_height,
127                                        bool backwards,
128                                        libmv_Marker *libmv_marker)
129 {
130         const int frame_dimensions[2] = {frame_width, frame_height};
131         int i;
132         libmv_marker->clip = clip;
133         libmv_marker->frame = marker->framenr;
134         libmv_marker->track = track_index;
135
136         normalized_to_libmv_frame(marker->pos,
137                                   frame_dimensions,
138                                   libmv_marker->center);
139         for (i = 0; i < 4; ++i) {
140                 normalized_relative_to_libmv_frame(marker->pattern_corners[i],
141                                                    marker->pos,
142                                                    frame_dimensions,
143                                                    libmv_marker->patch[i]);
144         }
145
146         normalized_relative_to_libmv_frame(marker->search_min,
147                                            marker->pos,
148                                            frame_dimensions,
149                                            libmv_marker->search_region_min);
150
151         normalized_relative_to_libmv_frame(marker->search_max,
152                                            marker->pos,
153                                            frame_dimensions,
154                                            libmv_marker->search_region_max);
155
156         /* TODO(sergey): All the markers does have 1.0 weight. */
157         libmv_marker->weight = 1.0f;
158
159         if (marker->flag & MARKER_TRACKED) {
160                 libmv_marker->source = LIBMV_MARKER_SOURCE_TRACKED;
161         }
162         else {
163                 libmv_marker->source = LIBMV_MARKER_SOURCE_MANUAL;
164         }
165         libmv_marker->status = LIBMV_MARKER_STATUS_UNKNOWN;
166         libmv_marker->model_type = LIBMV_MARKER_MODEL_TYPE_POINT;
167         libmv_marker->model_id = 0;
168
169         /* TODO(sergey): We currently don't support reference marker from
170          * different clip.
171          */
172         libmv_marker->reference_clip = clip;
173
174         if (track->pattern_match == TRACK_MATCH_KEYFRAME) {
175                 MovieTrackingMarker *keyframe_marker =
176                         tracking_get_keyframed_marker(track,
177                                                       marker->framenr,
178                                                       backwards);
179                 libmv_marker->reference_frame = keyframe_marker->framenr;
180         }
181         else {
182                 libmv_marker->reference_frame = backwards ?
183                                                 marker->framenr - 1 :
184                                                 marker->framenr;
185         }
186
187         libmv_marker->visible_channels =
188                 ((track->flag & TRACK_DISABLE_RED)   ? 0 : LIBMV_MARKER_CHANNEL_R) |
189                 ((track->flag & TRACK_DISABLE_GREEN) ? 0 : LIBMV_MARKER_CHANNEL_G) |
190                 ((track->flag & TRACK_DISABLE_BLUE)  ? 0 : LIBMV_MARKER_CHANNEL_B);
191 }
192
193 static void libmv_marker_to_dna_marker(libmv_Marker *libmv_marker,
194                                        int frame_width,
195                                        int frame_height,
196                                        MovieTrackingMarker *marker)
197 {
198         const int frame_dimensions[2] = {frame_width, frame_height};
199         int i;
200         marker->framenr = libmv_marker->frame;
201
202         libmv_frame_to_normalized(libmv_marker->center,
203                                   frame_dimensions,
204                                   marker->pos);
205         for (i = 0; i < 4; ++i) {
206                 libmv_frame_to_normalized_relative(libmv_marker->patch[i],
207                                                    libmv_marker->center,
208                                                    frame_dimensions,
209                                                    marker->pattern_corners[i]);
210         }
211
212         libmv_frame_to_normalized_relative(libmv_marker->search_region_min,
213                                            libmv_marker->center,
214                                            frame_dimensions,
215                                            marker->search_min);
216
217         libmv_frame_to_normalized_relative(libmv_marker->search_region_max,
218                                            libmv_marker->center,
219                                            frame_dimensions,
220                                            marker->search_max);
221
222         marker->flag = 0;
223         if (libmv_marker->source == LIBMV_MARKER_SOURCE_TRACKED) {
224                 marker->flag |= MARKER_TRACKED;
225         }
226         else {
227                 marker->flag &= ~MARKER_TRACKED;
228         }
229 }
230
231 static bool check_track_trackable(MovieClip *clip,
232                                   MovieTrackingTrack *track,
233                                   MovieClipUser *user)
234 {
235         if (TRACK_SELECTED(track) &&
236             (track->flag & (TRACK_LOCKED | TRACK_HIDDEN)) == 0)
237         {
238                 MovieTrackingMarker *marker;
239                 int frame;
240                 frame = BKE_movieclip_remap_scene_to_clip_frame(clip, user->framenr);
241                 marker = BKE_tracking_marker_get(track, frame);
242                 return (marker->flag & MARKER_DISABLED) == 0;
243         }
244         return false;
245 }
246
247 /* Returns false if marker crossed margin area from frame bounds. */
248 static bool tracking_check_marker_margin(libmv_Marker *libmv_marker,
249                                          int margin,
250                                          int frame_width,
251                                          int frame_height)
252 {
253         float patch_min[2], patch_max[2];
254         float margin_left, margin_top, margin_right, margin_bottom;
255
256         INIT_MINMAX2(patch_min, patch_max);
257         minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[0]);
258         minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[1]);
259         minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[2]);
260         minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[3]);
261
262         margin_left   = max_ff(libmv_marker->center[0] - patch_min[0], margin);
263         margin_top    = max_ff(patch_max[1] - libmv_marker->center[1], margin);
264         margin_right  = max_ff(patch_max[0] - libmv_marker->center[0], margin);
265         margin_bottom = max_ff(libmv_marker->center[1] - patch_min[1], margin);
266
267         if (libmv_marker->center[0] < margin_left ||
268             libmv_marker->center[0] > frame_width - margin_right ||
269             libmv_marker->center[1] < margin_bottom ||
270             libmv_marker->center[1] > frame_height - margin_top)
271         {
272                 return false;
273         }
274
275         return true;
276 }
277
278 AutoTrackContext *BKE_autotrack_context_new(MovieClip *clip,
279                                             MovieClipUser *user,
280                                             const bool backwards,
281                                             const bool sequence)
282 {
283         AutoTrackContext *context = MEM_callocN(sizeof(AutoTrackContext),
284                                                 "autotrack context");
285         MovieTracking *tracking = &clip->tracking;
286         MovieTrackingTrack *track;
287         ListBase *tracksbase = BKE_tracking_get_active_tracks(tracking);
288         int i, track_index, frame_width, frame_height;
289
290         BKE_movieclip_get_size(clip, user, &frame_width, &frame_height);
291
292         /* TODO(sergey): Currently using only a single clip. */
293         context->clips[0] = clip;
294         context->num_clips = 1;
295
296         context->user = *user;
297         context->user.render_size = MCLIP_PROXY_RENDER_SIZE_FULL;
298         context->user.render_flag = 0;
299         context->frame_width = frame_width;
300         context->frame_height = frame_height;
301         context->backwards = backwards;
302         context->sequence = sequence;
303         context->first_frame = user->framenr;
304         context->sync_frame = user->framenr;
305         context->first_sync = true;
306
307         BLI_spin_init(&context->spin_lock);
308
309         context->image_accessor =
310                 tracking_image_accessor_new(context->clips, 1, user->framenr);
311         context->autotrack =
312                 libmv_autoTrackNew(context->image_accessor->libmv_accessor);
313
314         /* Fill in Autotrack with all markers we know. */
315         track_index = 0;
316         for (track = tracksbase->first;
317              track;
318              track = track->next)
319         {
320                 if (check_track_trackable(clip, track, user)) {
321                         context->num_tracks++;
322                 }
323
324                 for (i = 0; i < track->markersnr; ++i) {
325                         MovieTrackingMarker *marker = marker = track->markers + i;
326                         if ((marker->flag & MARKER_DISABLED) == 0) {
327                                 libmv_Marker libmv_marker;
328                                 dna_marker_to_libmv_marker(track,
329                                                            marker,
330                                                            0,
331                                                            track_index,
332                                                            frame_width,
333                                                            frame_height,
334                                                            backwards,
335                                                            &libmv_marker);
336                                 libmv_autoTrackAddMarker(context->autotrack,
337                                                          &libmv_marker);
338                         }
339                 }
340                 track_index++;
341         }
342
343         /* Create per-track tracking options. */
344         context->options =
345                 MEM_callocN(sizeof(AutoTrackOptions) * context->num_tracks,
346                             "auto track options");
347         i = track_index = 0;
348         for (track = tracksbase->first;
349              track;
350              track = track->next)
351         {
352                 if (check_track_trackable(clip, track, user)) {
353                         AutoTrackOptions *options = &context->options[i++];
354                         /* TODO(sergey): Single clip only for now. */
355                         options->clip_index = 0;
356                         options->track_index = track_index;
357                         options->track = track;
358                         tracking_configure_tracker(track,
359                                                    NULL,
360                                                    &options->track_region_options);
361                         options->use_keyframe_match =
362                                 track->pattern_match == TRACK_MATCH_KEYFRAME;
363                 }
364                 ++track_index;
365         }
366
367         return context;
368 }
369
370 bool BKE_autotrack_context_step(AutoTrackContext *context)
371 {
372         int frame_delta = context->backwards ? -1 : 1;
373         bool ok = false;
374         int track;
375
376 #pragma omp parallel for if(context->num_tracks > 1)
377         for (track = 0; track < context->num_tracks; ++track) {
378                 AutoTrackOptions *options = &context->options[track];
379                 libmv_Marker libmv_current_marker,
380                              libmv_reference_marker,
381                              libmv_tracked_marker;
382                 libmv_TrackRegionResult libmv_result;
383                 int frame = BKE_movieclip_remap_scene_to_clip_frame(
384                         context->clips[options->clip_index],
385                         context->user.framenr);
386
387                 if (libmv_autoTrackGetMarker(context->autotrack,
388                                              options->clip_index,
389                                              frame,
390                                              options->track_index,
391                                              &libmv_current_marker))
392                 {
393                         if (!tracking_check_marker_margin(&libmv_current_marker,
394                                                           options->track->margin,
395                                                           context->frame_width,
396                                                           context->frame_height))
397                         {
398                                 continue;
399                         }
400
401                         libmv_tracked_marker = libmv_current_marker;
402                         libmv_tracked_marker.frame = frame + frame_delta;
403
404                         if (options->use_keyframe_match) {
405                                 libmv_tracked_marker.reference_frame =
406                                         libmv_current_marker.reference_frame;
407                                 libmv_autoTrackGetMarker(context->autotrack,
408                                                      options->clip_index,
409                                                      libmv_tracked_marker.reference_frame,
410                                                      options->track_index,
411                                                      &libmv_reference_marker);
412                         }
413                         else {
414                                 libmv_tracked_marker.reference_frame = frame;
415                                 libmv_reference_marker = libmv_current_marker;
416                         }
417
418                         if (libmv_autoTrackMarker(context->autotrack,
419                                                   &options->track_region_options,
420                                                   &libmv_tracked_marker,
421                                                   &libmv_result))
422                         {
423                                 BLI_spin_lock(&context->spin_lock);
424                                 libmv_autoTrackAddMarker(context->autotrack,
425                                                          &libmv_tracked_marker);
426                                 BLI_spin_unlock(&context->spin_lock);
427                         }
428                         else {
429                                 options->is_failed = true;
430                                 options->failed_frame = frame;
431                         }
432                         ok = true;
433                 }
434         }
435
436         BLI_spin_lock(&context->spin_lock);
437         context->user.framenr += frame_delta;
438         BLI_spin_unlock(&context->spin_lock);
439
440         return ok;
441 }
442
443 void BKE_autotrack_context_sync(AutoTrackContext *context)
444 {
445         int newframe = context->user.framenr,
446             frame_delta = context->backwards ? -1 : 1;
447         int clip, frame;
448
449         BLI_spin_lock(&context->spin_lock);
450         for (frame = context->sync_frame;
451              frame != (context->backwards ? newframe - 1 : newframe + 1);
452              frame += frame_delta)
453         {
454                 MovieTrackingMarker marker;
455                 libmv_Marker libmv_marker;
456                 int clip = 0;
457                 int track;
458                 for (track = 0; track < context->num_tracks; ++track) {
459                         AutoTrackOptions *options = &context->options[track];
460                         int track_frame = BKE_movieclip_remap_scene_to_clip_frame(
461                                 context->clips[options->clip_index], frame);
462                         if (options->is_failed) {
463                                 if (options->failed_frame == track_frame) {
464                                         MovieTrackingMarker *prev_marker =
465                                                 BKE_tracking_marker_get_exact(
466                                                           options->track,
467                                                           frame);
468                                         if (prev_marker) {
469                                                 marker = *prev_marker;
470                                                 marker.framenr = context->backwards ?
471                                                                  track_frame - 1 :
472                                                                  track_frame + 1;
473                                                 marker.flag |= MARKER_DISABLED;
474                                                 BKE_tracking_marker_insert(options->track, &marker);
475                                         }
476                                 }
477                                 continue;
478                         }
479                         if (libmv_autoTrackGetMarker(context->autotrack,
480                                                      clip,
481                                                      track_frame,
482                                                      options->track_index,
483                                                      &libmv_marker))
484                         {
485                                 libmv_marker_to_dna_marker(&libmv_marker,
486                                                            context->frame_width,
487                                                            context->frame_height,
488                                                            &marker);
489                                 if (context->first_sync && frame == context->sync_frame) {
490                                         tracking_marker_insert_disabled(options->track,
491                                                                         &marker,
492                                                                         !context->backwards,
493                                                                         false);
494                                 }
495                                 BKE_tracking_marker_insert(options->track, &marker);
496                                 tracking_marker_insert_disabled(options->track,
497                                                                 &marker,
498                                                                 context->backwards,
499                                                                 false);
500                         }
501                 }
502         }
503         BLI_spin_unlock(&context->spin_lock);
504
505         for (clip = 0; clip < context->num_clips; ++clip) {
506                 MovieTracking *tracking = &context->clips[clip]->tracking;
507                 BKE_tracking_dopesheet_tag_update(tracking);
508         }
509
510         context->sync_frame = newframe;
511         context->first_sync = false;
512 }
513
514 void BKE_autotrack_context_sync_user(AutoTrackContext *context,
515                                      MovieClipUser *user)
516 {
517         user->framenr = context->sync_frame;
518 }
519
520 void BKE_autotrack_context_finish(AutoTrackContext *context)
521 {
522         int clip_index;
523
524         for (clip_index = 0; clip_index < context->num_clips; ++clip_index) {
525                 MovieClip *clip = context->clips[clip_index];
526                 ListBase *plane_tracks_base =
527                         BKE_tracking_get_active_plane_tracks(&clip->tracking);
528                 MovieTrackingPlaneTrack *plane_track;
529
530                 for (plane_track = plane_tracks_base->first;
531                      plane_track;
532                      plane_track = plane_track->next)
533                 {
534                         if ((plane_track->flag & PLANE_TRACK_AUTOKEY) == 0) {
535                                 int track;
536                                 for (track = 0; track < context->num_tracks; ++track) {
537                                         MovieTrackingTrack *old_track;
538                                         bool do_update = false;
539                                         int j;
540
541                                         old_track = context->options[track].track;
542                                         for (j = 0; j < plane_track->point_tracksnr; j++) {
543                                                 if (plane_track->point_tracks[j] == old_track) {
544                                                         do_update = true;
545                                                         break;
546                                                 }
547                                         }
548
549                                         if (do_update) {
550                                                 BKE_tracking_track_plane_from_existing_motion(
551                                                         plane_track,
552                                                         context->first_frame);
553                                                 break;
554                                         }
555                                 }
556                         }
557                 }
558         }
559 }
560
561 void BKE_autotrack_context_free(AutoTrackContext *context)
562 {
563         libmv_autoTrackDestroy(context->autotrack);
564         tracking_image_accessor_destroy(context->image_accessor);
565         MEM_freeN(context->options);
566         BLI_spin_end(&context->spin_lock);
567         MEM_freeN(context);
568 }