Merge branch 'master' into blender2.8
[blender.git] / source / blender / blenkernel / intern / tracking_auto.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2011 Blender Foundation.
19  * All rights reserved.
20  *
21  * Contributor(s): Blender Foundation,
22  *                 Sergey Sharybin
23  *                 Keir Mierle
24  *
25  * ***** END GPL LICENSE BLOCK *****
26  */
27
28 /** \file blender/blenkernel/intern/tracking_auto.c
29  *  \ingroup bke
30  */
31
32 #include <stdlib.h>
33
34 #include "MEM_guardedalloc.h"
35
36 #include "DNA_movieclip_types.h"
37 #include "DNA_object_types.h"   /* SELECT */
38
39 #include "BLI_utildefines.h"
40 #include "BLI_listbase.h"
41 #include "BLI_threads.h"
42 #include "BLI_math.h"
43
44 #include "BKE_movieclip.h"
45 #include "BKE_tracking.h"
46
47 #include "libmv-capi.h"
48 #include "tracking_private.h"
49
50 typedef struct AutoTrackOptions {
51         int clip_index;  /** Index of the clip this track belogs to. */
52         int track_index;  /* Index of the track in AutoTrack tracks structure. */
53         MovieTrackingTrack *track;  /* Pointer to an original track/ */
54         libmv_TrackRegionOptions track_region_options;  /* Options for the region
55                                                            tracker. */
56         bool use_keyframe_match;  /* Keyframe pattern matching. */
57
58         /* TODO(sergey): A bit awkward to keep it in here, only used to
59          * place a disabled marker once the tracking fails,
60          * Wither find a more clear way to do it or call it track context
61          * or state, not options.
62          */
63         bool is_failed;
64         int failed_frame;
65 } AutoTrackOptions;
66
67 typedef struct AutoTrackContext {
68         MovieClip *clips[MAX_ACCESSOR_CLIP];
69         int num_clips;
70
71         MovieClipUser user;
72         int frame_width, frame_height;
73
74         struct libmv_AutoTrack *autotrack;
75         TrackingImageAccessor *image_accessor;
76
77         int num_tracks;  /* Number of tracks being tracked. */
78         AutoTrackOptions *options;  /* Per-tracking track options. */
79
80         /* Array of all tracks, indexed by track_index. */
81         MovieTrackingTrack **tracks;
82
83         bool backwards;
84         bool sequence;
85         int first_frame;
86         int sync_frame;
87         bool first_sync;
88         SpinLock spin_lock;
89 } AutoTrackContext;
90
91 static void normalized_to_libmv_frame(const float normalized[2],
92                                       const int frame_dimensions[2],
93                                       float result[2])
94 {
95         result[0] = normalized[0] * frame_dimensions[0] - 0.5f;
96         result[1] = normalized[1] * frame_dimensions[1] - 0.5f;
97 }
98
99 static void normalized_relative_to_libmv_frame(const float normalized[2],
100                                                const float origin[2],
101                                                const int frame_dimensions[2],
102                                                float result[2])
103 {
104         result[0] = (normalized[0] + origin[0]) * frame_dimensions[0] - 0.5f;
105         result[1] = (normalized[1] + origin[1]) * frame_dimensions[1] - 0.5f;
106 }
107
108 static void libmv_frame_to_normalized(const float frame_coord[2],
109                                       const int frame_dimensions[2],
110                                       float result[2])
111 {
112         result[0] = (frame_coord[0] + 0.5f) / frame_dimensions[0];
113         result[1] = (frame_coord[1] + 0.5f) / frame_dimensions[1];
114 }
115
116 static void libmv_frame_to_normalized_relative(const float frame_coord[2],
117                                                const float origin[2],
118                                                const int frame_dimensions[2],
119                                                float result[2])
120 {
121         result[0] = (frame_coord[0] - origin[0]) / frame_dimensions[0];
122         result[1] = (frame_coord[1] - origin[1]) / frame_dimensions[1];
123 }
124
125 static void dna_marker_to_libmv_marker(/*const*/ MovieTrackingTrack *track,
126                                        /*const*/ MovieTrackingMarker *marker,
127                                        int clip,
128                                        int track_index,
129                                        int frame_width,
130                                        int frame_height,
131                                        bool backwards,
132                                        libmv_Marker *libmv_marker)
133 {
134         const int frame_dimensions[2] = {frame_width, frame_height};
135         int i;
136         libmv_marker->clip = clip;
137         libmv_marker->frame = marker->framenr;
138         libmv_marker->track = track_index;
139
140         normalized_to_libmv_frame(marker->pos,
141                                   frame_dimensions,
142                                   libmv_marker->center);
143         for (i = 0; i < 4; ++i) {
144                 normalized_relative_to_libmv_frame(marker->pattern_corners[i],
145                                                    marker->pos,
146                                                    frame_dimensions,
147                                                    libmv_marker->patch[i]);
148         }
149
150         normalized_relative_to_libmv_frame(marker->search_min,
151                                            marker->pos,
152                                            frame_dimensions,
153                                            libmv_marker->search_region_min);
154
155         normalized_relative_to_libmv_frame(marker->search_max,
156                                            marker->pos,
157                                            frame_dimensions,
158                                            libmv_marker->search_region_max);
159
160         /* TODO(sergey): All the markers does have 1.0 weight. */
161         libmv_marker->weight = 1.0f;
162
163         if (marker->flag & MARKER_TRACKED) {
164                 libmv_marker->source = LIBMV_MARKER_SOURCE_TRACKED;
165         }
166         else {
167                 libmv_marker->source = LIBMV_MARKER_SOURCE_MANUAL;
168         }
169         libmv_marker->status = LIBMV_MARKER_STATUS_UNKNOWN;
170         libmv_marker->model_type = LIBMV_MARKER_MODEL_TYPE_POINT;
171         libmv_marker->model_id = 0;
172
173         /* TODO(sergey): We currently don't support reference marker from
174          * different clip.
175          */
176         libmv_marker->reference_clip = clip;
177
178         if (track->pattern_match == TRACK_MATCH_KEYFRAME) {
179                 MovieTrackingMarker *keyframe_marker =
180                         tracking_get_keyframed_marker(track,
181                                                       marker->framenr,
182                                                       backwards);
183                 libmv_marker->reference_frame = keyframe_marker->framenr;
184         }
185         else {
186                 libmv_marker->reference_frame = backwards ?
187                                                 marker->framenr - 1 :
188                                                 marker->framenr;
189         }
190
191         libmv_marker->disabled_channels =
192                 ((track->flag & TRACK_DISABLE_RED)   ? LIBMV_MARKER_CHANNEL_R : 0) |
193                 ((track->flag & TRACK_DISABLE_GREEN) ? LIBMV_MARKER_CHANNEL_G : 0) |
194                 ((track->flag & TRACK_DISABLE_BLUE)  ? LIBMV_MARKER_CHANNEL_B : 0);
195 }
196
197 static void libmv_marker_to_dna_marker(libmv_Marker *libmv_marker,
198                                        int frame_width,
199                                        int frame_height,
200                                        MovieTrackingMarker *marker)
201 {
202         const int frame_dimensions[2] = {frame_width, frame_height};
203         int i;
204         marker->framenr = libmv_marker->frame;
205
206         libmv_frame_to_normalized(libmv_marker->center,
207                                   frame_dimensions,
208                                   marker->pos);
209         for (i = 0; i < 4; ++i) {
210                 libmv_frame_to_normalized_relative(libmv_marker->patch[i],
211                                                    libmv_marker->center,
212                                                    frame_dimensions,
213                                                    marker->pattern_corners[i]);
214         }
215
216         libmv_frame_to_normalized_relative(libmv_marker->search_region_min,
217                                            libmv_marker->center,
218                                            frame_dimensions,
219                                            marker->search_min);
220
221         libmv_frame_to_normalized_relative(libmv_marker->search_region_max,
222                                            libmv_marker->center,
223                                            frame_dimensions,
224                                            marker->search_max);
225
226         marker->flag = 0;
227         if (libmv_marker->source == LIBMV_MARKER_SOURCE_TRACKED) {
228                 marker->flag |= MARKER_TRACKED;
229         }
230         else {
231                 marker->flag &= ~MARKER_TRACKED;
232         }
233 }
234
235 static bool check_track_trackable(MovieClip *clip,
236                                   MovieTrackingTrack *track,
237                                   MovieClipUser *user)
238 {
239         if (TRACK_SELECTED(track) &&
240             (track->flag & (TRACK_LOCKED | TRACK_HIDDEN)) == 0)
241         {
242                 MovieTrackingMarker *marker;
243                 int frame;
244                 frame = BKE_movieclip_remap_scene_to_clip_frame(clip, user->framenr);
245                 marker = BKE_tracking_marker_get(track, frame);
246                 return (marker->flag & MARKER_DISABLED) == 0;
247         }
248         return false;
249 }
250
251 /* Returns false if marker crossed margin area from frame bounds. */
252 static bool tracking_check_marker_margin(libmv_Marker *libmv_marker,
253                                          int margin,
254                                          int frame_width,
255                                          int frame_height)
256 {
257         float patch_min[2], patch_max[2];
258         float margin_left, margin_top, margin_right, margin_bottom;
259
260         INIT_MINMAX2(patch_min, patch_max);
261         minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[0]);
262         minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[1]);
263         minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[2]);
264         minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[3]);
265
266         margin_left   = max_ff(libmv_marker->center[0] - patch_min[0], margin);
267         margin_top    = max_ff(patch_max[1] - libmv_marker->center[1], margin);
268         margin_right  = max_ff(patch_max[0] - libmv_marker->center[0], margin);
269         margin_bottom = max_ff(libmv_marker->center[1] - patch_min[1], margin);
270
271         if (libmv_marker->center[0] < margin_left ||
272             libmv_marker->center[0] > frame_width - margin_right ||
273             libmv_marker->center[1] < margin_bottom ||
274             libmv_marker->center[1] > frame_height - margin_top)
275         {
276                 return false;
277         }
278
279         return true;
280 }
281
282 AutoTrackContext *BKE_autotrack_context_new(MovieClip *clip,
283                                             MovieClipUser *user,
284                                             const bool backwards,
285                                             const bool sequence)
286 {
287         AutoTrackContext *context = MEM_callocN(sizeof(AutoTrackContext),
288                                                 "autotrack context");
289         MovieTracking *tracking = &clip->tracking;
290         MovieTrackingTrack *track;
291         ListBase *tracksbase = BKE_tracking_get_active_tracks(tracking);
292         int i, track_index, frame_width, frame_height;
293
294         BKE_movieclip_get_size(clip, user, &frame_width, &frame_height);
295
296         /* TODO(sergey): Currently using only a single clip. */
297         context->clips[0] = clip;
298         context->num_clips = 1;
299
300         context->user = *user;
301         context->user.render_size = MCLIP_PROXY_RENDER_SIZE_FULL;
302         context->user.render_flag = 0;
303         context->frame_width = frame_width;
304         context->frame_height = frame_height;
305         context->backwards = backwards;
306         context->sequence = sequence;
307         context->first_frame = user->framenr;
308         context->sync_frame = user->framenr;
309         context->first_sync = true;
310
311         BLI_spin_init(&context->spin_lock);
312
313         int num_total_tracks = BLI_listbase_count(tracksbase);
314         context->tracks =
315                 MEM_callocN(sizeof(MovieTrackingTrack *) * num_total_tracks,
316                             "auto track pointers");
317
318         context->image_accessor =
319                 tracking_image_accessor_new(context->clips, 1,
320                                             context->tracks, num_total_tracks,
321                                             user->framenr);
322         context->autotrack =
323                 libmv_autoTrackNew(context->image_accessor->libmv_accessor);
324
325         /* Fill in Autotrack with all markers we know. */
326         track_index = 0;
327         for (track = tracksbase->first;
328              track;
329              track = track->next)
330         {
331                 if (check_track_trackable(clip, track, user)) {
332                         context->num_tracks++;
333                 }
334
335                 for (i = 0; i < track->markersnr; ++i) {
336                         MovieTrackingMarker *marker = track->markers + i;
337                         if ((marker->flag & MARKER_DISABLED) == 0) {
338                                 libmv_Marker libmv_marker;
339                                 dna_marker_to_libmv_marker(track,
340                                                            marker,
341                                                            0,
342                                                            track_index,
343                                                            frame_width,
344                                                            frame_height,
345                                                            backwards,
346                                                            &libmv_marker);
347                                 libmv_autoTrackAddMarker(context->autotrack,
348                                                          &libmv_marker);
349                         }
350                 }
351                 track_index++;
352         }
353
354         /* Create per-track tracking options. */
355         context->options =
356                 MEM_callocN(sizeof(AutoTrackOptions) * context->num_tracks,
357                             "auto track options");
358         i = track_index = 0;
359         for (track = tracksbase->first;
360              track;
361              track = track->next)
362         {
363                 if (check_track_trackable(clip, track, user)) {
364                         AutoTrackOptions *options = &context->options[i++];
365                         /* TODO(sergey): Single clip only for now. */
366                         options->clip_index = 0;
367                         options->track_index = track_index;
368                         options->track = track;
369                         tracking_configure_tracker(track,
370                                                    NULL,
371                                                    &options->track_region_options);
372                         options->use_keyframe_match =
373                                 track->pattern_match == TRACK_MATCH_KEYFRAME;
374                 }
375                 context->tracks[track_index] = track;
376                 ++track_index;
377         }
378
379         return context;
380 }
381
382 bool BKE_autotrack_context_step(AutoTrackContext *context)
383 {
384         int frame_delta = context->backwards ? -1 : 1;
385         bool ok = false;
386         int track;
387
388 #pragma omp parallel for if (context->num_tracks > 1)
389         for (track = 0; track < context->num_tracks; ++track) {
390                 AutoTrackOptions *options = &context->options[track];
391                 if (options->is_failed) {
392                         continue;
393                 }
394                 libmv_Marker libmv_current_marker,
395                              libmv_reference_marker,
396                              libmv_tracked_marker;
397                 libmv_TrackRegionResult libmv_result;
398                 int frame = BKE_movieclip_remap_scene_to_clip_frame(
399                         context->clips[options->clip_index],
400                         context->user.framenr);
401                 bool has_marker;
402
403                 BLI_spin_lock(&context->spin_lock);
404                 has_marker = libmv_autoTrackGetMarker(context->autotrack,
405                                                       options->clip_index,
406                                                       frame,
407                                                       options->track_index,
408                                                       &libmv_current_marker);
409                 BLI_spin_unlock(&context->spin_lock);
410
411                 if (has_marker) {
412                         if (!tracking_check_marker_margin(&libmv_current_marker,
413                                                           options->track->margin,
414                                                           context->frame_width,
415                                                           context->frame_height))
416                         {
417                                 continue;
418                         }
419
420                         libmv_tracked_marker = libmv_current_marker;
421                         libmv_tracked_marker.frame = frame + frame_delta;
422
423                         if (options->use_keyframe_match) {
424                                 libmv_tracked_marker.reference_frame =
425                                         libmv_current_marker.reference_frame;
426                                 libmv_autoTrackGetMarker(context->autotrack,
427                                                      options->clip_index,
428                                                      libmv_tracked_marker.reference_frame,
429                                                      options->track_index,
430                                                      &libmv_reference_marker);
431                         }
432                         else {
433                                 libmv_tracked_marker.reference_frame = frame;
434                                 libmv_reference_marker = libmv_current_marker;
435                         }
436
437                         if (libmv_autoTrackMarker(context->autotrack,
438                                                   &options->track_region_options,
439                                                   &libmv_tracked_marker,
440                                                   &libmv_result))
441                         {
442                                 BLI_spin_lock(&context->spin_lock);
443                                 libmv_autoTrackAddMarker(context->autotrack,
444                                                          &libmv_tracked_marker);
445                                 BLI_spin_unlock(&context->spin_lock);
446                         }
447                         else {
448                                 options->is_failed = true;
449                                 options->failed_frame = frame + frame_delta;
450                         }
451                         ok = true;
452                 }
453         }
454
455         BLI_spin_lock(&context->spin_lock);
456         context->user.framenr += frame_delta;
457         BLI_spin_unlock(&context->spin_lock);
458
459         return ok;
460 }
461
462 void BKE_autotrack_context_sync(AutoTrackContext *context)
463 {
464         int newframe, frame_delta = context->backwards ? -1 : 1;
465         int frame;
466
467         BLI_spin_lock(&context->spin_lock);
468         newframe = context->user.framenr;
469         for (frame = context->sync_frame;
470              frame != (context->backwards ? newframe - 1 : newframe + 1);
471              frame += frame_delta)
472         {
473                 MovieTrackingMarker marker;
474                 libmv_Marker libmv_marker;
475                 int clip = 0;
476                 int track;
477                 for (track = 0; track < context->num_tracks; ++track) {
478                         AutoTrackOptions *options = &context->options[track];
479                         int track_frame = BKE_movieclip_remap_scene_to_clip_frame(
480                                 context->clips[options->clip_index], frame);
481                         if (options->is_failed) {
482                                 if (options->failed_frame == track_frame) {
483                                         MovieTrackingMarker *prev_marker =
484                                                 BKE_tracking_marker_get_exact(
485                                                         options->track,
486                                                         context->backwards
487                                                                 ? frame + 1
488                                                                 : frame - 1);
489                                         if (prev_marker) {
490                                                 marker = *prev_marker;
491                                                 marker.framenr = track_frame;
492                                                 marker.flag |= MARKER_DISABLED;
493                                                 BKE_tracking_marker_insert(options->track, &marker);
494                                                 continue;
495                                         }
496                                 }
497                                 if ((context->backwards && options->failed_frame > track_frame) ||
498                                     (!context->backwards && options->failed_frame < track_frame))
499                                 {
500                                         continue;
501                                 }
502                         }
503                         if (libmv_autoTrackGetMarker(context->autotrack,
504                                                      clip,
505                                                      track_frame,
506                                                      options->track_index,
507                                                      &libmv_marker))
508                         {
509                                 libmv_marker_to_dna_marker(&libmv_marker,
510                                                            context->frame_width,
511                                                            context->frame_height,
512                                                            &marker);
513                                 if (context->first_sync && frame == context->sync_frame) {
514                                         tracking_marker_insert_disabled(options->track,
515                                                                         &marker,
516                                                                         !context->backwards,
517                                                                         false);
518                                 }
519                                 BKE_tracking_marker_insert(options->track, &marker);
520                                 tracking_marker_insert_disabled(options->track,
521                                                                 &marker,
522                                                                 context->backwards,
523                                                                 false);
524                         }
525                 }
526         }
527         BLI_spin_unlock(&context->spin_lock);
528
529         for (int clip = 0; clip < context->num_clips; ++clip) {
530                 MovieTracking *tracking = &context->clips[clip]->tracking;
531                 BKE_tracking_dopesheet_tag_update(tracking);
532         }
533
534         context->sync_frame = newframe;
535         context->first_sync = false;
536 }
537
538 void BKE_autotrack_context_sync_user(AutoTrackContext *context,
539                                      MovieClipUser *user)
540 {
541         user->framenr = context->sync_frame;
542 }
543
544 void BKE_autotrack_context_finish(AutoTrackContext *context)
545 {
546         int clip_index;
547
548         for (clip_index = 0; clip_index < context->num_clips; ++clip_index) {
549                 MovieClip *clip = context->clips[clip_index];
550                 ListBase *plane_tracks_base =
551                         BKE_tracking_get_active_plane_tracks(&clip->tracking);
552                 MovieTrackingPlaneTrack *plane_track;
553
554                 for (plane_track = plane_tracks_base->first;
555                      plane_track;
556                      plane_track = plane_track->next)
557                 {
558                         if ((plane_track->flag & PLANE_TRACK_AUTOKEY) == 0) {
559                                 int track;
560                                 for (track = 0; track < context->num_tracks; ++track) {
561                                         if (BKE_tracking_plane_track_has_point_track(plane_track,
562                                                                                      context->options[track].track))
563                                         {
564                                                 BKE_tracking_track_plane_from_existing_motion(
565                                                         plane_track,
566                                                         context->first_frame);
567                                                 break;
568                                         }
569                                 }
570                         }
571                 }
572         }
573 }
574
575 void BKE_autotrack_context_free(AutoTrackContext *context)
576 {
577         libmv_autoTrackDestroy(context->autotrack);
578         tracking_image_accessor_destroy(context->image_accessor);
579         MEM_freeN(context->options);
580         MEM_freeN(context->tracks);
581         BLI_spin_end(&context->spin_lock);
582         MEM_freeN(context);
583 }