Cleanup: whitespace, unused var
[blender.git] / source / blender / blenkernel / intern / tracking_solver.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2011 Blender Foundation.
19  * All rights reserved.
20  *
21  * Contributor(s): Blender Foundation,
22  *                 Sergey Sharybin
23  *                 Keir Mierle
24  *
25  * ***** END GPL LICENSE BLOCK *****
26  */
27
28 /** \file blender/blenkernel/intern/tracking_solver.c
29  *  \ingroup bke
30  *
31  * This file contains blender-side implementation of camera solver.
32  */
33
34 #include <limits.h>
35
36 #include "MEM_guardedalloc.h"
37
38 #include "DNA_anim_types.h"
39 #include "DNA_movieclip_types.h"
40
41 #include "BLI_utildefines.h"
42 #include "BLI_math.h"
43 #include "BLI_listbase.h"
44 #include "BLI_string.h"
45
46 #include "BLF_translation.h"
47
48 #include "BKE_fcurve.h"
49 #include "BKE_tracking.h"
50 #include "BKE_movieclip.h"
51
52 #include "RNA_access.h"
53
54 #include "libmv-capi.h"
55 #include "tracking_private.h"
56
57 typedef struct MovieReconstructContext {
58         struct libmv_Tracks *tracks;
59         bool select_keyframes;
60         int keyframe1, keyframe2;
61         int refine_flags;
62
63         struct libmv_Reconstruction *reconstruction;
64
65         char object_name[MAX_NAME];
66         bool is_camera;
67         short motion_flag;
68
69         libmv_CameraIntrinsicsOptions camera_intrinsics_options;
70
71         float reprojection_error;
72
73         TracksMap *tracks_map;
74
75         int sfra, efra;
76 } MovieReconstructContext;
77
78 typedef struct ReconstructProgressData {
79         short *stop;
80         short *do_update;
81         float *progress;
82         char *stats_message;
83         int message_size;
84 } ReconstructProgressData;
85
86 /* Create new libmv Tracks structure from blender's tracks list. */
87 static struct libmv_Tracks *libmv_tracks_new(MovieClip *clip, ListBase *tracksbase, int width, int height)
88 {
89         int tracknr = 0;
90         MovieTrackingTrack *track;
91         struct libmv_Tracks *tracks = libmv_tracksNew();
92
93         track = tracksbase->first;
94         while (track) {
95                 FCurve *weight_fcurve;
96                 int a = 0;
97
98                 weight_fcurve = id_data_find_fcurve(&clip->id, track, &RNA_MovieTrackingTrack,
99                                                     "weight", 0, NULL);
100
101                 for (a = 0; a < track->markersnr; a++) {
102                         MovieTrackingMarker *marker = &track->markers[a];
103
104                         if ((marker->flag & MARKER_DISABLED) == 0) {
105                                 float weight = track->weight;
106
107                                 if (weight_fcurve) {
108                                         int scene_framenr =
109                                                 BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
110                                         weight = evaluate_fcurve(weight_fcurve, scene_framenr);
111                                 }
112
113                                 libmv_tracksInsert(tracks, marker->framenr, tracknr,
114                                                    (marker->pos[0] + track->offset[0]) * width,
115                                                    (marker->pos[1] + track->offset[1]) * height,
116                                                    weight);
117                         }
118                 }
119
120                 track = track->next;
121                 tracknr++;
122         }
123
124         return tracks;
125 }
126
127 /* Retrieve refined camera intrinsics from libmv to blender. */
128 static void reconstruct_retrieve_libmv_intrinsics(MovieReconstructContext *context, MovieTracking *tracking)
129 {
130         struct libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
131         struct libmv_CameraIntrinsics *libmv_intrinsics = libmv_reconstructionExtractIntrinsics(libmv_reconstruction);
132
133         libmv_CameraIntrinsicsOptions camera_intrinsics_options;
134         libmv_cameraIntrinsicsExtractOptions(libmv_intrinsics, &camera_intrinsics_options);
135
136         tracking_trackingCameraFromIntrinscisOptions(tracking,
137                                                      &camera_intrinsics_options);
138 }
139
140 /* Retrieve reconstructed tracks from libmv to blender.
141  * Actually, this also copies reconstructed cameras
142  * from libmv to movie clip datablock.
143  */
144 static bool reconstruct_retrieve_libmv_tracks(MovieReconstructContext *context, MovieTracking *tracking)
145 {
146         struct libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
147         MovieTrackingReconstruction *reconstruction = NULL;
148         MovieReconstructedCamera *reconstructed;
149         MovieTrackingTrack *track;
150         ListBase *tracksbase =  NULL;
151         int tracknr = 0, a;
152         bool ok = true;
153         bool origin_set = false;
154         int sfra = context->sfra, efra = context->efra;
155         float imat[4][4];
156
157         if (context->is_camera) {
158                 tracksbase = &tracking->tracks;
159                 reconstruction = &tracking->reconstruction;
160         }
161         else {
162                 MovieTrackingObject *object = BKE_tracking_object_get_named(tracking, context->object_name);
163
164                 tracksbase = &object->tracks;
165                 reconstruction = &object->reconstruction;
166         }
167
168         unit_m4(imat);
169
170         track = tracksbase->first;
171         while (track) {
172                 double pos[3];
173
174                 if (libmv_reprojectionPointForTrack(libmv_reconstruction, tracknr, pos)) {
175                         track->bundle_pos[0] = pos[0];
176                         track->bundle_pos[1] = pos[1];
177                         track->bundle_pos[2] = pos[2];
178
179                         track->flag |= TRACK_HAS_BUNDLE;
180                         track->error = libmv_reprojectionErrorForTrack(libmv_reconstruction, tracknr);
181                 }
182                 else {
183                         track->flag &= ~TRACK_HAS_BUNDLE;
184                         ok = false;
185
186                         printf("Unable to reconstruct position for track #%d '%s'\n", tracknr, track->name);
187                 }
188
189                 track = track->next;
190                 tracknr++;
191         }
192
193         if (reconstruction->cameras)
194                 MEM_freeN(reconstruction->cameras);
195
196         reconstruction->camnr = 0;
197         reconstruction->cameras = NULL;
198         reconstructed = MEM_callocN((efra - sfra + 1) * sizeof(MovieReconstructedCamera),
199                                     "temp reconstructed camera");
200
201         for (a = sfra; a <= efra; a++) {
202                 double matd[4][4];
203
204                 if (libmv_reprojectionCameraForImage(libmv_reconstruction, a, matd)) {
205                         int i, j;
206                         float mat[4][4];
207                         float error = libmv_reprojectionErrorForImage(libmv_reconstruction, a);
208
209                         for (i = 0; i < 4; i++) {
210                                 for (j = 0; j < 4; j++)
211                                         mat[i][j] = matd[i][j];
212                         }
213
214                         /* Ensure first camera has got zero rotation and transform.
215                          * This is essential for object tracking to work -- this way
216                          * we'll always know object and environment are properly
217                          * oriented.
218                          *
219                          * There's one weak part tho, which is requirement object
220                          * motion starts at the same frame as camera motion does,
221                          * otherwise that;' be a russian roulette whether object is
222                          * aligned correct or not.
223                          */
224                         if (!origin_set) {
225                                 invert_m4_m4(imat, mat);
226                                 unit_m4(mat);
227                                 origin_set = true;
228                         }
229                         else {
230                                 mul_m4_m4m4(mat, imat, mat);
231                         }
232
233                         copy_m4_m4(reconstructed[reconstruction->camnr].mat, mat);
234                         reconstructed[reconstruction->camnr].framenr = a;
235                         reconstructed[reconstruction->camnr].error = error;
236                         reconstruction->camnr++;
237                 }
238                 else {
239                         ok = false;
240                         printf("No camera for frame %d\n", a);
241                 }
242         }
243
244         if (reconstruction->camnr) {
245                 int size = reconstruction->camnr * sizeof(MovieReconstructedCamera);
246                 reconstruction->cameras = MEM_callocN(size, "reconstructed camera");
247                 memcpy(reconstruction->cameras, reconstructed, size);
248         }
249
250         if (origin_set) {
251                 track = tracksbase->first;
252                 while (track) {
253                         if (track->flag & TRACK_HAS_BUNDLE)
254                                 mul_v3_m4v3(track->bundle_pos, imat, track->bundle_pos);
255
256                         track = track->next;
257                 }
258         }
259
260         MEM_freeN(reconstructed);
261
262         return ok;
263 }
264
265 /* Retrieve all the libmv data from context to blender's side data blocks. */
266 static int reconstruct_retrieve_libmv(MovieReconstructContext *context, MovieTracking *tracking)
267 {
268         /* take the intrinsics back from libmv */
269         reconstruct_retrieve_libmv_intrinsics(context, tracking);
270
271         return reconstruct_retrieve_libmv_tracks(context, tracking);
272 }
273
274 /* Convert blender's refinement flags to libmv's. */
275 static int reconstruct_refine_intrinsics_get_flags(MovieTracking *tracking, MovieTrackingObject *object)
276 {
277         int refine = tracking->settings.refine_camera_intrinsics;
278         int flags = 0;
279
280         if ((object->flag & TRACKING_OBJECT_CAMERA) == 0)
281                 return 0;
282
283         if (refine & REFINE_FOCAL_LENGTH)
284                 flags |= LIBMV_REFINE_FOCAL_LENGTH;
285
286         if (refine & REFINE_PRINCIPAL_POINT)
287                 flags |= LIBMV_REFINE_PRINCIPAL_POINT;
288
289         if (refine & REFINE_RADIAL_DISTORTION_K1)
290                 flags |= LIBMV_REFINE_RADIAL_DISTORTION_K1;
291
292         if (refine & REFINE_RADIAL_DISTORTION_K2)
293                 flags |= LIBMV_REFINE_RADIAL_DISTORTION_K2;
294
295         return flags;
296 }
297
298 /* Count tracks which has markers at both of keyframes. */
299 static int reconstruct_count_tracks_on_both_keyframes(MovieTracking *tracking, MovieTrackingObject *object)
300 {
301         ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
302         int tot = 0;
303         int frame1 = object->keyframe1, frame2 = object->keyframe2;
304         MovieTrackingTrack *track;
305
306         track = tracksbase->first;
307         while (track) {
308                 if (BKE_tracking_track_has_enabled_marker_at_frame(track, frame1)) {
309                         if (BKE_tracking_track_has_enabled_marker_at_frame(track, frame2)) {
310                                 tot++;
311                         }
312                 }
313
314                 track = track->next;
315         }
316
317         return tot;
318 }
319
320 /* Perform early check on whether everything is fine to start reconstruction. */
321 bool BKE_tracking_reconstruction_check(MovieTracking *tracking, MovieTrackingObject *object,
322                                        char *error_msg, int error_size)
323 {
324         if (tracking->settings.motion_flag & TRACKING_MOTION_MODAL) {
325                 /* TODO: check for number of tracks? */
326                 return true;
327         }
328         else if ((tracking->settings.reconstruction_flag & TRACKING_USE_KEYFRAME_SELECTION) == 0) {
329                 /* automatic keyframe selection does not require any pre-process checks */
330                 if (reconstruct_count_tracks_on_both_keyframes(tracking, object) < 8) {
331                         BLI_strncpy(error_msg,
332                                     N_("At least 8 common tracks on both keyframes are needed for reconstruction"),
333                                     error_size);
334
335                         return false;
336                 }
337         }
338
339 #ifndef WITH_LIBMV
340         BLI_strncpy(error_msg, N_("Blender is compiled without motion tracking library"), error_size);
341         return false;
342 #endif
343
344         return true;
345 }
346
347 /* Create context for camera/object motion reconstruction.
348  * Copies all data needed for reconstruction from movie
349  * clip datablock, so editing this clip is safe during
350  * reconstruction job is in progress.
351  */
352 MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieClip *clip, MovieTrackingObject *object,
353                                                                  int keyframe1, int keyframe2, int width, int height)
354 {
355         MovieTracking *tracking = &clip->tracking;
356         MovieReconstructContext *context = MEM_callocN(sizeof(MovieReconstructContext), "MovieReconstructContext data");
357         ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
358         float aspy = 1.0f / tracking->camera.pixel_aspect;
359         int num_tracks = BLI_listbase_count(tracksbase);
360         int sfra = INT_MAX, efra = INT_MIN;
361         MovieTrackingTrack *track;
362
363         BLI_strncpy(context->object_name, object->name, sizeof(context->object_name));
364         context->is_camera = object->flag & TRACKING_OBJECT_CAMERA;
365         context->motion_flag = tracking->settings.motion_flag;
366
367         context->select_keyframes =
368                 (tracking->settings.reconstruction_flag & TRACKING_USE_KEYFRAME_SELECTION) != 0;
369
370         tracking_cameraIntrinscisOptionsFromTracking(tracking,
371                                                      width, height,
372                                                      &context->camera_intrinsics_options);
373
374         context->tracks_map = tracks_map_new(context->object_name, context->is_camera, num_tracks, 0);
375
376         track = tracksbase->first;
377         while (track) {
378                 int first = 0, last = track->markersnr - 1;
379                 MovieTrackingMarker *first_marker = &track->markers[0];
380                 MovieTrackingMarker *last_marker = &track->markers[track->markersnr - 1];
381
382                 /* find first not-disabled marker */
383                 while (first <= track->markersnr - 1 && first_marker->flag & MARKER_DISABLED) {
384                         first++;
385                         first_marker++;
386                 }
387
388                 /* find last not-disabled marker */
389                 while (last >= 0 && last_marker->flag & MARKER_DISABLED) {
390                         last--;
391                         last_marker--;
392                 }
393
394                 if (first <= track->markersnr - 1)
395                         sfra = min_ii(sfra, first_marker->framenr);
396
397                 if (last >= 0)
398                         efra = max_ii(efra, last_marker->framenr);
399
400                 tracks_map_insert(context->tracks_map, track, NULL);
401
402                 track = track->next;
403         }
404
405         context->sfra = sfra;
406         context->efra = efra;
407
408         context->tracks = libmv_tracks_new(clip, tracksbase, width, height * aspy);
409         context->keyframe1 = keyframe1;
410         context->keyframe2 = keyframe2;
411         context->refine_flags = reconstruct_refine_intrinsics_get_flags(tracking, object);
412
413         return context;
414 }
415
416 /* Free memory used by a reconstruction process. */
417 void BKE_tracking_reconstruction_context_free(MovieReconstructContext *context)
418 {
419         if (context->reconstruction)
420                 libmv_reconstructionDestroy(context->reconstruction);
421
422         libmv_tracksDestroy(context->tracks);
423
424         tracks_map_free(context->tracks_map, NULL);
425
426         MEM_freeN(context);
427 }
428
429 /* Callback which is called from libmv side to update progress in the interface. */
430 static void reconstruct_update_solve_cb(void *customdata, double progress, const char *message)
431 {
432         ReconstructProgressData *progressdata = customdata;
433
434         if (progressdata->progress) {
435                 *progressdata->progress = progress;
436                 *progressdata->do_update = true;
437         }
438
439         BLI_snprintf(progressdata->stats_message, progressdata->message_size, "Solving camera | %s", message);
440 }
441
442 /* Fill in reconstruction options structure from reconstruction context. */
443 static void reconstructionOptionsFromContext(libmv_ReconstructionOptions *reconstruction_options,
444                                              MovieReconstructContext *context)
445 {
446         reconstruction_options->select_keyframes = context->select_keyframes;
447
448         reconstruction_options->keyframe1 = context->keyframe1;
449         reconstruction_options->keyframe2 = context->keyframe2;
450
451         reconstruction_options->refine_intrinsics = context->refine_flags;
452 }
453
454 /* Solve camera/object motion and reconstruct 3D markers position
455  * from a prepared reconstruction context.
456  *
457  * stop is not actually used at this moment, so reconstruction
458  * job could not be stopped.
459  *
460  * do_update, progress and stat_message are set by reconstruction
461  * callback in libmv side and passing to an interface.
462  */
463 void BKE_tracking_reconstruction_solve(MovieReconstructContext *context, short *stop, short *do_update,
464                                        float *progress, char *stats_message, int message_size)
465 {
466         float error;
467
468         ReconstructProgressData progressdata;
469
470         libmv_ReconstructionOptions reconstruction_options;
471
472         progressdata.stop = stop;
473         progressdata.do_update = do_update;
474         progressdata.progress = progress;
475         progressdata.stats_message = stats_message;
476         progressdata.message_size = message_size;
477
478         reconstructionOptionsFromContext(&reconstruction_options, context);
479
480         if (context->motion_flag & TRACKING_MOTION_MODAL) {
481                 context->reconstruction = libmv_solveModal(context->tracks,
482                                                            &context->camera_intrinsics_options,
483                                                            &reconstruction_options,
484                                                            reconstruct_update_solve_cb, &progressdata);
485         }
486         else {
487                 context->reconstruction = libmv_solveReconstruction(context->tracks,
488                                                                     &context->camera_intrinsics_options,
489                                                                     &reconstruction_options,
490                                                                     reconstruct_update_solve_cb, &progressdata);
491
492                 if (context->select_keyframes) {
493                         /* store actual keyframes used for reconstruction to update them in the interface later */
494                         context->keyframe1 = reconstruction_options.keyframe1;
495                         context->keyframe2 = reconstruction_options.keyframe2;
496                 }
497         }
498
499         error = libmv_reprojectionError(context->reconstruction);
500
501         context->reprojection_error = error;
502 }
503
504 /* Finish reconstruction process by copying reconstructed data
505  * to an actual movie clip datablock.
506  */
507 bool BKE_tracking_reconstruction_finish(MovieReconstructContext *context, MovieTracking *tracking)
508 {
509         MovieTrackingReconstruction *reconstruction;
510         MovieTrackingObject *object;
511
512         if (!libmv_reconstructionIsValid(context->reconstruction)) {
513                 printf("Failed solve the motion: most likely there are no good keyframes\n");
514                 return false;
515         }
516
517         tracks_map_merge(context->tracks_map, tracking);
518         BKE_tracking_dopesheet_tag_update(tracking);
519
520         object = BKE_tracking_object_get_named(tracking, context->object_name);
521         
522         if (context->is_camera)
523                 reconstruction = &tracking->reconstruction;
524         else
525                 reconstruction = &object->reconstruction;
526
527         /* update keyframe in the interface */
528         if (context->select_keyframes) {
529                 object->keyframe1 = context->keyframe1;
530                 object->keyframe2 = context->keyframe2;
531         }
532
533         reconstruction->error = context->reprojection_error;
534         reconstruction->flag |= TRACKING_RECONSTRUCTED;
535
536         if (!reconstruct_retrieve_libmv(context, tracking))
537                 return false;
538
539         return true;
540 }
541
542 static void tracking_scale_reconstruction(ListBase *tracksbase, MovieTrackingReconstruction *reconstruction,
543                                           const float scale[3])
544 {
545         MovieTrackingTrack *track;
546         int i;
547         float first_camera_delta[3] = {0.0f, 0.0f, 0.0f};
548
549         if (reconstruction->camnr > 0) {
550                 mul_v3_v3v3(first_camera_delta, reconstruction->cameras[0].mat[3], scale);
551         }
552
553         for (i = 0; i < reconstruction->camnr; i++) {
554                 MovieReconstructedCamera *camera = &reconstruction->cameras[i];
555                 mul_v3_v3(camera->mat[3], scale);
556                 sub_v3_v3(camera->mat[3], first_camera_delta);
557         }
558
559         for (track = tracksbase->first; track; track = track->next) {
560                 if (track->flag & TRACK_HAS_BUNDLE) {
561                         mul_v3_v3(track->bundle_pos, scale);
562                         sub_v3_v3(track->bundle_pos, first_camera_delta);
563                 }
564         }
565 }
566
567 /* Apply scale on all reconstructed cameras and bundles,
568  * used by camera scale apply operator.
569  */
570 void BKE_tracking_reconstruction_scale(MovieTracking *tracking, float scale[3])
571 {
572         MovieTrackingObject *object;
573
574         for (object = tracking->objects.first; object; object = object->next) {
575                 ListBase *tracksbase;
576                 MovieTrackingReconstruction *reconstruction;
577
578                 tracksbase = BKE_tracking_object_get_tracks(tracking, object);
579                 reconstruction = BKE_tracking_object_get_reconstruction(tracking, object);
580
581                 tracking_scale_reconstruction(tracksbase, reconstruction, scale);
582         }
583 }