Cleanup: style, use braces for blenkernel
[blender.git] / source / blender / blenkernel / intern / tracking_solver.c
1 /*
2  * This program is free software; you can redistribute it and/or
3  * modify it under the terms of the GNU General Public License
4  * as published by the Free Software Foundation; either version 2
5  * of the License, or (at your option) any later version.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * along with this program; if not, write to the Free Software Foundation,
14  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15  *
16  * The Original Code is Copyright (C) 2011 Blender Foundation.
17  * All rights reserved.
18  */
19
20 /** \file
21  * \ingroup bke
22  *
23  * This file contains blender-side implementation of camera solver.
24  */
25
26 #include <limits.h>
27
28 #include "MEM_guardedalloc.h"
29
30 #include "DNA_anim_types.h"
31 #include "DNA_movieclip_types.h"
32
33 #include "BLI_utildefines.h"
34 #include "BLI_math.h"
35 #include "BLI_listbase.h"
36 #include "BLI_string.h"
37
38 #include "BLT_translation.h"
39
40 #include "BKE_fcurve.h"
41 #include "BKE_tracking.h"
42 #include "BKE_movieclip.h"
43
44 #include "RNA_access.h"
45
46 #include "libmv-capi.h"
47 #include "tracking_private.h"
48
49 typedef struct MovieReconstructContext {
50   struct libmv_Tracks *tracks;
51   bool select_keyframes;
52   int keyframe1, keyframe2;
53   int refine_flags;
54
55   struct libmv_Reconstruction *reconstruction;
56
57   char object_name[MAX_NAME];
58   bool is_camera;
59   short motion_flag;
60
61   libmv_CameraIntrinsicsOptions camera_intrinsics_options;
62
63   float reprojection_error;
64
65   TracksMap *tracks_map;
66
67   int sfra, efra;
68 } MovieReconstructContext;
69
70 typedef struct ReconstructProgressData {
71   short *stop;
72   short *do_update;
73   float *progress;
74   char *stats_message;
75   int message_size;
76 } ReconstructProgressData;
77
78 /* Create new libmv Tracks structure from blender's tracks list. */
79 static struct libmv_Tracks *libmv_tracks_new(MovieClip *clip,
80                                              ListBase *tracksbase,
81                                              int width,
82                                              int height)
83 {
84   int tracknr = 0;
85   MovieTrackingTrack *track;
86   struct libmv_Tracks *tracks = libmv_tracksNew();
87
88   track = tracksbase->first;
89   while (track) {
90     FCurve *weight_fcurve;
91     int a = 0;
92
93     weight_fcurve = id_data_find_fcurve(
94         &clip->id, track, &RNA_MovieTrackingTrack, "weight", 0, NULL);
95
96     for (a = 0; a < track->markersnr; a++) {
97       MovieTrackingMarker *marker = &track->markers[a];
98
99       if ((marker->flag & MARKER_DISABLED) == 0) {
100         float weight = track->weight;
101
102         if (weight_fcurve) {
103           int scene_framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
104           weight = evaluate_fcurve(weight_fcurve, scene_framenr);
105         }
106
107         libmv_tracksInsert(tracks,
108                            marker->framenr,
109                            tracknr,
110                            (marker->pos[0] + track->offset[0]) * width,
111                            (marker->pos[1] + track->offset[1]) * height,
112                            weight);
113       }
114     }
115
116     track = track->next;
117     tracknr++;
118   }
119
120   return tracks;
121 }
122
123 /* Retrieve refined camera intrinsics from libmv to blender. */
124 static void reconstruct_retrieve_libmv_intrinsics(MovieReconstructContext *context,
125                                                   MovieTracking *tracking)
126 {
127   struct libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
128   struct libmv_CameraIntrinsics *libmv_intrinsics = libmv_reconstructionExtractIntrinsics(
129       libmv_reconstruction);
130
131   libmv_CameraIntrinsicsOptions camera_intrinsics_options;
132   libmv_cameraIntrinsicsExtractOptions(libmv_intrinsics, &camera_intrinsics_options);
133
134   tracking_trackingCameraFromIntrinscisOptions(tracking, &camera_intrinsics_options);
135 }
136
137 /* Retrieve reconstructed tracks from libmv to blender.
138  * Actually, this also copies reconstructed cameras
139  * from libmv to movie clip datablock.
140  */
141 static bool reconstruct_retrieve_libmv_tracks(MovieReconstructContext *context,
142                                               MovieTracking *tracking)
143 {
144   struct libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
145   MovieTrackingReconstruction *reconstruction = NULL;
146   MovieReconstructedCamera *reconstructed;
147   MovieTrackingTrack *track;
148   ListBase *tracksbase = NULL;
149   int tracknr = 0, a;
150   bool ok = true;
151   bool origin_set = false;
152   int sfra = context->sfra, efra = context->efra;
153   float imat[4][4];
154
155   if (context->is_camera) {
156     tracksbase = &tracking->tracks;
157     reconstruction = &tracking->reconstruction;
158   }
159   else {
160     MovieTrackingObject *object = BKE_tracking_object_get_named(tracking, context->object_name);
161
162     tracksbase = &object->tracks;
163     reconstruction = &object->reconstruction;
164   }
165
166   unit_m4(imat);
167
168   track = tracksbase->first;
169   while (track) {
170     double pos[3];
171
172     if (libmv_reprojectionPointForTrack(libmv_reconstruction, tracknr, pos)) {
173       track->bundle_pos[0] = pos[0];
174       track->bundle_pos[1] = pos[1];
175       track->bundle_pos[2] = pos[2];
176
177       track->flag |= TRACK_HAS_BUNDLE;
178       track->error = libmv_reprojectionErrorForTrack(libmv_reconstruction, tracknr);
179     }
180     else {
181       track->flag &= ~TRACK_HAS_BUNDLE;
182       ok = false;
183
184       printf("Unable to reconstruct position for track #%d '%s'\n", tracknr, track->name);
185     }
186
187     track = track->next;
188     tracknr++;
189   }
190
191   if (reconstruction->cameras) {
192     MEM_freeN(reconstruction->cameras);
193   }
194
195   reconstruction->camnr = 0;
196   reconstruction->cameras = NULL;
197   reconstructed = MEM_callocN((efra - sfra + 1) * sizeof(MovieReconstructedCamera),
198                               "temp reconstructed camera");
199
200   for (a = sfra; a <= efra; a++) {
201     double matd[4][4];
202
203     if (libmv_reprojectionCameraForImage(libmv_reconstruction, a, matd)) {
204       int i, j;
205       float mat[4][4];
206       float error = libmv_reprojectionErrorForImage(libmv_reconstruction, a);
207
208       for (i = 0; i < 4; i++) {
209         for (j = 0; j < 4; j++) {
210           mat[i][j] = matd[i][j];
211         }
212       }
213
214       /* Ensure first camera has got zero rotation and transform.
215        * This is essential for object tracking to work -- this way
216        * we'll always know object and environment are properly
217        * oriented.
218        *
219        * There's one weak part tho, which is requirement object
220        * motion starts at the same frame as camera motion does,
221        * otherwise that;' be a russian roulette whether object is
222        * aligned correct or not.
223        */
224       if (!origin_set) {
225         invert_m4_m4(imat, mat);
226         unit_m4(mat);
227         origin_set = true;
228       }
229       else {
230         mul_m4_m4m4(mat, imat, mat);
231       }
232
233       copy_m4_m4(reconstructed[reconstruction->camnr].mat, mat);
234       reconstructed[reconstruction->camnr].framenr = a;
235       reconstructed[reconstruction->camnr].error = error;
236       reconstruction->camnr++;
237     }
238     else {
239       ok = false;
240       printf("No camera for frame %d\n", a);
241     }
242   }
243
244   if (reconstruction->camnr) {
245     int size = reconstruction->camnr * sizeof(MovieReconstructedCamera);
246     reconstruction->cameras = MEM_callocN(size, "reconstructed camera");
247     memcpy(reconstruction->cameras, reconstructed, size);
248   }
249
250   if (origin_set) {
251     track = tracksbase->first;
252     while (track) {
253       if (track->flag & TRACK_HAS_BUNDLE) {
254         mul_v3_m4v3(track->bundle_pos, imat, track->bundle_pos);
255       }
256
257       track = track->next;
258     }
259   }
260
261   MEM_freeN(reconstructed);
262
263   return ok;
264 }
265
266 /* Retrieve all the libmv data from context to blender's side data blocks. */
267 static int reconstruct_retrieve_libmv(MovieReconstructContext *context, MovieTracking *tracking)
268 {
269   /* take the intrinsics back from libmv */
270   reconstruct_retrieve_libmv_intrinsics(context, tracking);
271
272   return reconstruct_retrieve_libmv_tracks(context, tracking);
273 }
274
275 /* Convert blender's refinement flags to libmv's. */
276 static int reconstruct_refine_intrinsics_get_flags(MovieTracking *tracking,
277                                                    MovieTrackingObject *object)
278 {
279   int refine = tracking->settings.refine_camera_intrinsics;
280   int flags = 0;
281
282   if ((object->flag & TRACKING_OBJECT_CAMERA) == 0) {
283     return 0;
284   }
285
286   if (refine & REFINE_FOCAL_LENGTH) {
287     flags |= LIBMV_REFINE_FOCAL_LENGTH;
288   }
289
290   if (refine & REFINE_PRINCIPAL_POINT) {
291     flags |= LIBMV_REFINE_PRINCIPAL_POINT;
292   }
293
294   if (refine & REFINE_RADIAL_DISTORTION_K1) {
295     flags |= LIBMV_REFINE_RADIAL_DISTORTION_K1;
296   }
297
298   if (refine & REFINE_RADIAL_DISTORTION_K2) {
299     flags |= LIBMV_REFINE_RADIAL_DISTORTION_K2;
300   }
301
302   return flags;
303 }
304
305 /* Count tracks which has markers at both of keyframes. */
306 static int reconstruct_count_tracks_on_both_keyframes(MovieTracking *tracking,
307                                                       MovieTrackingObject *object)
308 {
309   ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
310   int tot = 0;
311   int frame1 = object->keyframe1, frame2 = object->keyframe2;
312   MovieTrackingTrack *track;
313
314   track = tracksbase->first;
315   while (track) {
316     if (BKE_tracking_track_has_enabled_marker_at_frame(track, frame1)) {
317       if (BKE_tracking_track_has_enabled_marker_at_frame(track, frame2)) {
318         tot++;
319       }
320     }
321
322     track = track->next;
323   }
324
325   return tot;
326 }
327
328 /* Perform early check on whether everything is fine to start reconstruction. */
329 bool BKE_tracking_reconstruction_check(MovieTracking *tracking,
330                                        MovieTrackingObject *object,
331                                        char *error_msg,
332                                        int error_size)
333 {
334   if (tracking->settings.motion_flag & TRACKING_MOTION_MODAL) {
335     /* TODO: check for number of tracks? */
336     return true;
337   }
338   else if ((tracking->settings.reconstruction_flag & TRACKING_USE_KEYFRAME_SELECTION) == 0) {
339     /* automatic keyframe selection does not require any pre-process checks */
340     if (reconstruct_count_tracks_on_both_keyframes(tracking, object) < 8) {
341       BLI_strncpy(error_msg,
342                   N_("At least 8 common tracks on both keyframes are needed for reconstruction"),
343                   error_size);
344
345       return false;
346     }
347   }
348
349 #ifndef WITH_LIBMV
350   BLI_strncpy(error_msg, N_("Blender is compiled without motion tracking library"), error_size);
351   return false;
352 #endif
353
354   return true;
355 }
356
357 /* Create context for camera/object motion reconstruction.
358  * Copies all data needed for reconstruction from movie
359  * clip datablock, so editing this clip is safe during
360  * reconstruction job is in progress.
361  */
362 MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieClip *clip,
363                                                                  MovieTrackingObject *object,
364                                                                  int keyframe1,
365                                                                  int keyframe2,
366                                                                  int width,
367                                                                  int height)
368 {
369   MovieTracking *tracking = &clip->tracking;
370   MovieReconstructContext *context = MEM_callocN(sizeof(MovieReconstructContext),
371                                                  "MovieReconstructContext data");
372   ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
373   float aspy = 1.0f / tracking->camera.pixel_aspect;
374   int num_tracks = BLI_listbase_count(tracksbase);
375   int sfra = INT_MAX, efra = INT_MIN;
376   MovieTrackingTrack *track;
377
378   BLI_strncpy(context->object_name, object->name, sizeof(context->object_name));
379   context->is_camera = object->flag & TRACKING_OBJECT_CAMERA;
380   context->motion_flag = tracking->settings.motion_flag;
381
382   context->select_keyframes = (tracking->settings.reconstruction_flag &
383                                TRACKING_USE_KEYFRAME_SELECTION) != 0;
384
385   tracking_cameraIntrinscisOptionsFromTracking(
386       tracking, width, height, &context->camera_intrinsics_options);
387
388   context->tracks_map = tracks_map_new(context->object_name, context->is_camera, num_tracks, 0);
389
390   track = tracksbase->first;
391   while (track) {
392     int first = 0, last = track->markersnr - 1;
393     MovieTrackingMarker *first_marker = &track->markers[0];
394     MovieTrackingMarker *last_marker = &track->markers[track->markersnr - 1];
395
396     /* find first not-disabled marker */
397     while (first <= track->markersnr - 1 && first_marker->flag & MARKER_DISABLED) {
398       first++;
399       first_marker++;
400     }
401
402     /* find last not-disabled marker */
403     while (last >= 0 && last_marker->flag & MARKER_DISABLED) {
404       last--;
405       last_marker--;
406     }
407
408     if (first <= track->markersnr - 1) {
409       sfra = min_ii(sfra, first_marker->framenr);
410     }
411
412     if (last >= 0) {
413       efra = max_ii(efra, last_marker->framenr);
414     }
415
416     tracks_map_insert(context->tracks_map, track, NULL);
417
418     track = track->next;
419   }
420
421   context->sfra = sfra;
422   context->efra = efra;
423
424   context->tracks = libmv_tracks_new(clip, tracksbase, width, height * aspy);
425   context->keyframe1 = keyframe1;
426   context->keyframe2 = keyframe2;
427   context->refine_flags = reconstruct_refine_intrinsics_get_flags(tracking, object);
428
429   return context;
430 }
431
432 /* Free memory used by a reconstruction process. */
433 void BKE_tracking_reconstruction_context_free(MovieReconstructContext *context)
434 {
435   if (context->reconstruction) {
436     libmv_reconstructionDestroy(context->reconstruction);
437   }
438
439   libmv_tracksDestroy(context->tracks);
440
441   tracks_map_free(context->tracks_map, NULL);
442
443   MEM_freeN(context);
444 }
445
446 /* Callback which is called from libmv side to update progress in the interface. */
447 static void reconstruct_update_solve_cb(void *customdata, double progress, const char *message)
448 {
449   ReconstructProgressData *progressdata = customdata;
450
451   if (progressdata->progress) {
452     *progressdata->progress = progress;
453     *progressdata->do_update = true;
454   }
455
456   BLI_snprintf(
457       progressdata->stats_message, progressdata->message_size, "Solving camera | %s", message);
458 }
459
460 /* Fill in reconstruction options structure from reconstruction context. */
461 static void reconstructionOptionsFromContext(libmv_ReconstructionOptions *reconstruction_options,
462                                              MovieReconstructContext *context)
463 {
464   reconstruction_options->select_keyframes = context->select_keyframes;
465
466   reconstruction_options->keyframe1 = context->keyframe1;
467   reconstruction_options->keyframe2 = context->keyframe2;
468
469   reconstruction_options->refine_intrinsics = context->refine_flags;
470 }
471
472 /* Solve camera/object motion and reconstruct 3D markers position
473  * from a prepared reconstruction context.
474  *
475  * stop is not actually used at this moment, so reconstruction
476  * job could not be stopped.
477  *
478  * do_update, progress and stat_message are set by reconstruction
479  * callback in libmv side and passing to an interface.
480  */
481 void BKE_tracking_reconstruction_solve(MovieReconstructContext *context,
482                                        short *stop,
483                                        short *do_update,
484                                        float *progress,
485                                        char *stats_message,
486                                        int message_size)
487 {
488   float error;
489
490   ReconstructProgressData progressdata;
491
492   libmv_ReconstructionOptions reconstruction_options;
493
494   progressdata.stop = stop;
495   progressdata.do_update = do_update;
496   progressdata.progress = progress;
497   progressdata.stats_message = stats_message;
498   progressdata.message_size = message_size;
499
500   reconstructionOptionsFromContext(&reconstruction_options, context);
501
502   if (context->motion_flag & TRACKING_MOTION_MODAL) {
503     context->reconstruction = libmv_solveModal(context->tracks,
504                                                &context->camera_intrinsics_options,
505                                                &reconstruction_options,
506                                                reconstruct_update_solve_cb,
507                                                &progressdata);
508   }
509   else {
510     context->reconstruction = libmv_solveReconstruction(context->tracks,
511                                                         &context->camera_intrinsics_options,
512                                                         &reconstruction_options,
513                                                         reconstruct_update_solve_cb,
514                                                         &progressdata);
515
516     if (context->select_keyframes) {
517       /* store actual keyframes used for reconstruction to update them in the interface later */
518       context->keyframe1 = reconstruction_options.keyframe1;
519       context->keyframe2 = reconstruction_options.keyframe2;
520     }
521   }
522
523   error = libmv_reprojectionError(context->reconstruction);
524
525   context->reprojection_error = error;
526 }
527
528 /* Finish reconstruction process by copying reconstructed data
529  * to an actual movie clip datablock.
530  */
531 bool BKE_tracking_reconstruction_finish(MovieReconstructContext *context, MovieTracking *tracking)
532 {
533   MovieTrackingReconstruction *reconstruction;
534   MovieTrackingObject *object;
535
536   if (!libmv_reconstructionIsValid(context->reconstruction)) {
537     printf("Failed solve the motion: most likely there are no good keyframes\n");
538     return false;
539   }
540
541   tracks_map_merge(context->tracks_map, tracking);
542   BKE_tracking_dopesheet_tag_update(tracking);
543
544   object = BKE_tracking_object_get_named(tracking, context->object_name);
545
546   if (context->is_camera) {
547     reconstruction = &tracking->reconstruction;
548   }
549   else {
550     reconstruction = &object->reconstruction;
551   }
552
553   /* update keyframe in the interface */
554   if (context->select_keyframes) {
555     object->keyframe1 = context->keyframe1;
556     object->keyframe2 = context->keyframe2;
557   }
558
559   reconstruction->error = context->reprojection_error;
560   reconstruction->flag |= TRACKING_RECONSTRUCTED;
561
562   if (!reconstruct_retrieve_libmv(context, tracking)) {
563     return false;
564   }
565
566   return true;
567 }
568
569 static void tracking_scale_reconstruction(ListBase *tracksbase,
570                                           MovieTrackingReconstruction *reconstruction,
571                                           const float scale[3])
572 {
573   MovieTrackingTrack *track;
574   int i;
575   float first_camera_delta[3] = {0.0f, 0.0f, 0.0f};
576
577   if (reconstruction->camnr > 0) {
578     mul_v3_v3v3(first_camera_delta, reconstruction->cameras[0].mat[3], scale);
579   }
580
581   for (i = 0; i < reconstruction->camnr; i++) {
582     MovieReconstructedCamera *camera = &reconstruction->cameras[i];
583     mul_v3_v3(camera->mat[3], scale);
584     sub_v3_v3(camera->mat[3], first_camera_delta);
585   }
586
587   for (track = tracksbase->first; track; track = track->next) {
588     if (track->flag & TRACK_HAS_BUNDLE) {
589       mul_v3_v3(track->bundle_pos, scale);
590       sub_v3_v3(track->bundle_pos, first_camera_delta);
591     }
592   }
593 }
594
595 /* Apply scale on all reconstructed cameras and bundles,
596  * used by camera scale apply operator.
597  */
598 void BKE_tracking_reconstruction_scale(MovieTracking *tracking, float scale[3])
599 {
600   MovieTrackingObject *object;
601
602   for (object = tracking->objects.first; object; object = object->next) {
603     ListBase *tracksbase;
604     MovieTrackingReconstruction *reconstruction;
605
606     tracksbase = BKE_tracking_object_get_tracks(tracking, object);
607     reconstruction = BKE_tracking_object_get_reconstruction(tracking, object);
608
609     tracking_scale_reconstruction(tracksbase, reconstruction, scale);
610   }
611 }