Merge branch 'blender2.7'
[blender.git] / source / blender / blenkernel / intern / tracking_solver.c
1 /*
2  * This program is free software; you can redistribute it and/or
3  * modify it under the terms of the GNU General Public License
4  * as published by the Free Software Foundation; either version 2
5  * of the License, or (at your option) any later version.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * along with this program; if not, write to the Free Software Foundation,
14  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15  *
16  * The Original Code is Copyright (C) 2011 Blender Foundation.
17  * All rights reserved.
18  */
19
20 /** \file
21  * \ingroup bke
22  *
23  * This file contains blender-side implementation of camera solver.
24  */
25
26 #include <limits.h>
27
28 #include "MEM_guardedalloc.h"
29
30 #include "DNA_anim_types.h"
31 #include "DNA_movieclip_types.h"
32
33 #include "BLI_utildefines.h"
34 #include "BLI_math.h"
35 #include "BLI_listbase.h"
36 #include "BLI_string.h"
37
38 #include "BLT_translation.h"
39
40 #include "BKE_fcurve.h"
41 #include "BKE_tracking.h"
42 #include "BKE_movieclip.h"
43
44 #include "RNA_access.h"
45
46 #include "libmv-capi.h"
47 #include "tracking_private.h"
48
49 typedef struct MovieReconstructContext {
50         struct libmv_Tracks *tracks;
51         bool select_keyframes;
52         int keyframe1, keyframe2;
53         int refine_flags;
54
55         struct libmv_Reconstruction *reconstruction;
56
57         char object_name[MAX_NAME];
58         bool is_camera;
59         short motion_flag;
60
61         libmv_CameraIntrinsicsOptions camera_intrinsics_options;
62
63         float reprojection_error;
64
65         TracksMap *tracks_map;
66
67         int sfra, efra;
68 } MovieReconstructContext;
69
70 typedef struct ReconstructProgressData {
71         short *stop;
72         short *do_update;
73         float *progress;
74         char *stats_message;
75         int message_size;
76 } ReconstructProgressData;
77
78 /* Create new libmv Tracks structure from blender's tracks list. */
79 static struct libmv_Tracks *libmv_tracks_new(MovieClip *clip, ListBase *tracksbase, int width, int height)
80 {
81         int tracknr = 0;
82         MovieTrackingTrack *track;
83         struct libmv_Tracks *tracks = libmv_tracksNew();
84
85         track = tracksbase->first;
86         while (track) {
87                 FCurve *weight_fcurve;
88                 int a = 0;
89
90                 weight_fcurve = id_data_find_fcurve(&clip->id, track, &RNA_MovieTrackingTrack,
91                                                     "weight", 0, NULL);
92
93                 for (a = 0; a < track->markersnr; a++) {
94                         MovieTrackingMarker *marker = &track->markers[a];
95
96                         if ((marker->flag & MARKER_DISABLED) == 0) {
97                                 float weight = track->weight;
98
99                                 if (weight_fcurve) {
100                                         int scene_framenr =
101                                                 BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
102                                         weight = evaluate_fcurve(weight_fcurve, scene_framenr);
103                                 }
104
105                                 libmv_tracksInsert(tracks, marker->framenr, tracknr,
106                                                    (marker->pos[0] + track->offset[0]) * width,
107                                                    (marker->pos[1] + track->offset[1]) * height,
108                                                    weight);
109                         }
110                 }
111
112                 track = track->next;
113                 tracknr++;
114         }
115
116         return tracks;
117 }
118
119 /* Retrieve refined camera intrinsics from libmv to blender. */
120 static void reconstruct_retrieve_libmv_intrinsics(MovieReconstructContext *context, MovieTracking *tracking)
121 {
122         struct libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
123         struct libmv_CameraIntrinsics *libmv_intrinsics = libmv_reconstructionExtractIntrinsics(libmv_reconstruction);
124
125         libmv_CameraIntrinsicsOptions camera_intrinsics_options;
126         libmv_cameraIntrinsicsExtractOptions(libmv_intrinsics, &camera_intrinsics_options);
127
128         tracking_trackingCameraFromIntrinscisOptions(tracking,
129                                                      &camera_intrinsics_options);
130 }
131
132 /* Retrieve reconstructed tracks from libmv to blender.
133  * Actually, this also copies reconstructed cameras
134  * from libmv to movie clip datablock.
135  */
136 static bool reconstruct_retrieve_libmv_tracks(MovieReconstructContext *context, MovieTracking *tracking)
137 {
138         struct libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
139         MovieTrackingReconstruction *reconstruction = NULL;
140         MovieReconstructedCamera *reconstructed;
141         MovieTrackingTrack *track;
142         ListBase *tracksbase =  NULL;
143         int tracknr = 0, a;
144         bool ok = true;
145         bool origin_set = false;
146         int sfra = context->sfra, efra = context->efra;
147         float imat[4][4];
148
149         if (context->is_camera) {
150                 tracksbase = &tracking->tracks;
151                 reconstruction = &tracking->reconstruction;
152         }
153         else {
154                 MovieTrackingObject *object = BKE_tracking_object_get_named(tracking, context->object_name);
155
156                 tracksbase = &object->tracks;
157                 reconstruction = &object->reconstruction;
158         }
159
160         unit_m4(imat);
161
162         track = tracksbase->first;
163         while (track) {
164                 double pos[3];
165
166                 if (libmv_reprojectionPointForTrack(libmv_reconstruction, tracknr, pos)) {
167                         track->bundle_pos[0] = pos[0];
168                         track->bundle_pos[1] = pos[1];
169                         track->bundle_pos[2] = pos[2];
170
171                         track->flag |= TRACK_HAS_BUNDLE;
172                         track->error = libmv_reprojectionErrorForTrack(libmv_reconstruction, tracknr);
173                 }
174                 else {
175                         track->flag &= ~TRACK_HAS_BUNDLE;
176                         ok = false;
177
178                         printf("Unable to reconstruct position for track #%d '%s'\n", tracknr, track->name);
179                 }
180
181                 track = track->next;
182                 tracknr++;
183         }
184
185         if (reconstruction->cameras)
186                 MEM_freeN(reconstruction->cameras);
187
188         reconstruction->camnr = 0;
189         reconstruction->cameras = NULL;
190         reconstructed = MEM_callocN((efra - sfra + 1) * sizeof(MovieReconstructedCamera),
191                                     "temp reconstructed camera");
192
193         for (a = sfra; a <= efra; a++) {
194                 double matd[4][4];
195
196                 if (libmv_reprojectionCameraForImage(libmv_reconstruction, a, matd)) {
197                         int i, j;
198                         float mat[4][4];
199                         float error = libmv_reprojectionErrorForImage(libmv_reconstruction, a);
200
201                         for (i = 0; i < 4; i++) {
202                                 for (j = 0; j < 4; j++)
203                                         mat[i][j] = matd[i][j];
204                         }
205
206                         /* Ensure first camera has got zero rotation and transform.
207                          * This is essential for object tracking to work -- this way
208                          * we'll always know object and environment are properly
209                          * oriented.
210                          *
211                          * There's one weak part tho, which is requirement object
212                          * motion starts at the same frame as camera motion does,
213                          * otherwise that;' be a russian roulette whether object is
214                          * aligned correct or not.
215                          */
216                         if (!origin_set) {
217                                 invert_m4_m4(imat, mat);
218                                 unit_m4(mat);
219                                 origin_set = true;
220                         }
221                         else {
222                                 mul_m4_m4m4(mat, imat, mat);
223                         }
224
225                         copy_m4_m4(reconstructed[reconstruction->camnr].mat, mat);
226                         reconstructed[reconstruction->camnr].framenr = a;
227                         reconstructed[reconstruction->camnr].error = error;
228                         reconstruction->camnr++;
229                 }
230                 else {
231                         ok = false;
232                         printf("No camera for frame %d\n", a);
233                 }
234         }
235
236         if (reconstruction->camnr) {
237                 int size = reconstruction->camnr * sizeof(MovieReconstructedCamera);
238                 reconstruction->cameras = MEM_callocN(size, "reconstructed camera");
239                 memcpy(reconstruction->cameras, reconstructed, size);
240         }
241
242         if (origin_set) {
243                 track = tracksbase->first;
244                 while (track) {
245                         if (track->flag & TRACK_HAS_BUNDLE)
246                                 mul_v3_m4v3(track->bundle_pos, imat, track->bundle_pos);
247
248                         track = track->next;
249                 }
250         }
251
252         MEM_freeN(reconstructed);
253
254         return ok;
255 }
256
257 /* Retrieve all the libmv data from context to blender's side data blocks. */
258 static int reconstruct_retrieve_libmv(MovieReconstructContext *context, MovieTracking *tracking)
259 {
260         /* take the intrinsics back from libmv */
261         reconstruct_retrieve_libmv_intrinsics(context, tracking);
262
263         return reconstruct_retrieve_libmv_tracks(context, tracking);
264 }
265
266 /* Convert blender's refinement flags to libmv's. */
267 static int reconstruct_refine_intrinsics_get_flags(MovieTracking *tracking, MovieTrackingObject *object)
268 {
269         int refine = tracking->settings.refine_camera_intrinsics;
270         int flags = 0;
271
272         if ((object->flag & TRACKING_OBJECT_CAMERA) == 0)
273                 return 0;
274
275         if (refine & REFINE_FOCAL_LENGTH)
276                 flags |= LIBMV_REFINE_FOCAL_LENGTH;
277
278         if (refine & REFINE_PRINCIPAL_POINT)
279                 flags |= LIBMV_REFINE_PRINCIPAL_POINT;
280
281         if (refine & REFINE_RADIAL_DISTORTION_K1)
282                 flags |= LIBMV_REFINE_RADIAL_DISTORTION_K1;
283
284         if (refine & REFINE_RADIAL_DISTORTION_K2)
285                 flags |= LIBMV_REFINE_RADIAL_DISTORTION_K2;
286
287         return flags;
288 }
289
290 /* Count tracks which has markers at both of keyframes. */
291 static int reconstruct_count_tracks_on_both_keyframes(MovieTracking *tracking, MovieTrackingObject *object)
292 {
293         ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
294         int tot = 0;
295         int frame1 = object->keyframe1, frame2 = object->keyframe2;
296         MovieTrackingTrack *track;
297
298         track = tracksbase->first;
299         while (track) {
300                 if (BKE_tracking_track_has_enabled_marker_at_frame(track, frame1)) {
301                         if (BKE_tracking_track_has_enabled_marker_at_frame(track, frame2)) {
302                                 tot++;
303                         }
304                 }
305
306                 track = track->next;
307         }
308
309         return tot;
310 }
311
312 /* Perform early check on whether everything is fine to start reconstruction. */
313 bool BKE_tracking_reconstruction_check(MovieTracking *tracking, MovieTrackingObject *object,
314                                        char *error_msg, int error_size)
315 {
316         if (tracking->settings.motion_flag & TRACKING_MOTION_MODAL) {
317                 /* TODO: check for number of tracks? */
318                 return true;
319         }
320         else if ((tracking->settings.reconstruction_flag & TRACKING_USE_KEYFRAME_SELECTION) == 0) {
321                 /* automatic keyframe selection does not require any pre-process checks */
322                 if (reconstruct_count_tracks_on_both_keyframes(tracking, object) < 8) {
323                         BLI_strncpy(error_msg,
324                                     N_("At least 8 common tracks on both keyframes are needed for reconstruction"),
325                                     error_size);
326
327                         return false;
328                 }
329         }
330
331 #ifndef WITH_LIBMV
332         BLI_strncpy(error_msg, N_("Blender is compiled without motion tracking library"), error_size);
333         return false;
334 #endif
335
336         return true;
337 }
338
339 /* Create context for camera/object motion reconstruction.
340  * Copies all data needed for reconstruction from movie
341  * clip datablock, so editing this clip is safe during
342  * reconstruction job is in progress.
343  */
344 MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieClip *clip, MovieTrackingObject *object,
345                                                                  int keyframe1, int keyframe2, int width, int height)
346 {
347         MovieTracking *tracking = &clip->tracking;
348         MovieReconstructContext *context = MEM_callocN(sizeof(MovieReconstructContext), "MovieReconstructContext data");
349         ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
350         float aspy = 1.0f / tracking->camera.pixel_aspect;
351         int num_tracks = BLI_listbase_count(tracksbase);
352         int sfra = INT_MAX, efra = INT_MIN;
353         MovieTrackingTrack *track;
354
355         BLI_strncpy(context->object_name, object->name, sizeof(context->object_name));
356         context->is_camera = object->flag & TRACKING_OBJECT_CAMERA;
357         context->motion_flag = tracking->settings.motion_flag;
358
359         context->select_keyframes =
360                 (tracking->settings.reconstruction_flag & TRACKING_USE_KEYFRAME_SELECTION) != 0;
361
362         tracking_cameraIntrinscisOptionsFromTracking(tracking,
363                                                      width, height,
364                                                      &context->camera_intrinsics_options);
365
366         context->tracks_map = tracks_map_new(context->object_name, context->is_camera, num_tracks, 0);
367
368         track = tracksbase->first;
369         while (track) {
370                 int first = 0, last = track->markersnr - 1;
371                 MovieTrackingMarker *first_marker = &track->markers[0];
372                 MovieTrackingMarker *last_marker = &track->markers[track->markersnr - 1];
373
374                 /* find first not-disabled marker */
375                 while (first <= track->markersnr - 1 && first_marker->flag & MARKER_DISABLED) {
376                         first++;
377                         first_marker++;
378                 }
379
380                 /* find last not-disabled marker */
381                 while (last >= 0 && last_marker->flag & MARKER_DISABLED) {
382                         last--;
383                         last_marker--;
384                 }
385
386                 if (first <= track->markersnr - 1)
387                         sfra = min_ii(sfra, first_marker->framenr);
388
389                 if (last >= 0)
390                         efra = max_ii(efra, last_marker->framenr);
391
392                 tracks_map_insert(context->tracks_map, track, NULL);
393
394                 track = track->next;
395         }
396
397         context->sfra = sfra;
398         context->efra = efra;
399
400         context->tracks = libmv_tracks_new(clip, tracksbase, width, height * aspy);
401         context->keyframe1 = keyframe1;
402         context->keyframe2 = keyframe2;
403         context->refine_flags = reconstruct_refine_intrinsics_get_flags(tracking, object);
404
405         return context;
406 }
407
408 /* Free memory used by a reconstruction process. */
409 void BKE_tracking_reconstruction_context_free(MovieReconstructContext *context)
410 {
411         if (context->reconstruction)
412                 libmv_reconstructionDestroy(context->reconstruction);
413
414         libmv_tracksDestroy(context->tracks);
415
416         tracks_map_free(context->tracks_map, NULL);
417
418         MEM_freeN(context);
419 }
420
421 /* Callback which is called from libmv side to update progress in the interface. */
422 static void reconstruct_update_solve_cb(void *customdata, double progress, const char *message)
423 {
424         ReconstructProgressData *progressdata = customdata;
425
426         if (progressdata->progress) {
427                 *progressdata->progress = progress;
428                 *progressdata->do_update = true;
429         }
430
431         BLI_snprintf(progressdata->stats_message, progressdata->message_size, "Solving camera | %s", message);
432 }
433
434 /* Fill in reconstruction options structure from reconstruction context. */
435 static void reconstructionOptionsFromContext(libmv_ReconstructionOptions *reconstruction_options,
436                                              MovieReconstructContext *context)
437 {
438         reconstruction_options->select_keyframes = context->select_keyframes;
439
440         reconstruction_options->keyframe1 = context->keyframe1;
441         reconstruction_options->keyframe2 = context->keyframe2;
442
443         reconstruction_options->refine_intrinsics = context->refine_flags;
444 }
445
446 /* Solve camera/object motion and reconstruct 3D markers position
447  * from a prepared reconstruction context.
448  *
449  * stop is not actually used at this moment, so reconstruction
450  * job could not be stopped.
451  *
452  * do_update, progress and stat_message are set by reconstruction
453  * callback in libmv side and passing to an interface.
454  */
455 void BKE_tracking_reconstruction_solve(MovieReconstructContext *context, short *stop, short *do_update,
456                                        float *progress, char *stats_message, int message_size)
457 {
458         float error;
459
460         ReconstructProgressData progressdata;
461
462         libmv_ReconstructionOptions reconstruction_options;
463
464         progressdata.stop = stop;
465         progressdata.do_update = do_update;
466         progressdata.progress = progress;
467         progressdata.stats_message = stats_message;
468         progressdata.message_size = message_size;
469
470         reconstructionOptionsFromContext(&reconstruction_options, context);
471
472         if (context->motion_flag & TRACKING_MOTION_MODAL) {
473                 context->reconstruction = libmv_solveModal(context->tracks,
474                                                            &context->camera_intrinsics_options,
475                                                            &reconstruction_options,
476                                                            reconstruct_update_solve_cb, &progressdata);
477         }
478         else {
479                 context->reconstruction = libmv_solveReconstruction(context->tracks,
480                                                                     &context->camera_intrinsics_options,
481                                                                     &reconstruction_options,
482                                                                     reconstruct_update_solve_cb, &progressdata);
483
484                 if (context->select_keyframes) {
485                         /* store actual keyframes used for reconstruction to update them in the interface later */
486                         context->keyframe1 = reconstruction_options.keyframe1;
487                         context->keyframe2 = reconstruction_options.keyframe2;
488                 }
489         }
490
491         error = libmv_reprojectionError(context->reconstruction);
492
493         context->reprojection_error = error;
494 }
495
496 /* Finish reconstruction process by copying reconstructed data
497  * to an actual movie clip datablock.
498  */
499 bool BKE_tracking_reconstruction_finish(MovieReconstructContext *context, MovieTracking *tracking)
500 {
501         MovieTrackingReconstruction *reconstruction;
502         MovieTrackingObject *object;
503
504         if (!libmv_reconstructionIsValid(context->reconstruction)) {
505                 printf("Failed solve the motion: most likely there are no good keyframes\n");
506                 return false;
507         }
508
509         tracks_map_merge(context->tracks_map, tracking);
510         BKE_tracking_dopesheet_tag_update(tracking);
511
512         object = BKE_tracking_object_get_named(tracking, context->object_name);
513
514         if (context->is_camera)
515                 reconstruction = &tracking->reconstruction;
516         else
517                 reconstruction = &object->reconstruction;
518
519         /* update keyframe in the interface */
520         if (context->select_keyframes) {
521                 object->keyframe1 = context->keyframe1;
522                 object->keyframe2 = context->keyframe2;
523         }
524
525         reconstruction->error = context->reprojection_error;
526         reconstruction->flag |= TRACKING_RECONSTRUCTED;
527
528         if (!reconstruct_retrieve_libmv(context, tracking))
529                 return false;
530
531         return true;
532 }
533
534 static void tracking_scale_reconstruction(ListBase *tracksbase, MovieTrackingReconstruction *reconstruction,
535                                           const float scale[3])
536 {
537         MovieTrackingTrack *track;
538         int i;
539         float first_camera_delta[3] = {0.0f, 0.0f, 0.0f};
540
541         if (reconstruction->camnr > 0) {
542                 mul_v3_v3v3(first_camera_delta, reconstruction->cameras[0].mat[3], scale);
543         }
544
545         for (i = 0; i < reconstruction->camnr; i++) {
546                 MovieReconstructedCamera *camera = &reconstruction->cameras[i];
547                 mul_v3_v3(camera->mat[3], scale);
548                 sub_v3_v3(camera->mat[3], first_camera_delta);
549         }
550
551         for (track = tracksbase->first; track; track = track->next) {
552                 if (track->flag & TRACK_HAS_BUNDLE) {
553                         mul_v3_v3(track->bundle_pos, scale);
554                         sub_v3_v3(track->bundle_pos, first_camera_delta);
555                 }
556         }
557 }
558
559 /* Apply scale on all reconstructed cameras and bundles,
560  * used by camera scale apply operator.
561  */
562 void BKE_tracking_reconstruction_scale(MovieTracking *tracking, float scale[3])
563 {
564         MovieTrackingObject *object;
565
566         for (object = tracking->objects.first; object; object = object->next) {
567                 ListBase *tracksbase;
568                 MovieTrackingReconstruction *reconstruction;
569
570                 tracksbase = BKE_tracking_object_get_tracks(tracking, object);
571                 reconstruction = BKE_tracking_object_get_reconstruction(tracking, object);
572
573                 tracking_scale_reconstruction(tracksbase, reconstruction, scale);
574         }
575 }