Merge branch 'blender2.7'
[blender.git] / source / blender / blenkernel / intern / tracking_solver.c
1 /*
2  * This program is free software; you can redistribute it and/or
3  * modify it under the terms of the GNU General Public License
4  * as published by the Free Software Foundation; either version 2
5  * of the License, or (at your option) any later version.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * along with this program; if not, write to the Free Software Foundation,
14  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15  *
16  * The Original Code is Copyright (C) 2011 Blender Foundation.
17  * All rights reserved.
18  */
19
20 /** \file \ingroup bke
21  *
22  * This file contains blender-side implementation of camera solver.
23  */
24
25 #include <limits.h>
26
27 #include "MEM_guardedalloc.h"
28
29 #include "DNA_anim_types.h"
30 #include "DNA_movieclip_types.h"
31
32 #include "BLI_utildefines.h"
33 #include "BLI_math.h"
34 #include "BLI_listbase.h"
35 #include "BLI_string.h"
36
37 #include "BLT_translation.h"
38
39 #include "BKE_fcurve.h"
40 #include "BKE_tracking.h"
41 #include "BKE_movieclip.h"
42
43 #include "RNA_access.h"
44
45 #include "libmv-capi.h"
46 #include "tracking_private.h"
47
48 typedef struct MovieReconstructContext {
49         struct libmv_Tracks *tracks;
50         bool select_keyframes;
51         int keyframe1, keyframe2;
52         int refine_flags;
53
54         struct libmv_Reconstruction *reconstruction;
55
56         char object_name[MAX_NAME];
57         bool is_camera;
58         short motion_flag;
59
60         libmv_CameraIntrinsicsOptions camera_intrinsics_options;
61
62         float reprojection_error;
63
64         TracksMap *tracks_map;
65
66         int sfra, efra;
67 } MovieReconstructContext;
68
69 typedef struct ReconstructProgressData {
70         short *stop;
71         short *do_update;
72         float *progress;
73         char *stats_message;
74         int message_size;
75 } ReconstructProgressData;
76
77 /* Create new libmv Tracks structure from blender's tracks list. */
78 static struct libmv_Tracks *libmv_tracks_new(MovieClip *clip, ListBase *tracksbase, int width, int height)
79 {
80         int tracknr = 0;
81         MovieTrackingTrack *track;
82         struct libmv_Tracks *tracks = libmv_tracksNew();
83
84         track = tracksbase->first;
85         while (track) {
86                 FCurve *weight_fcurve;
87                 int a = 0;
88
89                 weight_fcurve = id_data_find_fcurve(&clip->id, track, &RNA_MovieTrackingTrack,
90                                                     "weight", 0, NULL);
91
92                 for (a = 0; a < track->markersnr; a++) {
93                         MovieTrackingMarker *marker = &track->markers[a];
94
95                         if ((marker->flag & MARKER_DISABLED) == 0) {
96                                 float weight = track->weight;
97
98                                 if (weight_fcurve) {
99                                         int scene_framenr =
100                                                 BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
101                                         weight = evaluate_fcurve(weight_fcurve, scene_framenr);
102                                 }
103
104                                 libmv_tracksInsert(tracks, marker->framenr, tracknr,
105                                                    (marker->pos[0] + track->offset[0]) * width,
106                                                    (marker->pos[1] + track->offset[1]) * height,
107                                                    weight);
108                         }
109                 }
110
111                 track = track->next;
112                 tracknr++;
113         }
114
115         return tracks;
116 }
117
118 /* Retrieve refined camera intrinsics from libmv to blender. */
119 static void reconstruct_retrieve_libmv_intrinsics(MovieReconstructContext *context, MovieTracking *tracking)
120 {
121         struct libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
122         struct libmv_CameraIntrinsics *libmv_intrinsics = libmv_reconstructionExtractIntrinsics(libmv_reconstruction);
123
124         libmv_CameraIntrinsicsOptions camera_intrinsics_options;
125         libmv_cameraIntrinsicsExtractOptions(libmv_intrinsics, &camera_intrinsics_options);
126
127         tracking_trackingCameraFromIntrinscisOptions(tracking,
128                                                      &camera_intrinsics_options);
129 }
130
131 /* Retrieve reconstructed tracks from libmv to blender.
132  * Actually, this also copies reconstructed cameras
133  * from libmv to movie clip datablock.
134  */
135 static bool reconstruct_retrieve_libmv_tracks(MovieReconstructContext *context, MovieTracking *tracking)
136 {
137         struct libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
138         MovieTrackingReconstruction *reconstruction = NULL;
139         MovieReconstructedCamera *reconstructed;
140         MovieTrackingTrack *track;
141         ListBase *tracksbase =  NULL;
142         int tracknr = 0, a;
143         bool ok = true;
144         bool origin_set = false;
145         int sfra = context->sfra, efra = context->efra;
146         float imat[4][4];
147
148         if (context->is_camera) {
149                 tracksbase = &tracking->tracks;
150                 reconstruction = &tracking->reconstruction;
151         }
152         else {
153                 MovieTrackingObject *object = BKE_tracking_object_get_named(tracking, context->object_name);
154
155                 tracksbase = &object->tracks;
156                 reconstruction = &object->reconstruction;
157         }
158
159         unit_m4(imat);
160
161         track = tracksbase->first;
162         while (track) {
163                 double pos[3];
164
165                 if (libmv_reprojectionPointForTrack(libmv_reconstruction, tracknr, pos)) {
166                         track->bundle_pos[0] = pos[0];
167                         track->bundle_pos[1] = pos[1];
168                         track->bundle_pos[2] = pos[2];
169
170                         track->flag |= TRACK_HAS_BUNDLE;
171                         track->error = libmv_reprojectionErrorForTrack(libmv_reconstruction, tracknr);
172                 }
173                 else {
174                         track->flag &= ~TRACK_HAS_BUNDLE;
175                         ok = false;
176
177                         printf("Unable to reconstruct position for track #%d '%s'\n", tracknr, track->name);
178                 }
179
180                 track = track->next;
181                 tracknr++;
182         }
183
184         if (reconstruction->cameras)
185                 MEM_freeN(reconstruction->cameras);
186
187         reconstruction->camnr = 0;
188         reconstruction->cameras = NULL;
189         reconstructed = MEM_callocN((efra - sfra + 1) * sizeof(MovieReconstructedCamera),
190                                     "temp reconstructed camera");
191
192         for (a = sfra; a <= efra; a++) {
193                 double matd[4][4];
194
195                 if (libmv_reprojectionCameraForImage(libmv_reconstruction, a, matd)) {
196                         int i, j;
197                         float mat[4][4];
198                         float error = libmv_reprojectionErrorForImage(libmv_reconstruction, a);
199
200                         for (i = 0; i < 4; i++) {
201                                 for (j = 0; j < 4; j++)
202                                         mat[i][j] = matd[i][j];
203                         }
204
205                         /* Ensure first camera has got zero rotation and transform.
206                          * This is essential for object tracking to work -- this way
207                          * we'll always know object and environment are properly
208                          * oriented.
209                          *
210                          * There's one weak part tho, which is requirement object
211                          * motion starts at the same frame as camera motion does,
212                          * otherwise that;' be a russian roulette whether object is
213                          * aligned correct or not.
214                          */
215                         if (!origin_set) {
216                                 invert_m4_m4(imat, mat);
217                                 unit_m4(mat);
218                                 origin_set = true;
219                         }
220                         else {
221                                 mul_m4_m4m4(mat, imat, mat);
222                         }
223
224                         copy_m4_m4(reconstructed[reconstruction->camnr].mat, mat);
225                         reconstructed[reconstruction->camnr].framenr = a;
226                         reconstructed[reconstruction->camnr].error = error;
227                         reconstruction->camnr++;
228                 }
229                 else {
230                         ok = false;
231                         printf("No camera for frame %d\n", a);
232                 }
233         }
234
235         if (reconstruction->camnr) {
236                 int size = reconstruction->camnr * sizeof(MovieReconstructedCamera);
237                 reconstruction->cameras = MEM_callocN(size, "reconstructed camera");
238                 memcpy(reconstruction->cameras, reconstructed, size);
239         }
240
241         if (origin_set) {
242                 track = tracksbase->first;
243                 while (track) {
244                         if (track->flag & TRACK_HAS_BUNDLE)
245                                 mul_v3_m4v3(track->bundle_pos, imat, track->bundle_pos);
246
247                         track = track->next;
248                 }
249         }
250
251         MEM_freeN(reconstructed);
252
253         return ok;
254 }
255
256 /* Retrieve all the libmv data from context to blender's side data blocks. */
257 static int reconstruct_retrieve_libmv(MovieReconstructContext *context, MovieTracking *tracking)
258 {
259         /* take the intrinsics back from libmv */
260         reconstruct_retrieve_libmv_intrinsics(context, tracking);
261
262         return reconstruct_retrieve_libmv_tracks(context, tracking);
263 }
264
265 /* Convert blender's refinement flags to libmv's. */
266 static int reconstruct_refine_intrinsics_get_flags(MovieTracking *tracking, MovieTrackingObject *object)
267 {
268         int refine = tracking->settings.refine_camera_intrinsics;
269         int flags = 0;
270
271         if ((object->flag & TRACKING_OBJECT_CAMERA) == 0)
272                 return 0;
273
274         if (refine & REFINE_FOCAL_LENGTH)
275                 flags |= LIBMV_REFINE_FOCAL_LENGTH;
276
277         if (refine & REFINE_PRINCIPAL_POINT)
278                 flags |= LIBMV_REFINE_PRINCIPAL_POINT;
279
280         if (refine & REFINE_RADIAL_DISTORTION_K1)
281                 flags |= LIBMV_REFINE_RADIAL_DISTORTION_K1;
282
283         if (refine & REFINE_RADIAL_DISTORTION_K2)
284                 flags |= LIBMV_REFINE_RADIAL_DISTORTION_K2;
285
286         return flags;
287 }
288
289 /* Count tracks which has markers at both of keyframes. */
290 static int reconstruct_count_tracks_on_both_keyframes(MovieTracking *tracking, MovieTrackingObject *object)
291 {
292         ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
293         int tot = 0;
294         int frame1 = object->keyframe1, frame2 = object->keyframe2;
295         MovieTrackingTrack *track;
296
297         track = tracksbase->first;
298         while (track) {
299                 if (BKE_tracking_track_has_enabled_marker_at_frame(track, frame1)) {
300                         if (BKE_tracking_track_has_enabled_marker_at_frame(track, frame2)) {
301                                 tot++;
302                         }
303                 }
304
305                 track = track->next;
306         }
307
308         return tot;
309 }
310
311 /* Perform early check on whether everything is fine to start reconstruction. */
312 bool BKE_tracking_reconstruction_check(MovieTracking *tracking, MovieTrackingObject *object,
313                                        char *error_msg, int error_size)
314 {
315         if (tracking->settings.motion_flag & TRACKING_MOTION_MODAL) {
316                 /* TODO: check for number of tracks? */
317                 return true;
318         }
319         else if ((tracking->settings.reconstruction_flag & TRACKING_USE_KEYFRAME_SELECTION) == 0) {
320                 /* automatic keyframe selection does not require any pre-process checks */
321                 if (reconstruct_count_tracks_on_both_keyframes(tracking, object) < 8) {
322                         BLI_strncpy(error_msg,
323                                     N_("At least 8 common tracks on both keyframes are needed for reconstruction"),
324                                     error_size);
325
326                         return false;
327                 }
328         }
329
330 #ifndef WITH_LIBMV
331         BLI_strncpy(error_msg, N_("Blender is compiled without motion tracking library"), error_size);
332         return false;
333 #endif
334
335         return true;
336 }
337
338 /* Create context for camera/object motion reconstruction.
339  * Copies all data needed for reconstruction from movie
340  * clip datablock, so editing this clip is safe during
341  * reconstruction job is in progress.
342  */
343 MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieClip *clip, MovieTrackingObject *object,
344                                                                  int keyframe1, int keyframe2, int width, int height)
345 {
346         MovieTracking *tracking = &clip->tracking;
347         MovieReconstructContext *context = MEM_callocN(sizeof(MovieReconstructContext), "MovieReconstructContext data");
348         ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
349         float aspy = 1.0f / tracking->camera.pixel_aspect;
350         int num_tracks = BLI_listbase_count(tracksbase);
351         int sfra = INT_MAX, efra = INT_MIN;
352         MovieTrackingTrack *track;
353
354         BLI_strncpy(context->object_name, object->name, sizeof(context->object_name));
355         context->is_camera = object->flag & TRACKING_OBJECT_CAMERA;
356         context->motion_flag = tracking->settings.motion_flag;
357
358         context->select_keyframes =
359                 (tracking->settings.reconstruction_flag & TRACKING_USE_KEYFRAME_SELECTION) != 0;
360
361         tracking_cameraIntrinscisOptionsFromTracking(tracking,
362                                                      width, height,
363                                                      &context->camera_intrinsics_options);
364
365         context->tracks_map = tracks_map_new(context->object_name, context->is_camera, num_tracks, 0);
366
367         track = tracksbase->first;
368         while (track) {
369                 int first = 0, last = track->markersnr - 1;
370                 MovieTrackingMarker *first_marker = &track->markers[0];
371                 MovieTrackingMarker *last_marker = &track->markers[track->markersnr - 1];
372
373                 /* find first not-disabled marker */
374                 while (first <= track->markersnr - 1 && first_marker->flag & MARKER_DISABLED) {
375                         first++;
376                         first_marker++;
377                 }
378
379                 /* find last not-disabled marker */
380                 while (last >= 0 && last_marker->flag & MARKER_DISABLED) {
381                         last--;
382                         last_marker--;
383                 }
384
385                 if (first <= track->markersnr - 1)
386                         sfra = min_ii(sfra, first_marker->framenr);
387
388                 if (last >= 0)
389                         efra = max_ii(efra, last_marker->framenr);
390
391                 tracks_map_insert(context->tracks_map, track, NULL);
392
393                 track = track->next;
394         }
395
396         context->sfra = sfra;
397         context->efra = efra;
398
399         context->tracks = libmv_tracks_new(clip, tracksbase, width, height * aspy);
400         context->keyframe1 = keyframe1;
401         context->keyframe2 = keyframe2;
402         context->refine_flags = reconstruct_refine_intrinsics_get_flags(tracking, object);
403
404         return context;
405 }
406
407 /* Free memory used by a reconstruction process. */
408 void BKE_tracking_reconstruction_context_free(MovieReconstructContext *context)
409 {
410         if (context->reconstruction)
411                 libmv_reconstructionDestroy(context->reconstruction);
412
413         libmv_tracksDestroy(context->tracks);
414
415         tracks_map_free(context->tracks_map, NULL);
416
417         MEM_freeN(context);
418 }
419
420 /* Callback which is called from libmv side to update progress in the interface. */
421 static void reconstruct_update_solve_cb(void *customdata, double progress, const char *message)
422 {
423         ReconstructProgressData *progressdata = customdata;
424
425         if (progressdata->progress) {
426                 *progressdata->progress = progress;
427                 *progressdata->do_update = true;
428         }
429
430         BLI_snprintf(progressdata->stats_message, progressdata->message_size, "Solving camera | %s", message);
431 }
432
433 /* Fill in reconstruction options structure from reconstruction context. */
434 static void reconstructionOptionsFromContext(libmv_ReconstructionOptions *reconstruction_options,
435                                              MovieReconstructContext *context)
436 {
437         reconstruction_options->select_keyframes = context->select_keyframes;
438
439         reconstruction_options->keyframe1 = context->keyframe1;
440         reconstruction_options->keyframe2 = context->keyframe2;
441
442         reconstruction_options->refine_intrinsics = context->refine_flags;
443 }
444
445 /* Solve camera/object motion and reconstruct 3D markers position
446  * from a prepared reconstruction context.
447  *
448  * stop is not actually used at this moment, so reconstruction
449  * job could not be stopped.
450  *
451  * do_update, progress and stat_message are set by reconstruction
452  * callback in libmv side and passing to an interface.
453  */
454 void BKE_tracking_reconstruction_solve(MovieReconstructContext *context, short *stop, short *do_update,
455                                        float *progress, char *stats_message, int message_size)
456 {
457         float error;
458
459         ReconstructProgressData progressdata;
460
461         libmv_ReconstructionOptions reconstruction_options;
462
463         progressdata.stop = stop;
464         progressdata.do_update = do_update;
465         progressdata.progress = progress;
466         progressdata.stats_message = stats_message;
467         progressdata.message_size = message_size;
468
469         reconstructionOptionsFromContext(&reconstruction_options, context);
470
471         if (context->motion_flag & TRACKING_MOTION_MODAL) {
472                 context->reconstruction = libmv_solveModal(context->tracks,
473                                                            &context->camera_intrinsics_options,
474                                                            &reconstruction_options,
475                                                            reconstruct_update_solve_cb, &progressdata);
476         }
477         else {
478                 context->reconstruction = libmv_solveReconstruction(context->tracks,
479                                                                     &context->camera_intrinsics_options,
480                                                                     &reconstruction_options,
481                                                                     reconstruct_update_solve_cb, &progressdata);
482
483                 if (context->select_keyframes) {
484                         /* store actual keyframes used for reconstruction to update them in the interface later */
485                         context->keyframe1 = reconstruction_options.keyframe1;
486                         context->keyframe2 = reconstruction_options.keyframe2;
487                 }
488         }
489
490         error = libmv_reprojectionError(context->reconstruction);
491
492         context->reprojection_error = error;
493 }
494
495 /* Finish reconstruction process by copying reconstructed data
496  * to an actual movie clip datablock.
497  */
498 bool BKE_tracking_reconstruction_finish(MovieReconstructContext *context, MovieTracking *tracking)
499 {
500         MovieTrackingReconstruction *reconstruction;
501         MovieTrackingObject *object;
502
503         if (!libmv_reconstructionIsValid(context->reconstruction)) {
504                 printf("Failed solve the motion: most likely there are no good keyframes\n");
505                 return false;
506         }
507
508         tracks_map_merge(context->tracks_map, tracking);
509         BKE_tracking_dopesheet_tag_update(tracking);
510
511         object = BKE_tracking_object_get_named(tracking, context->object_name);
512
513         if (context->is_camera)
514                 reconstruction = &tracking->reconstruction;
515         else
516                 reconstruction = &object->reconstruction;
517
518         /* update keyframe in the interface */
519         if (context->select_keyframes) {
520                 object->keyframe1 = context->keyframe1;
521                 object->keyframe2 = context->keyframe2;
522         }
523
524         reconstruction->error = context->reprojection_error;
525         reconstruction->flag |= TRACKING_RECONSTRUCTED;
526
527         if (!reconstruct_retrieve_libmv(context, tracking))
528                 return false;
529
530         return true;
531 }
532
533 static void tracking_scale_reconstruction(ListBase *tracksbase, MovieTrackingReconstruction *reconstruction,
534                                           const float scale[3])
535 {
536         MovieTrackingTrack *track;
537         int i;
538         float first_camera_delta[3] = {0.0f, 0.0f, 0.0f};
539
540         if (reconstruction->camnr > 0) {
541                 mul_v3_v3v3(first_camera_delta, reconstruction->cameras[0].mat[3], scale);
542         }
543
544         for (i = 0; i < reconstruction->camnr; i++) {
545                 MovieReconstructedCamera *camera = &reconstruction->cameras[i];
546                 mul_v3_v3(camera->mat[3], scale);
547                 sub_v3_v3(camera->mat[3], first_camera_delta);
548         }
549
550         for (track = tracksbase->first; track; track = track->next) {
551                 if (track->flag & TRACK_HAS_BUNDLE) {
552                         mul_v3_v3(track->bundle_pos, scale);
553                         sub_v3_v3(track->bundle_pos, first_camera_delta);
554                 }
555         }
556 }
557
558 /* Apply scale on all reconstructed cameras and bundles,
559  * used by camera scale apply operator.
560  */
561 void BKE_tracking_reconstruction_scale(MovieTracking *tracking, float scale[3])
562 {
563         MovieTrackingObject *object;
564
565         for (object = tracking->objects.first; object; object = object->next) {
566                 ListBase *tracksbase;
567                 MovieTrackingReconstruction *reconstruction;
568
569                 tracksbase = BKE_tracking_object_get_tracks(tracking, object);
570                 reconstruction = BKE_tracking_object_get_reconstruction(tracking, object);
571
572                 tracking_scale_reconstruction(tracksbase, reconstruction, scale);
573         }
574 }