Camera tracking: support of tripod motion solving
authorSergey Sharybin <sergey.vfx@gmail.com>
Sat, 28 Apr 2012 14:54:45 +0000 (14:54 +0000)
committerSergey Sharybin <sergey.vfx@gmail.com>
Sat, 28 Apr 2012 14:54:45 +0000 (14:54 +0000)
Expose option into interface to use modal solver which currently
supports only tripod motion.

This solver requires two tracks at least to reconstruct motion.
Using more tracks aren't improving solution in general, just adds
instability into solution and slows down things a lot.

Refirement of camera intrinsics was disabled due to it's not only
refines camera intrinsics but also adjusts camera position which
isn't necessary here

To use this solver just activate "Tripod Motion" checkbox in
solver panel.

Merged from tomato: svn merge ^/branches/soc-2011-tomato -r45622:45624 -r46036:46037

P.S. Quite experimental yet, requires more checking and probably
tweaks to prevent camera jumps when tracks apperars/disappears
from the screen.

1  2 
extern/libmv/libmv-capi.cpp
extern/libmv/libmv-capi.h
extern/libmv/libmv/simple_pipeline/modal_solver.cc
release/scripts/startup/bl_ui/space_clip.py
source/blender/blenkernel/intern/tracking.c
source/blender/makesrna/intern/rna_tracking.c

@@@ -423,26 -449,50 +449,45 @@@ libmv_Reconstruction *libmv_solveRecons
        libmv::EuclideanCompleteReconstruction(normalized_tracks, reconstruction, &update_callback);
  
        if (refine_intrinsics) {
-               /* only a few combinations are supported but trust the caller */
-               int libmv_refine_flags = 0;
-               if (refine_intrinsics & LIBMV_REFINE_FOCAL_LENGTH) {
-                       libmv_refine_flags |= libmv::BUNDLE_FOCAL_LENGTH;
-               }
-               if (refine_intrinsics & LIBMV_REFINE_PRINCIPAL_POINT) {
-                       libmv_refine_flags |= libmv::BUNDLE_PRINCIPAL_POINT;
-               }
-               if (refine_intrinsics & LIBMV_REFINE_RADIAL_DISTORTION_K1) {
-                       libmv_refine_flags |= libmv::BUNDLE_RADIAL_K1;
-               }
-               if (refine_intrinsics & LIBMV_REFINE_RADIAL_DISTORTION_K2) {
-                       libmv_refine_flags |= libmv::BUNDLE_RADIAL_K2;
-               }
+               libmv_solveRefineIntrinsics((libmv::Tracks *)tracks, intrinsics, reconstruction,
+                       refine_intrinsics, progress_update_callback, callback_customdata);
+       }
  
-               progress_update_callback(callback_customdata, 1.0, "Refining solution");
-               libmv::EuclideanBundleCommonIntrinsics(*(libmv::Tracks *)tracks, libmv_refine_flags,
-                       reconstruction, intrinsics);
+       progress_update_callback(callback_customdata, 1.0, "Finishing solution");
+       libmv_reconstruction->tracks = *(libmv::Tracks *)tracks;
+       libmv_reconstruction->error = libmv::EuclideanReprojectionError(*(libmv::Tracks *)tracks, *reconstruction, *intrinsics);
+       return (libmv_Reconstruction *)libmv_reconstruction;
+ }
 -struct libmv_Reconstruction *libmv_solveModal(struct libmv_Tracks *tracks, int refine_intrinsics, double focal_length,
++struct libmv_Reconstruction *libmv_solveModal(struct libmv_Tracks *tracks, double focal_length,
+                       double principal_x, double principal_y, double k1, double k2, double k3,
+                       reconstruct_progress_update_cb progress_update_callback, void *callback_customdata)
+ {
+       /* Invert the camera intrinsics. */
+       libmv::vector<libmv::Marker> markers = ((libmv::Tracks*)tracks)->AllMarkers();
+       libmv_Reconstruction *libmv_reconstruction = new libmv_Reconstruction();
+       libmv::EuclideanReconstruction *reconstruction = &libmv_reconstruction->reconstruction;
+       libmv::CameraIntrinsics *intrinsics = &libmv_reconstruction->intrinsics;
+       ReconstructUpdateCallback update_callback =
+               ReconstructUpdateCallback(progress_update_callback, callback_customdata);
+       intrinsics->SetFocalLength(focal_length, focal_length);
+       intrinsics->SetPrincipalPoint(principal_x, principal_y);
+       intrinsics->SetRadialDistortion(k1, k2, k3);
+       for (int i = 0; i < markers.size(); ++i) {
+               intrinsics->InvertIntrinsics(markers[i].x,
+                       markers[i].y,
+                       &(markers[i].x),
+                       &(markers[i].y));
        }
  
 -      if (refine_intrinsics) {
 -              libmv_solveRefineIntrinsics((libmv::Tracks *)tracks, intrinsics, reconstruction,
 -                      refine_intrinsics, progress_update_callback, callback_customdata);
 -      }
 -
+       libmv::Tracks normalized_tracks(markers);
+       libmv::ModalSolver(normalized_tracks, reconstruction, &update_callback);
        progress_update_callback(callback_customdata, 1.0, "Finishing solution");
        libmv_reconstruction->tracks = *(libmv::Tracks *)tracks;
        libmv_reconstruction->error = libmv::EuclideanReprojectionError(*(libmv::Tracks *)tracks, *reconstruction, *intrinsics);
@@@ -68,6 -68,9 +68,9 @@@ int libmv_refineParametersAreValid(int 
  struct libmv_Reconstruction *libmv_solveReconstruction(struct libmv_Tracks *tracks, int keyframe1, int keyframe2,
                        int refine_intrinsics, double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
                        reconstruct_progress_update_cb progress_update_callback, void *callback_customdata);
 -struct libmv_Reconstruction *libmv_solveModal(struct libmv_Tracks *tracks, int refine_intrinsics, double focal_length,
++struct libmv_Reconstruction *libmv_solveModal(struct libmv_Tracks *tracks, double focal_length,
+                       double principal_x, double principal_y, double k1, double k2, double k3,
+                       reconstruct_progress_update_cb progress_update_callback, void *callback_customdata);
  int libmv_reporojectionPointForTrack(struct libmv_Reconstruction *libmv_reconstruction, int track, double pos[3]);
  double libmv_reporojectionErrorForTrack(struct libmv_Reconstruction *libmv_reconstruction, int track);
  double libmv_reporojectionErrorForImage(struct libmv_Reconstruction *libmv_reconstruction, int image);
index 0000000,ede0071..bb49b30
mode 000000,100644..100644
--- /dev/null
@@@ -1,0 -1,130 +1,126 @@@
 -          points.push_back(R * point->X);
+ // Copyright (c) 2012 libmv authors.
+ //
+ // Permission is hereby granted, free of charge, to any person obtaining a copy
+ // of this software and associated documentation files (the "Software"), to
+ // deal in the Software without restriction, including without limitation the
+ // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ // sell copies of the Software, and to permit persons to whom the Software is
+ // furnished to do so, subject to the following conditions:
+ //
+ // The above copyright notice and this permission notice shall be included in
+ // all copies or substantial portions of the Software.
+ //
+ // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ // IN THE SOFTWARE.
+ #include <cstdio>
+ #include "libmv/logging/logging.h"
+ #include "libmv/simple_pipeline/modal_solver.h"
+ #include "libmv/simple_pipeline/rigid_registration.h"
+ #ifdef _MSC_VER
+ #  define snprintf _snprintf
+ #endif
+ namespace libmv {
+ static void ProjectMarkerOnSphere(Marker &marker, Vec3 &X) {
+   X(0) = marker.x;
+   X(1) = marker.y;
+   X(2) = 1.0;
+   X *= 5.0 / X.norm();
+ }
+ static void ModalSolverLogProress(ProgressUpdateCallback *update_callback,
+     double progress)
+ {
+   if (update_callback) {
+     char message[256];
+     snprintf(message, sizeof(message), "Solving progress %d%%", (int)(progress * 100));
+     update_callback->invoke(progress, message);
+   }
+ }
+ void ModalSolver(Tracks &tracks,
+                  EuclideanReconstruction *reconstruction,
+                  ProgressUpdateCallback *update_callback) {
+   int max_image = tracks.MaxImage();
+   int max_track = tracks.MaxTrack();
+   LG << "Max image: " << max_image;
+   LG << "Max track: " << max_track;
+   Mat3 R = Mat3::Identity();
+   for (int image = 0; image <= max_image; ++image) {
+     vector<Marker> all_markers = tracks.MarkersInImage(image);
+     ModalSolverLogProress(update_callback, (float) image / max_image);
+     // Skip empty frames without doing anything
+     if (all_markers.size() == 0) {
+       LG << "Skipping frame: " << image;
+       continue;
+     }
+     vector<Vec3> points, reference_points;
+     // Cnstruct pairs of markers from current and previous image,
+     // to reproject them and find rigid transformation between
+     // previous and current image
+     for (int track = 0; track <= max_track; ++track) {
+       EuclideanPoint *point = reconstruction->PointForTrack(track);
+       if (point) {
+         Marker marker = tracks.MarkerInImageForTrack(image, track);
+         if (marker.image == image) {
+           Vec3 X;
+           LG << "Use track " << track << " for rigid registration between image " <<
+             image - 1 << " and " << image;
+           ProjectMarkerOnSphere(marker, X);
 -      Mat3 dR = Mat3::Identity();
 -
 -      // Find rigid delta transformation from previous image to current image
 -      RigidRegistration(reference_points, points, dR);
 -
 -      R *= dR;
++          points.push_back(point->X);
+           reference_points.push_back(X);
+         }
+       }
+     }
+     if (points.size()) {
++      // Find rigid delta transformation to current image
++      RigidRegistration(reference_points, points, R);
+     }
+     reconstruction->InsertCamera(image, R, Vec3::Zero());
+     // Review if there's new tracks for which position might be reconstructed
+     for (int track = 0; track <= max_track; ++track) {
+       if (!reconstruction->PointForTrack(track)) {
+         Marker marker = tracks.MarkerInImageForTrack(image, track);
+         if (marker.image == image) {
+           // New track appeared on this image, project it's position onto sphere
+           LG << "Projecting track " << track << " at image " << image;
+           Vec3 X;
+           ProjectMarkerOnSphere(marker, X);
+           reconstruction->InsertPoint(track, R.inverse() * X);
+         }
+       }
+     }
+   }
+ }
+ }  // namespace libmv
@@@ -232,7 -236,7 +236,7 @@@ class CLIP_PT_tools_solve(Panel)
          col.prop(settings, "keyframe_b")
  
          col = layout.column(align=True)
--        col.active = tracking_object.is_camera
++        col.active = tracking_object.is_camera and not settings.use_tripod_solver
          col.label(text="Refine:")
          col.prop(settings, "refine_intrinsics", text="")
  
@@@ -1894,13 -1900,23 +1900,22 @@@ void BKE_tracking_solve_reconstruction(
        progressdata.stats_message = stats_message;
        progressdata.message_size = message_size;
  
-       context->reconstruction = libmv_solveReconstruction(context->tracks,
-               context->keyframe1, context->keyframe2,
-               context->refine_flags,
-               context->focal_length,
-               context->principal_point[0], context->principal_point[1],
-               context->k1, context->k2, context->k3,
-               solve_reconstruction_update_cb, &progressdata);
+       if (context->motion_flag & TRACKING_MOTION_MODAL) {
+               context->reconstruction = libmv_solveModal(context->tracks,
 -                      context->refine_flags,
+                       context->focal_length,
+                       context->principal_point[0], context->principal_point[1],
+                       context->k1, context->k2, context->k3,
+                       solve_reconstruction_update_cb, &progressdata);
+       }
+       else {
+               context->reconstruction = libmv_solveReconstruction(context->tracks,
+                       context->keyframe1, context->keyframe2,
+                       context->refine_flags,
+                       context->focal_length,
+                       context->principal_point[0], context->principal_point[1],
+                       context->k1, context->k2, context->k3,
+                       solve_reconstruction_update_cb, &progressdata);
+       }
  
        error = libmv_reprojectionError(context->reconstruction);