Camera Tracking: allow fallback to reprojection resection by user demand
authorSergey Sharybin <sergey.vfx@gmail.com>
Mon, 5 Nov 2012 08:04:27 +0000 (08:04 +0000)
committerSergey Sharybin <sergey.vfx@gmail.com>
Mon, 5 Nov 2012 08:04:27 +0000 (08:04 +0000)
This fixes some "regressions" introduced in rev50781 which lead to much
worse solution in some cases. Now it's possible to bring old behavior back.

Perhaps it's more like temporal solution for time being smarter solution is
found. But finding such a solution isn't so fast, so let's bring manual
control over reprojection usage.

But anyway, imo it's now nice to have a structure which could be used to
pass different settings to the solver.

14 files changed:
extern/libmv/libmv-capi.cpp
extern/libmv/libmv-capi.h
extern/libmv/libmv/multiview/euclidean_resection.cc
extern/libmv/libmv/multiview/euclidean_resection.h
extern/libmv/libmv/simple_pipeline/pipeline.cc
extern/libmv/libmv/simple_pipeline/pipeline.h
extern/libmv/libmv/simple_pipeline/reconstruction.h
extern/libmv/libmv/simple_pipeline/resect.cc
extern/libmv/libmv/simple_pipeline/resect.h
release/scripts/startup/bl_ui/space_clip.py
source/blender/blenkernel/intern/tracking.c
source/blender/blenloader/intern/readfile.c
source/blender/makesdna/DNA_tracking_types.h
source/blender/makesrna/intern/rna_tracking.c

index 0f73234..a15927f 100644 (file)
@@ -550,7 +550,8 @@ static void libmv_solveRefineIntrinsics(libmv::Tracks *tracks, libmv::CameraIntr
 }
 
 libmv_Reconstruction *libmv_solveReconstruction(libmv_Tracks *tracks, int keyframe1, int keyframe2,
-                       int refine_intrinsics, double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
+                       int refine_intrinsics, double focal_length, double principal_x, double principal_y,
+                       double k1, double k2, double k3, struct libmv_reconstructionOptions *options,
                        reconstruct_progress_update_cb progress_update_callback, void *callback_customdata)
 {
        /* Invert the camera intrinsics. */
@@ -558,6 +559,7 @@ libmv_Reconstruction *libmv_solveReconstruction(libmv_Tracks *tracks, int keyfra
        libmv_Reconstruction *libmv_reconstruction = new libmv_Reconstruction();
        libmv::EuclideanReconstruction *reconstruction = &libmv_reconstruction->reconstruction;
        libmv::CameraIntrinsics *intrinsics = &libmv_reconstruction->intrinsics;
+       libmv::ReconstructionOptions reconstruction_options;
 
        ReconstructUpdateCallback update_callback =
                ReconstructUpdateCallback(progress_update_callback, callback_customdata);
@@ -566,6 +568,9 @@ libmv_Reconstruction *libmv_solveReconstruction(libmv_Tracks *tracks, int keyfra
        intrinsics->SetPrincipalPoint(principal_x, principal_y);
        intrinsics->SetRadialDistortion(k1, k2, k3);
 
+       reconstruction_options.success_threshold = options->success_threshold;
+       reconstruction_options.use_fallback_reconstruction = options->use_fallback_reconstruction;
+
        for (int i = 0; i < markers.size(); ++i) {
                intrinsics->InvertIntrinsics(markers[i].x,
                        markers[i].y,
@@ -584,7 +589,8 @@ libmv_Reconstruction *libmv_solveReconstruction(libmv_Tracks *tracks, int keyfra
 
        libmv::EuclideanReconstructTwoFrames(keyframe_markers, reconstruction);
        libmv::EuclideanBundle(normalized_tracks, reconstruction);
-       libmv::EuclideanCompleteReconstruction(normalized_tracks, reconstruction, &update_callback);
+       libmv::EuclideanCompleteReconstruction(reconstruction_options, normalized_tracks,
+                                              reconstruction, &update_callback);
 
        if (refine_intrinsics) {
                libmv_solveRefineIntrinsics((libmv::Tracks *)tracks, intrinsics, reconstruction,
index 5253ba7..e5885e7 100644 (file)
@@ -91,13 +91,20 @@ void libmv_tracksDestroy(struct libmv_Tracks *libmv_tracks);
 #define LIBMV_REFINE_RADIAL_DISTORTION_K1 (1<<2)
 #define LIBMV_REFINE_RADIAL_DISTORTION_K2 (1<<4)
 
+/* TODO: make keyframes/distortion model a part of options? */
+struct libmv_reconstructionOptions {
+       double success_threshold;
+       int use_fallback_reconstruction;
+};
+
 typedef void (*reconstruct_progress_update_cb) (void *customdata, double progress, const char *message);
 
 int libmv_refineParametersAreValid(int parameters);
 
 struct libmv_Reconstruction *libmv_solveReconstruction(struct libmv_Tracks *tracks, int keyframe1, int keyframe2,
                        int refine_intrinsics, double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
-                       reconstruct_progress_update_cb progress_update_callback, void *callback_customdata);
+                       struct libmv_reconstructionOptions *options, reconstruct_progress_update_cb progress_update_callback,
+                       void *callback_customdata);
 struct libmv_Reconstruction *libmv_solveModal(struct libmv_Tracks *tracks, double focal_length,
                        double principal_x, double principal_y, double k1, double k2, double k3,
                        reconstruct_progress_update_cb progress_update_callback, void *callback_customdata);
index 6cad4ca..2605bf0 100644 (file)
@@ -37,13 +37,14 @@ typedef unsigned int uint;
 bool EuclideanResection(const Mat2X &x_camera, 
                         const Mat3X &X_world,
                         Mat3 *R, Vec3 *t,
-                        ResectionMethod method) {
+                        ResectionMethod method,
+                        double success_threshold) {
   switch (method) {
     case RESECTION_ANSAR_DANIILIDIS:
       EuclideanResectionAnsarDaniilidis(x_camera, X_world, R, t);
       break;
     case RESECTION_EPNP:
-      return EuclideanResectionEPnP(x_camera, X_world, R, t);      
+      return EuclideanResectionEPnP(x_camera, X_world, R, t, success_threshold);
       break;
     default:
       LOG(FATAL) << "Unknown resection method.";
@@ -435,8 +436,9 @@ static void ComputePointsCoordinatesInCameraFrame(
 }
 
 bool EuclideanResectionEPnP(const Mat2X &x_camera,
-                            const Mat3X &X_world, 
-                            Mat3 *R, Vec3 *t) {
+                            const Mat3X &X_world,
+                            Mat3 *R, Vec3 *t,
+                            double success_threshold) {
   CHECK(x_camera.cols() == X_world.cols());
   CHECK(x_camera.cols() > 3);
   size_t num_points = X_world.cols();
@@ -544,7 +546,12 @@ bool EuclideanResectionEPnP(const Mat2X &x_camera,
   //
   // TODO(keir): Decide if setting this to infinity, effectively disabling the
   // check, is the right approach. So far this seems the case.
-  double kSuccessThreshold = std::numeric_limits<double>::max();
+  //
+  // TODO(sergey): Made it an option for now, in some cases it makes sense to
+  // still fallback to reprojection solution (see bug [#32765] from Blender bug tracker)
+
+  // double kSuccessThreshold = std::numeric_limits<double>::max();
+  double kSuccessThreshold = success_threshold;
 
   // Find the first possible solution for R, t corresponding to:
   // Betas          = [b00 b01 b11 b02 b12 b22 b03 b13 b23 b33]
index f20513d..b0428ec 100644 (file)
@@ -45,11 +45,14 @@ enum ResectionMethod {
  * \param R         Solution for the camera rotation matrix
  * \param t         Solution for the camera translation vector
  * \param method    The resection method to use.
+ * \param success_threshold  Threshold of an error which is still considered a success
+ *                           (currently used by EPnP algorithm only)
  */
 bool EuclideanResection(const Mat2X &x_camera, 
                         const Mat3X &X_world,
                         Mat3 *R, Vec3 *t,
-                        ResectionMethod method = RESECTION_EPNP);
+                        ResectionMethod method = RESECTION_EPNP,
+                        double success_threshold = 1e-3);
 
 /**
  * Computes the extrinsic parameters, R and t for a calibrated camera
@@ -110,6 +113,7 @@ void EuclideanResectionAnsarDaniilidis(const Mat2X &x_camera,
  * \param X_world 3D points in the world coordinate system
  * \param R       Solution for the camera rotation matrix
  * \param t       Solution for the camera translation vector
+ * \param success_threshold  Threshold of an error which is still considered a success
  *
  * This is the algorithm described in:
  * "{EP$n$P: An Accurate $O(n)$ Solution to the P$n$P Problem", by V. Lepetit
@@ -118,7 +122,8 @@ void EuclideanResectionAnsarDaniilidis(const Mat2X &x_camera,
  */
 bool EuclideanResectionEPnP(const Mat2X &x_camera,
                             const Mat3X &X_world, 
-                            Mat3 *R, Vec3 *t);
+                            Mat3 *R, Vec3 *t,
+                            double success_threshold = 1e-3);
 
 } // namespace euclidean_resection
 } // namespace libmv
index 4773a70..efceda5 100644 (file)
@@ -50,9 +50,10 @@ struct EuclideanPipelineRoutines {
     EuclideanBundle(tracks, reconstruction);
   }
 
-  static bool Resect(const vector<Marker> &markers,
+  static bool Resect(const ReconstructionOptions &options,
+                     const vector<Marker> &markers,
                      EuclideanReconstruction *reconstruction, bool final_pass) {
-    return EuclideanResect(markers, reconstruction, final_pass);
+    return EuclideanResect(options, markers, reconstruction, final_pass);
   }
 
   static bool Intersect(const vector<Marker> &markers,
@@ -88,7 +89,8 @@ struct ProjectivePipelineRoutines {
     ProjectiveBundle(tracks, reconstruction);
   }
 
-  static bool Resect(const vector<Marker> &markers,
+  static bool Resect(const ReconstructionOptions &options,
+                     const vector<Marker> &markers,
                      ProjectiveReconstruction *reconstruction, bool final_pass) {
     return ProjectiveResect(markers, reconstruction);
   }
@@ -136,6 +138,7 @@ static void CompleteReconstructionLogProress(ProgressUpdateCallback *update_call
 
 template<typename PipelineRoutines>
 void InternalCompleteReconstruction(
+    const ReconstructionOptions &options,
     const Tracks &tracks,
     typename PipelineRoutines::Reconstruction *reconstruction,
     ProgressUpdateCallback *update_callback = NULL) {
@@ -204,7 +207,7 @@ void InternalCompleteReconstruction(
       if (reconstructed_markers.size() >= 5) {
         CompleteReconstructionLogProress(update_callback,
                                          (double)tot_resects/(max_image));
-        if (PipelineRoutines::Resect(reconstructed_markers, reconstruction, false)) {
+        if (PipelineRoutines::Resect(options, reconstructed_markers, reconstruction, false)) {
           num_resects++;
           tot_resects++;
           LG << "Ran Resect() for image " << image;
@@ -240,7 +243,7 @@ void InternalCompleteReconstruction(
     if (reconstructed_markers.size() >= 5) {
       CompleteReconstructionLogProress(update_callback,
                                        (double)tot_resects/(max_image));
-      if (PipelineRoutines::Resect(reconstructed_markers, reconstruction, true)) {
+      if (PipelineRoutines::Resect(options, reconstructed_markers, reconstruction, true)) {
         num_resects++;
         LG << "Ran final Resect() for image " << image;
       } else {
@@ -325,17 +328,21 @@ double ProjectiveReprojectionError(
                                                                intrinsics);
 }
 
-void EuclideanCompleteReconstruction(const Tracks &tracks,
+void EuclideanCompleteReconstruction(const ReconstructionOptions &options,
+                                     const Tracks &tracks,
                                      EuclideanReconstruction *reconstruction,
                                      ProgressUpdateCallback *update_callback) {
-  InternalCompleteReconstruction<EuclideanPipelineRoutines>(tracks,
+  InternalCompleteReconstruction<EuclideanPipelineRoutines>(options,
+                                                            tracks,
                                                             reconstruction,
                                                             update_callback);
 }
 
-void ProjectiveCompleteReconstruction(const Tracks &tracks,
+void ProjectiveCompleteReconstruction(const ReconstructionOptions &options,
+                                      const Tracks &tracks,
                                       ProjectiveReconstruction *reconstruction) {
-  InternalCompleteReconstruction<ProjectivePipelineRoutines>(tracks,
+  InternalCompleteReconstruction<ProjectivePipelineRoutines>(options,
+                                                             tracks,
                                                              reconstruction);
 }
 
index e940b57..11c1129 100644 (file)
@@ -39,6 +39,9 @@ namespace libmv {
     repeated until all points and cameras are estimated. Periodically, bundle
     adjustment is run to ensure a quality reconstruction.
 
+    \a options are used to define some specific befaviours based on settings
+    see documentation for ReconstructionOptions
+
     \a tracks should contain markers used in the reconstruction.
     \a reconstruction should contain at least some 3D points or some estimated
     cameras. The minimum number of cameras is two (with no 3D points) and the
@@ -46,7 +49,8 @@ namespace libmv {
 
     \sa EuclideanResect, EuclideanIntersect, EuclideanBundle
 */
-void EuclideanCompleteReconstruction(const Tracks &tracks,
+void EuclideanCompleteReconstruction(const ReconstructionOptions &options,
+                                     const Tracks &tracks,
                                      EuclideanReconstruction *reconstruction,
                                      ProgressUpdateCallback *update_callback = NULL);
 
@@ -63,6 +67,9 @@ void EuclideanCompleteReconstruction(const Tracks &tracks,
     repeated until all points and cameras are estimated. Periodically, bundle
     adjustment is run to ensure a quality reconstruction.
 
+    \a options are used to define some specific befaviours based on settings
+    see documentation for ReconstructionOptions
+
     \a tracks should contain markers used in the reconstruction.
     \a reconstruction should contain at least some 3D points or some estimated
     cameras. The minimum number of cameras is two (with no 3D points) and the
@@ -70,7 +77,8 @@ void EuclideanCompleteReconstruction(const Tracks &tracks,
 
     \sa ProjectiveResect, ProjectiveIntersect, ProjectiveBundle
 */
-void ProjectiveCompleteReconstruction(const Tracks &tracks,
+void ProjectiveCompleteReconstruction(const ReconstructionOptions &options,
+                                      const Tracks &tracks,
                                       ProjectiveReconstruction *reconstruction);
 
 
index 947a063..71789e3 100644 (file)
 
 namespace libmv {
 
+struct ReconstructionOptions {
+       // threshold value of reconstruction error which is still considered successful
+       // if reconstruction error bigger than this value, fallback reconstruction
+       // algorithm would be used (if enabled)
+       double success_threshold;
+
+       // use fallback reconstruction algorithm in cases main reconstruction algorithm
+       // failed to reconstruct
+       bool use_fallback_reconstruction;
+};
+
 /*!
     A EuclideanCamera is the location and rotation of the camera viewing \a image.
 
index 8737182..4c9ca6d 100644 (file)
@@ -90,7 +90,8 @@ struct EuclideanResectCostFunction {
 
 }  // namespace
 
-bool EuclideanResect(const vector<Marker> &markers,
+bool EuclideanResect(const ReconstructionOptions &options,
+                     const vector<Marker> &markers,
                      EuclideanReconstruction *reconstruction, bool final_pass) {
   if (markers.size() < 5) {
     return false;
@@ -104,13 +105,24 @@ bool EuclideanResect(const vector<Marker> &markers,
 
   Mat3 R;
   Vec3 t;
-  if (0 || !euclidean_resection::EuclideanResection(points_2d, points_3d, &R, &t)) {
+
+  double success_threshold = std::numeric_limits<double>::max();
+
+  if(options.use_fallback_reconstruction)
+    success_threshold = options.success_threshold;
+
+  if (0 || !euclidean_resection::EuclideanResection(points_2d, points_3d, &R, &t,
+                                                    euclidean_resection::RESECTION_EPNP,
+                                                    success_threshold))
+  {
     // printf("Resection for image %d failed\n", markers[0].image);
     LG << "Resection for image " << markers[0].image << " failed;"
        << " trying fallback projective resection.";
 
-    LG << "No fallback; failing resection for " << markers[0].image;
-    return false;
+    if (!options.use_fallback_reconstruction) {
+        LG << "No fallback; failing resection for " << markers[0].image;
+        return false;
+    }
 
     if (!final_pass) return false;
     // Euclidean resection failed. Fall back to projective resection, which is
index f8b5b9f..1691e7e 100644 (file)
@@ -35,6 +35,9 @@ namespace libmv {
     reconstruction object, and solves for the pose and orientation of the
     camera for that frame.
 
+    \a options are used to define some specific befaviours based on settings
+    see documentation for ReconstructionOptions
+
     \a markers should contain \l Marker markers \endlink belonging to tracks
     visible in the one frame to be resectioned. Each of the tracks associated
     with the markers must have a corresponding reconstructed 3D position in the
@@ -51,7 +54,8 @@ namespace libmv {
 
     \sa EuclideanIntersect, EuclideanReconstructTwoFrames
 */
-bool EuclideanResect(const vector<Marker> &markers,
+bool EuclideanResect(const ReconstructionOptions &options,
+                     const vector<Marker> &markers,
                      EuclideanReconstruction *reconstruction, bool final_pass);
 
 /*!
index f7b9f59..cb88226 100644 (file)
@@ -320,6 +320,13 @@ class CLIP_PT_tools_solve(CLIP_PT_tracking_panel, Panel):
         col.label(text="Refine:")
         col.prop(settings, "refine_intrinsics", text="")
 
+        col = layout.column(align=True)
+        col.active = not settings.use_tripod_solver
+        col.prop(settings, "use_fallback_reconstruction", text="Allow Fallback")
+        sub = col.column()
+        sub.active = settings.use_fallback_reconstruction
+        sub.prop(settings, "reconstruction_success_threshold")
+
 
 class CLIP_PT_tools_cleanup(CLIP_PT_tracking_panel, Panel):
     bl_space_type = 'CLIP_EDITOR'
index 26775ea..89446a1 100644 (file)
@@ -173,6 +173,7 @@ void BKE_tracking_settings_init(MovieTracking *tracking)
        tracking->settings.default_search_size = 61;
        tracking->settings.dist = 1;
        tracking->settings.object_distance = 1;
+       tracking->settings.reconstruction_success_threshold = 1e-3;
 
        tracking->stabilization.scaleinf = 1.0f;
        tracking->stabilization.locinf = 1.0f;
@@ -2561,6 +2562,9 @@ typedef struct MovieReconstructContext {
 
        TracksMap *tracks_map;
 
+       float success_threshold;
+       int use_fallback_reconstruction;
+
        int sfra, efra;
 } MovieReconstructContext;
 
@@ -2830,6 +2834,9 @@ MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieTracking *
        context->k2 = camera->k2;
        context->k3 = camera->k3;
 
+       context->success_threshold = tracking->settings.reconstruction_success_threshold;
+       context->use_fallback_reconstruction = tracking->settings.reconstruction_flag & TRACKING_USE_FALLBACK_RECONSTRUCTION;
+
        context->tracks_map = tracks_map_new(context->object_name, context->is_camera, num_tracks, 0);
 
        track = tracksbase->first;
@@ -2929,12 +2936,18 @@ void BKE_tracking_reconstruction_solve(MovieReconstructContext *context, short *
                                                           reconstruct_update_solve_cb, &progressdata);
        }
        else {
+               struct libmv_reconstructionOptions options;
+
+               options.success_threshold = context->success_threshold;
+               options.use_fallback_reconstruction = context->use_fallback_reconstruction;
+
                context->reconstruction = libmv_solveReconstruction(context->tracks,
                                                                    context->keyframe1, context->keyframe2,
                                                                    context->refine_flags,
                                                                    context->focal_length,
                                                                    context->principal_point[0], context->principal_point[1],
                                                                    context->k1, context->k2, context->k3,
+                                                                   &options,
                                                                    reconstruct_update_solve_cb, &progressdata);
        }
 
index 44561e7..3821387 100644 (file)
@@ -8310,6 +8310,19 @@ static void do_versions(FileData *fd, Library *lib, Main *main)
                }
        }
 
+       {
+               /* fallbck resection method settings */
+               {
+                       MovieClip *clip;
+
+                       for (clip = main->movieclip.first; clip; clip = clip->id.next) {
+                               if (clip->tracking.settings.reconstruction_success_threshold == 0.0f) {
+                                       clip->tracking.settings.reconstruction_success_threshold = 1e-3;
+                               }
+                       }
+               }
+       }
+
        /* WATCH IT!!!: pointers from libdata have not been converted yet here! */
        /* WATCH IT 2!: Userdef struct init has to be in editors/interface/resources.c! */
 
index c6cefce..6b918ea 100644 (file)
@@ -167,6 +167,9 @@ typedef struct MovieTrackingSettings {
                                             * were moved to per-tracking object settings
                                             */
 
+       float reconstruction_success_threshold;
+       int reconstruction_flag;
+
        /* which camera intrinsics to refine. uses on the REFINE_* flags */
        short refine_camera_intrinsics, pad2;
 
@@ -224,6 +227,7 @@ typedef struct MovieTrackingObject {
        ListBase tracks;        /* list of tracks use to tracking this object */
        MovieTrackingReconstruction reconstruction; /* reconstruction data for this object */
 
+       /* reconstruction options */
        int keyframe1, keyframe2;   /* two keyframes for reconstrution initialization */
 } MovieTrackingObject;
 
@@ -331,6 +335,9 @@ enum {
 #define TRACKING_SPEED_QUARTER      4
 #define TRACKING_SPEED_DOUBLE       5
 
+/* MovieTrackingObject->reconstruction_flag */
+#define TRACKING_USE_FALLBACK_RECONSTRUCTION  (1 << 0)
+
 /* MovieTrackingSettings->refine_camera_intrinsics */
 #define REFINE_FOCAL_LENGTH         (1 << 0)
 #define REFINE_PRINCIPAL_POINT      (1 << 1)
index 365b80b..51b50f7 100644 (file)
@@ -574,6 +574,19 @@ static void rna_def_trackingSettings(BlenderRNA *brna)
                                 "Limit speed of tracking to make visual feedback easier "
                                 "(this does not affect the tracking quality)");
 
+       /* reconstruction success_threshold */
+       prop = RNA_def_property(srna, "reconstruction_success_threshold", PROP_FLOAT, PROP_NONE);
+       RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
+       RNA_def_property_float_default(prop, 0.001f);
+       RNA_def_property_range(prop, 0, FLT_MAX);
+       RNA_def_property_ui_text(prop, "Success Threshold", "Threshold value of reconstruction error which is still considered successful");
+
+       /* use fallback reconstruction */
+       prop = RNA_def_property(srna, "use_fallback_reconstruction", PROP_BOOLEAN, PROP_NONE);
+       RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
+       RNA_def_property_boolean_sdna(prop, NULL, "reconstruction_flag", TRACKING_USE_FALLBACK_RECONSTRUCTION);
+       RNA_def_property_ui_text(prop, "Use Fallback", "Use fallback reconstruction algorithm in cases main reconstruction algorithm failed. Could give better solution with bad tracks");
+
        /* intrinsics refinement during bundle adjustment */
        prop = RNA_def_property(srna, "refine_intrinsics", PROP_ENUM, PROP_NONE);
        RNA_def_property_enum_sdna(prop, NULL, "refine_camera_intrinsics");