Weighted tracks
authorSergey Sharybin <sergey.vfx@gmail.com>
Sat, 26 Oct 2013 13:22:38 +0000 (13:22 +0000)
committerSergey Sharybin <sergey.vfx@gmail.com>
Sat, 26 Oct 2013 13:22:38 +0000 (13:22 +0000)
Added a weight slider to track which defines
how much particular track affects in a final
reconstruction. This weight is for sure
animateable.

Currently it affects on BA step only which in
most cases will work just fine.

The usecase of this slider is to have it set
to 1.0 most of the time where the track is
good, but blend it's weight down to 0 when
tracker looses the track. This will prevent
camera from jump.

Tutorial is to be done by Sebastian.

15 files changed:
extern/libmv/libmv-capi.cc
extern/libmv/libmv-capi.h
extern/libmv/libmv-capi_stub.cc
extern/libmv/libmv/simple_pipeline/bundle.cc
extern/libmv/libmv/simple_pipeline/modal_solver.cc
extern/libmv/libmv/simple_pipeline/tracks.cc
extern/libmv/libmv/simple_pipeline/tracks.h
release/scripts/startup/bl_ui/space_clip.py
source/blender/blenkernel/BKE_tracking.h
source/blender/blenkernel/intern/tracking.c
source/blender/blenloader/intern/readfile.c
source/blender/editors/space_clip/clip_utils.c
source/blender/editors/space_clip/tracking_ops.c
source/blender/makesdna/DNA_tracking_types.h
source/blender/makesrna/intern/rna_tracking.c

index 91a3b84581545c57e183e7d41b3de5bd8aaf9d3f..0585bd3e8acaf5266961952a87c7d4816fe8b060 100644 (file)
@@ -393,9 +393,9 @@ void libmv_tracksDestroy(struct libmv_Tracks *libmv_tracks)
        LIBMV_OBJECT_DELETE(libmv_tracks, Tracks);
 }
 
-void libmv_tracksInsert(struct libmv_Tracks *libmv_tracks, int image, int track, double x, double y)
+void libmv_tracksInsert(struct libmv_Tracks *libmv_tracks, int image, int track, double x, double y, double weight)
 {
-       ((libmv::Tracks*) libmv_tracks)->Insert(image, track, x, y);
+       ((libmv::Tracks*) libmv_tracks)->Insert(image, track, x, y, weight);
 }
 
 /* ************ Reconstruction ************ */
index 9541f411ba02bae22a135a97e6acc9bd5a66b7b0..13cc3ae8499965cd6b313e8eb96026d83118c951 100644 (file)
@@ -73,7 +73,7 @@ void libmv_samplePlanarPatch(const float *image, int width, int height,
 /* Tracks */
 struct libmv_Tracks *libmv_tracksNew(void);
 void libmv_tracksDestroy(struct libmv_Tracks *libmv_tracks);
-void libmv_tracksInsert(struct libmv_Tracks *libmv_tracks, int image, int track, double x, double y);
+void libmv_tracksInsert(struct libmv_Tracks *libmv_tracks, int image, int track, double x, double y, double weight);
 
 /* Reconstruction */
 #define LIBMV_REFINE_FOCAL_LENGTH          (1 << 0)
index e6d3753961b466658c9f7f280d60cbf22a0dc92f..bda9605b422d5fdaabceb1874efdd846a47de078 100644 (file)
@@ -85,7 +85,7 @@ struct libmv_Tracks *libmv_tracksNew(void)
 }
 
 void libmv_tracksInsert(struct libmv_Tracks * /*libmv_tracks*/, int /*image*/,
-                        int /*track*/, double /*x*/, double /*y*/)
+                        int /*track*/, double /*x*/, double /*y*/, double /*weight*/)
 {
 }
 
index c778c11b3f60fa6b3a6437785215bd7a30da2341..e7887256892866de18afbaf53d804040f05de37d 100644 (file)
@@ -60,8 +60,11 @@ namespace {
 //
 // This functor uses a radial distortion model.
 struct OpenCVReprojectionError {
-  OpenCVReprojectionError(const double observed_x, const double observed_y)
-      : observed_x(observed_x), observed_y(observed_y) {}
+  OpenCVReprojectionError(const double observed_x,
+                          const double observed_y,
+                          const double weight)
+      : observed_x_(observed_x), observed_y_(observed_y),
+        weight_(weight) {}
 
   template <typename T>
   bool operator()(const T* const intrinsics,
@@ -112,13 +115,14 @@ struct OpenCVReprojectionError {
                                           &predicted_y);
 
     // The error is the difference between the predicted and observed position.
-    residuals[0] = predicted_x - T(observed_x);
-    residuals[1] = predicted_y - T(observed_y);
+    residuals[0] = (predicted_x - T(observed_x_)) * weight_;
+    residuals[1] = (predicted_y - T(observed_y_)) * weight_;
     return true;
   }
 
-  const double observed_x;
-  const double observed_y;
+  const double observed_x_;
+  const double observed_y_;
+  const double weight_;
 };
 
 // Print a message to the log which camera intrinsics are gonna to be optimixed.
@@ -378,25 +382,31 @@ void EuclideanBundleCommonIntrinsics(const Tracks &tracks,
     // camera translaiton.
     double *current_camera_R_t = &all_cameras_R_t[camera->image](0);
 
-    problem.AddResidualBlock(new ceres::AutoDiffCostFunction<
-        OpenCVReprojectionError, 2, 8, 6, 3>(
-            new OpenCVReprojectionError(
-                marker.x,
-                marker.y)),
-        NULL,
-        ceres_intrinsics,
-        current_camera_R_t,
-        &point->X(0));
-
-    // We lock the first camera to better deal with scene orientation ambiguity.
-    if (!have_locked_camera) {
-      problem.SetParameterBlockConstant(current_camera_R_t);
-      have_locked_camera = true;
-    }
+    // Skip residual block for markers which does have absolutely
+    // no affect on the final solution.
+    // This way ceres is not gonna to go crazy.
+    if (marker.weight != 0.0) {
+      problem.AddResidualBlock(new ceres::AutoDiffCostFunction<
+          OpenCVReprojectionError, 2, 8, 6, 3>(
+              new OpenCVReprojectionError(
+                  marker.x,
+                  marker.y,
+                  marker.weight)),
+          NULL,
+          ceres_intrinsics,
+          current_camera_R_t,
+          &point->X(0));
+
+      // We lock the first camera to better deal with scene orientation ambiguity.
+      if (!have_locked_camera) {
+        problem.SetParameterBlockConstant(current_camera_R_t);
+        have_locked_camera = true;
+      }
 
-    if (bundle_constraints & BUNDLE_NO_TRANSLATION) {
-      problem.SetParameterization(current_camera_R_t,
+      if (bundle_constraints & BUNDLE_NO_TRANSLATION) {
+        problem.SetParameterization(current_camera_R_t,
                                   constant_translation_parameterization);
+      }
     }
 
     num_residuals++;
index 90dfde156604f0f115e9f78d0690f0ca5d7484a7..caccce68cbefd04b51842e28577334f5b46a6e56 100644 (file)
@@ -57,8 +57,10 @@ void ModalSolverLogProress(ProgressUpdateCallback *update_callback,
 struct ModalReprojectionError {
   ModalReprojectionError(double observed_x,
                          double observed_y,
+                         const double weight,
                          const Vec3 &bundle)
-    : observed_x(observed_x), observed_y(observed_y), bundle(bundle) { }
+    : observed_x_(observed_x), observed_y_(observed_y),
+      weight_(weight), bundle_(bundle) { }
 
   template <typename T>
   bool operator()(const T* quaternion,   // Rotation quaternion
@@ -68,9 +70,9 @@ struct ModalReprojectionError {
 
     // Convert bundle position from double to T.
     T X[3];
-    X[0] = T(bundle(0));
-    X[1] = T(bundle(1));
-    X[2] = T(bundle(2));
+    X[0] = T(bundle_(0));
+    X[1] = T(bundle_(1));
+    X[2] = T(bundle_(2));
 
     // Compute projective coordinates: x = RX.
     T x[3];
@@ -84,15 +86,16 @@ struct ModalReprojectionError {
 
     // The error is the difference between reprojected
     // and observed marker position.
-    residuals[0] = xn - T(observed_x);
-    residuals[1] = yn - T(observed_y);
+    residuals[0] = xn - T(observed_x_);
+    residuals[1] = yn - T(observed_y_);
 
     return true;
   }
 
-  double observed_x;
-  double observed_y;
-  Vec3 bundle;
+  double observed_x_;
+  double observed_y_;
+  double weight_;
+  Vec3 bundle_;
 };
 }  // namespace
 
@@ -180,11 +183,13 @@ void ModalSolver(const Tracks &tracks,
       Marker &marker = all_markers[i];
       EuclideanPoint *point = reconstruction->PointForTrack(marker.track);
 
-      if (point) {
+      if (point && marker.weight != 0.0) {
         problem.AddResidualBlock(new ceres::AutoDiffCostFunction<
             ModalReprojectionError,
             2, /* num_residuals */
-            4>(new ModalReprojectionError(marker.x, marker.y,
+            4>(new ModalReprojectionError(marker.x,
+                                          marker.y,
+                                          marker.weight,
                                           point->X)),
             NULL,
             &quaternion(0));
index f9e50d20af969937fff110d8934babaeb395006d..d5d009708bad863ffdb1b5d834431184d85684f0 100644 (file)
@@ -34,7 +34,7 @@ Tracks::Tracks(const Tracks &other) {
 
 Tracks::Tracks(const vector<Marker> &markers) : markers_(markers) {}
 
-void Tracks::Insert(int image, int track, double x, double y) {
+void Tracks::Insert(int image, int track, double x, double y, double weight) {
   // TODO(keir): Wow, this is quadratic for repeated insertions. Fix this by
   // adding a smarter data structure like a set<>.
   for (int i = 0; i < markers_.size(); ++i) {
@@ -45,7 +45,7 @@ void Tracks::Insert(int image, int track, double x, double y) {
       return;
     }
   }
-  Marker marker = { image, track, x, y };
+  Marker marker = { image, track, x, y, weight };
   markers_.push_back(marker);
 }
 
@@ -122,7 +122,7 @@ Marker Tracks::MarkerInImageForTrack(int image, int track) const {
       return markers_[i];
     }
   }
-  Marker null = { -1, -1, -1, -1 };
+  Marker null = { -1, -1, -1, -1, 0.0 };
   return null;
 }
 
index f9af3ada45bc4a32ebc252205f84fa672da67800..e2f8cf6b459bd0015bbfef6451fae33f19041241 100644 (file)
@@ -33,14 +33,20 @@ namespace libmv {
     in the image identified by \a image. All markers for to the same target
     form a track identified by a common \a track number.
 
+    \a weight is used by bundle adjustment and weight means how much the
+    track affects on a final solution.
+
     \note Markers are typically aggregated with the help of the \l Tracks class.
 
     \sa Tracks
 */
+// TODO(sergey): Consider using comment for every member separately
+//               instead of having one giantic comment block.
 struct Marker {
   int image;
   int track;
   double x, y;
+  double weight;
 };
 
 /*!
@@ -72,9 +78,14 @@ class Tracks {
       \a image and \a track are the keys used to retrieve the markers with the
       other methods in this class.
 
+      \a weight is used by bundle adjustment and weight means how much the
+      track affects on a final solution.
+
       \note To get an identifier for a new track, use \l MaxTrack() + 1.
   */
-  void Insert(int image, int track, double x, double y);
+  // TODO(sergey): Consider using InsetWeightedMarker istead of using
+  //               stupid default value?
+  void Insert(int image, int track, double x, double y, double weight = 1.0);
 
   /// Returns all the markers.
   vector<Marker> AllMarkers() const;
index 3247d2f5e4c92acd1404a29fdf4d565b570c9118..3db8697a45795d1f3a5035a6f114860b89c28cd5 100644 (file)
@@ -569,6 +569,8 @@ class CLIP_PT_track(CLIP_PT_tracking_panel, Panel):
         if act_track.use_custom_color:
             row.prop(act_track, "color", text="")
 
+        layout.prop(act_track, "weight")
+
         if act_track.has_bundle:
             label_text = "Average Error: %.4f" % (act_track.average_error)
             layout.label(text=label_text)
index 51f97180ddb11bcbe69af797fad50bb3103781e8..94e530529ecc8c44339e0bcf8a2bb12f1152f61f 100644 (file)
@@ -225,7 +225,7 @@ void BKE_tracking_homography_between_two_quads(/*const*/ float reference_corners
 bool BKE_tracking_reconstruction_check(struct MovieTracking *tracking, struct MovieTrackingObject *object,
                                        char *error_msg, int error_size);
 
-struct MovieReconstructContext *BKE_tracking_reconstruction_context_new(struct MovieTracking *tracking,
+struct MovieReconstructContext *BKE_tracking_reconstruction_context_new(struct MovieClip *clip,
                                                                         struct MovieTrackingObject *object,
                                                                         int keyframe1, int keyframe2,
                                                                         int width, int height);
index 0ce4d54a74cd166de2bafed71083a14bb01ee885..d519b93f96310858a52a3ab892532f7ac1d214b8 100644 (file)
@@ -36,6 +36,7 @@
 
 #include "MEM_guardedalloc.h"
 
+#include "DNA_anim_types.h"
 #include "DNA_gpencil_types.h"
 #include "DNA_camera_types.h"
 #include "DNA_movieclip_types.h"
@@ -53,6 +54,7 @@
 
 #include "BLF_translation.h"
 
+#include "BKE_fcurve.h"
 #include "BKE_global.h"
 #include "BKE_tracking.h"
 #include "BKE_movieclip.h"
@@ -62,6 +64,8 @@
 #include "IMB_imbuf_types.h"
 #include "IMB_imbuf.h"
 
+#include "RNA_access.h"
+
 #include "raskter.h"
 
 #include "libmv-capi.h"
@@ -590,6 +594,7 @@ MovieTrackingTrack *BKE_tracking_track_add(MovieTracking *tracking, ListBase *tr
        track->frames_limit = settings->default_frames_limit;
        track->flag = settings->default_flag;
        track->algorithm_flag = settings->default_algorithm_flag;
+       track->weight = 1.0f;
 
        memset(&marker, 0, sizeof(marker));
        marker.pos[0] = x;
@@ -3467,7 +3472,7 @@ typedef struct ReconstructProgressData {
 } ReconstructProgressData;
 
 /* Create new libmv Tracks structure from blender's tracks list. */
-static struct libmv_Tracks *libmv_tracks_new(ListBase *tracksbase, int width, int height)
+static struct libmv_Tracks *libmv_tracks_new(MovieClip *clip, ListBase *tracksbase, int width, int height)
 {
        int tracknr = 0;
        MovieTrackingTrack *track;
@@ -3475,15 +3480,28 @@ static struct libmv_Tracks *libmv_tracks_new(ListBase *tracksbase, int width, in
 
        track = tracksbase->first;
        while (track) {
+               FCurve *weight_fcurve;
                int a = 0;
 
+               weight_fcurve = id_data_find_fcurve(&clip->id, track, &RNA_MovieTrackingTrack,
+                                                   "weight", 0, NULL);
+
                for (a = 0; a < track->markersnr; a++) {
                        MovieTrackingMarker *marker = &track->markers[a];
 
                        if ((marker->flag & MARKER_DISABLED) == 0) {
+                               float weight = track->weight;
+
+                               if (weight_fcurve) {
+                                       int scene_framenr =
+                                               BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
+                                       weight = evaluate_fcurve(weight_fcurve, scene_framenr);
+                               }
+
                                libmv_tracksInsert(tracks, marker->framenr, tracknr,
                                                   (marker->pos[0] + track->offset[0]) * width,
-                                                  (marker->pos[1] + track->offset[1]) * height);
+                                                  (marker->pos[1] + track->offset[1]) * height,
+                                                  weight);
                        }
                }
 
@@ -3730,9 +3748,10 @@ bool BKE_tracking_reconstruction_check(MovieTracking *tracking, MovieTrackingObj
  * clip datablock, so editing this clip is safe during
  * reconstruction job is in progress.
  */
-MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieTracking *tracking, MovieTrackingObject *object,
+MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieClip *clip, MovieTrackingObject *object,
                                                                  int keyframe1, int keyframe2, int width, int height)
 {
+       MovieTracking *tracking = &clip->tracking;
        MovieReconstructContext *context = MEM_callocN(sizeof(MovieReconstructContext), "MovieReconstructContext data");
        MovieTrackingCamera *camera = &tracking->camera;
        ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
@@ -3793,7 +3812,7 @@ MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieTracking *
        context->sfra = sfra;
        context->efra = efra;
 
-       context->tracks = libmv_tracks_new(tracksbase, width, height * aspy);
+       context->tracks = libmv_tracks_new(clip, tracksbase, width, height * aspy);
        context->keyframe1 = keyframe1;
        context->keyframe2 = keyframe2;
        context->refine_flags = reconstruct_refine_intrinsics_get_flags(tracking, object);
index e6804725587ca2740b93d33d7cefa4ac905104f4..51158fc5321e3aee166c0c11e4ad1ad1bdc2c7ac 100644 (file)
@@ -9735,6 +9735,27 @@ static void do_versions(FileData *fd, Library *lib, Main *main)
                                }
                        }
                }
+
+               if (!DNA_struct_elem_find(fd->filesdna, "MovieTrackingTrack", "float", "weight")) {
+                       MovieClip *clip;
+                       for (clip = main->movieclip.first; clip; clip = clip->id.next) {
+                               MovieTracking *tracking = &clip->tracking;
+                               MovieTrackingObject *tracking_object;
+                               for (tracking_object = tracking->objects.first;
+                                    tracking_object;
+                                    tracking_object = tracking_object->next)
+                               {
+                                       ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, tracking_object);
+                                       MovieTrackingTrack *track;
+                                       for (track = tracksbase->first;
+                                            track;
+                                            track = track->next)
+                                       {
+                                               track->weight = 1.0f;
+                                       }
+                               }
+                       }
+               }
        }
 
        /* WATCH IT!!!: pointers from libdata have not been converted yet here! */
index dfd2b6e259d2fde48bebda1f18ae72f5125e4202..060531ae82c5ef7b6e21f7534dd9019bb99df32c 100644 (file)
@@ -37,7 +37,9 @@
 #include "BLI_utildefines.h"
 #include "BLI_math.h"
 #include "BLI_listbase.h"
+#include "BLI_string.h"
 
+#include "BKE_animsys.h"
 #include "BKE_context.h"
 #include "BKE_movieclip.h"
 #include "BKE_tracking.h"
@@ -184,8 +186,8 @@ void clip_delete_track(bContext *C, MovieClip *clip, MovieTrackingTrack *track)
        MovieTrackingPlaneTrack *plane_track, *next_plane_track;
        ListBase *tracksbase = BKE_tracking_get_active_tracks(tracking);
        ListBase *plane_tracks_base = BKE_tracking_get_active_plane_tracks(tracking);
-
        bool has_bundle = false, update_stab = false;
+       char track_name_escaped[MAX_NAME], prefix[MAX_NAME * 2];
 
        if (track == act_track)
                tracking->act_track = NULL;
@@ -245,6 +247,11 @@ void clip_delete_track(bContext *C, MovieClip *clip, MovieTrackingTrack *track)
                }
        }
 
+       /* Delete f-curves associated with the track (such as weight, i.e.) */
+       BLI_strescape(track_name_escaped, track->name, sizeof(track_name_escaped));
+       BLI_snprintf(prefix, sizeof(prefix), "tracks[\"%s\"]", track_name_escaped);
+       BKE_animdata_fix_paths_remove(&clip->id, prefix);
+
        BKE_tracking_track_free(track);
        BLI_freelinkN(tracksbase, track);
 
index 246ea7fe140ed5e89fe61db10924a6497f43133c..51d7bc3139ab8de7da7ceae24bcc1d7e0b1df568 100644 (file)
@@ -1550,7 +1550,7 @@ static int solve_camera_initjob(bContext *C, SolveCameraJob *scj, wmOperator *op
        scj->reports = op->reports;
        scj->user = sc->user;
 
-       scj->context = BKE_tracking_reconstruction_context_new(tracking, object,
+       scj->context = BKE_tracking_reconstruction_context_new(clip, object,
                                                               object->keyframe1, object->keyframe2, width, height);
 
        tracking->stats = MEM_callocN(sizeof(MovieTrackingStats), "solve camera stats");
index 65d36adde180e719a0e76de18fe8d9d2c2801103..ad9a2ea169cdd1474791c40e3eb3b5a32b9f9336 100644 (file)
@@ -142,6 +142,17 @@ typedef struct MovieTrackingTrack {
        float minimum_correlation;          /* minimal correlation which is still treated as successful tracking */
 
        struct bGPdata *gpd;        /* grease-pencil data */
+
+       /* Weight of this track.
+        *
+        * Weight defines how much the track affects on the final reconstruction,
+        * usually gets animated in a way so when track has just appeared it's
+        * weight is zero and then it gets faded up.
+        *
+        * Used to prevent jumps of the camera when tracks are appearing or
+        * disappearing.
+        */
+       float weight, pad;
 } MovieTrackingTrack;
 
 typedef struct MovieTrackingPlaneMarker {
index 8e26bb96a1fc46290ef6b1b686e38c177dd446ad..f62778a5c1e64b80c3c520e3256a18ab4f8ce15b 100644 (file)
@@ -1339,6 +1339,12 @@ static void rna_def_trackingTrack(BlenderRNA *brna)
        RNA_def_property_struct_type(prop, "GreasePencil");
        RNA_def_property_ui_text(prop, "Grease Pencil", "Grease pencil data for this track");
        RNA_def_property_update(prop, NC_MOVIECLIP | ND_DISPLAY, NULL);
+
+       /* weight */
+       prop = RNA_def_property(srna, "weight", PROP_FLOAT, PROP_FACTOR);
+       RNA_def_property_float_sdna(prop, NULL, "weight");
+       RNA_def_property_range(prop, 0.0f, 1.0f);
+       RNA_def_property_ui_text(prop, "Weight", "How much this track affects on a final solution");
 }
 
 static void rna_def_trackingPlaneMarker(BlenderRNA *brna)