Multi-threaded frame calculation for movie clip proxies
authorSergey Sharybin <sergey.vfx@gmail.com>
Fri, 15 Mar 2013 11:59:46 +0000 (11:59 +0000)
committerSergey Sharybin <sergey.vfx@gmail.com>
Fri, 15 Mar 2013 11:59:46 +0000 (11:59 +0000)
This commit implements multi-threaded calculation of frames
when building proxies. Both scaling and undistortion steps
are now threaded.

Frames and proxy resolution are still handled one-by-one,
saving files after every single step. So if HDD is not so
fast, this commit could have not so much benefit.

Internal changes:

- Added IMB_scaleImBuf_threaded which scales given image
  buffer in multiple threads and uses bilinear filtering.

- libmv's camera intrinsics now have SetThreads() method
  which is used to specify how many OpenMP threads to use
  for buffer distortion/undistortion.

  And yeah, this code is using OpenMP for threading.

- Reshuffled a bit libmv-capi calls and added function
  BKE_tracking_distortion_set_threads to specify number
  of threads used by intrinscis.

extern/libmv/libmv-capi.cpp
extern/libmv/libmv-capi.h
extern/libmv/libmv/simple_pipeline/camera_intrinsics.cc
extern/libmv/libmv/simple_pipeline/camera_intrinsics.h
source/blender/blenkernel/BKE_tracking.h
source/blender/blenkernel/intern/movieclip.c
source/blender/blenkernel/intern/tracking.c
source/blender/editors/space_clip/clip_ops.c
source/blender/imbuf/IMB_imbuf.h
source/blender/imbuf/intern/scaling.c

index a85c3268b16e1b63951733508913ea55642e9c57..eccf6fb7cb6a8840cff5596e650d8c1e370b3127 100644 (file)
@@ -821,6 +821,13 @@ struct libmv_CameraIntrinsics *libmv_ReconstructionExtractIntrinsics(struct libm
        return (struct libmv_CameraIntrinsics *)&libmv_Reconstruction->intrinsics;
 }
 
+struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsNewEmpty(void)
+{
+       libmv::CameraIntrinsics *camera_intrinsics = new libmv::CameraIntrinsics();
+
+       return (struct libmv_CameraIntrinsics *) camera_intrinsics;
+}
+
 struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsNew(libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options)
 {
        libmv::CameraIntrinsics *camera_intrinsics = new libmv::CameraIntrinsics();
@@ -884,6 +891,13 @@ void libmv_CameraIntrinsicsUpdate(struct libmv_CameraIntrinsics *libmv_intrinsic
        }
 }
 
+void libmv_CameraIntrinsicsSetThreads(struct libmv_CameraIntrinsics *libmv_intrinsics, int threads)
+{
+       libmv::CameraIntrinsics *camera_intrinsics = (libmv::CameraIntrinsics *) libmv_intrinsics;
+
+       camera_intrinsics->SetThreads(threads);
+}
+
 void libmv_CameraIntrinsicsExtract(struct libmv_CameraIntrinsics *libmv_intrinsics, double *focal_length,
                        double *principal_x, double *principal_y, double *k1, double *k2, double *k3, int *width, int *height)
 {
index c90322c56c5ec21f276b15293175090c43afc842..c3debe45fd104413935c6d6d3bef45d83e998d7e 100644 (file)
@@ -132,6 +132,8 @@ void libmv_destroyFeatures(struct libmv_Features *libmv_features);
 /* camera intrinsics */
 struct libmv_CameraIntrinsics *libmv_ReconstructionExtractIntrinsics(struct libmv_Reconstruction *libmv_Reconstruction);
 
+struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsNewEmpty(void);
+
 struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsNew(libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options);
 
 struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsCopy(struct libmv_CameraIntrinsics *libmv_intrinsics);
@@ -141,6 +143,8 @@ void libmv_CameraIntrinsicsDestroy(struct libmv_CameraIntrinsics *libmv_intrinsi
 void libmv_CameraIntrinsicsUpdate(struct libmv_CameraIntrinsics *libmv_intrinsics,
                                   libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options);
 
+void libmv_CameraIntrinsicsSetThreads(struct libmv_CameraIntrinsics *libmv_intrinsics, int threads);
+
 void libmv_CameraIntrinsicsExtract(struct libmv_CameraIntrinsics *libmv_intrinsics, double *focal_length,
                        double *principal_x, double *principal_y, double *k1, double *k2, double *k3, int *width, int *height);
 
index 658f65c1367f8570d0bc1bb00a3c32f8f3800932..543fddedd0c1461d075b653c05eb9022835b48dd 100644 (file)
@@ -62,7 +62,8 @@ CameraIntrinsics::CameraIntrinsics()
       p1_(0),
       p2_(0),
       distort_(0),
-      undistort_(0) {}
+      undistort_(0),
+      threads_(1) {}
 
 CameraIntrinsics::CameraIntrinsics(const CameraIntrinsics &from)
     : K_(from.K_),
@@ -72,7 +73,8 @@ CameraIntrinsics::CameraIntrinsics(const CameraIntrinsics &from)
       k2_(from.k2_),
       k3_(from.k3_),
       p1_(from.p1_),
-      p2_(from.p2_)
+      p2_(from.p2_),
+      threads_(from.threads_)
 {
   distort_ = copyGrid(from.distort_);
   undistort_ = copyGrid(from.undistort_);
@@ -120,6 +122,11 @@ void CameraIntrinsics::SetTangentialDistortion(double p1, double p2) {
   FreeLookupGrid();
 }
 
+void CameraIntrinsics::SetThreads(int threads)
+{
+       threads_ = threads;
+}
+
 void CameraIntrinsics::ApplyIntrinsics(double normalized_x,
                                        double normalized_y,
                                        double *image_x,
@@ -188,6 +195,7 @@ void CameraIntrinsics::ComputeLookupGrid(Grid* grid, int width, int height, doub
   double aspx = (double)w / image_width_;
   double aspy = (double)h / image_height_;
 
+  #pragma omp parallel for schedule(dynamic) num_threads(threads_) if (threads_ > 1 && height > 100)
   for (int y = 0; y < height; y++) {
     for (int x = 0; x < width; x++) {
       double src_x = (x - 0.5 * overscan * w) / aspx, src_y = (y - 0.5 * overscan * h) / aspy;
@@ -214,7 +222,8 @@ void CameraIntrinsics::ComputeLookupGrid(Grid* grid, int width, int height, doub
 // TODO(MatthiasF): cubic B-Spline image sampling, bilinear lookup
 template<typename T,int N>
 static void Warp(const Grid* grid, const T* src, T* dst,
-                 int width, int height) {
+                 int width, int height, int threads) {
+  #pragma omp parallel for schedule(dynamic) num_threads(threads) if (threads > 1 && height > 100)
   for (int y = 0; y < height; y++) {
     for (int x = 0; x < width; x++) {
       Offset offset = grid->offset[y*width+x];
@@ -306,37 +315,37 @@ void CameraIntrinsics::CheckUndistortLookupGrid(int width, int height, double ov
 
 void CameraIntrinsics::Distort(const float* src, float* dst, int width, int height, double overscan, int channels) {
   CheckDistortLookupGrid(width, height, overscan);
-       if(channels==1) Warp<float,1>(distort_,src,dst,width,height);
-  else if(channels==2) Warp<float,2>(distort_,src,dst,width,height);
-  else if(channels==3) Warp<float,3>(distort_,src,dst,width,height);
-  else if(channels==4) Warp<float,4>(distort_,src,dst,width,height);
+       if(channels==1) Warp<float,1>(distort_,src,dst,width,height,threads_);
+  else if(channels==2) Warp<float,2>(distort_,src,dst,width,height,threads_);
+  else if(channels==3) Warp<float,3>(distort_,src,dst,width,height,threads_);
+  else if(channels==4) Warp<float,4>(distort_,src,dst,width,height,threads_);
   //else assert("channels must be between 1 and 4");
 }
 
 void CameraIntrinsics::Distort(const unsigned char* src, unsigned char* dst, int width, int height, double overscan, int channels) {
   CheckDistortLookupGrid(width, height, overscan);
-       if(channels==1) Warp<unsigned char,1>(distort_,src,dst,width,height);
-  else if(channels==2) Warp<unsigned char,2>(distort_,src,dst,width,height);
-  else if(channels==3) Warp<unsigned char,3>(distort_,src,dst,width,height);
-  else if(channels==4) Warp<unsigned char,4>(distort_,src,dst,width,height);
+       if(channels==1) Warp<unsigned char,1>(distort_,src,dst,width,height,threads_);
+  else if(channels==2) Warp<unsigned char,2>(distort_,src,dst,width,height,threads_);
+  else if(channels==3) Warp<unsigned char,3>(distort_,src,dst,width,height,threads_);
+  else if(channels==4) Warp<unsigned char,4>(distort_,src,dst,width,height,threads_);
   //else assert("channels must be between 1 and 4");
 }
 
 void CameraIntrinsics::Undistort(const float* src, float* dst, int width, int height, double overscan, int channels) {
   CheckUndistortLookupGrid(width, height, overscan);
-       if(channels==1) Warp<float,1>(undistort_,src,dst,width,height);
-  else if(channels==2) Warp<float,2>(undistort_,src,dst,width,height);
-  else if(channels==3) Warp<float,3>(undistort_,src,dst,width,height);
-  else if(channels==4) Warp<float,4>(undistort_,src,dst,width,height);
+       if(channels==1) Warp<float,1>(undistort_,src,dst,width,height,threads_);
+  else if(channels==2) Warp<float,2>(undistort_,src,dst,width,height,threads_);
+  else if(channels==3) Warp<float,3>(undistort_,src,dst,width,height,threads_);
+  else if(channels==4) Warp<float,4>(undistort_,src,dst,width,height,threads_);
   //else assert("channels must be between 1 and 4");
 }
 
 void CameraIntrinsics::Undistort(const unsigned char* src, unsigned char* dst, int width, int height, double overscan, int channels) {
   CheckUndistortLookupGrid(width, height, overscan);
-       if(channels==1) Warp<unsigned char,1>(undistort_,src,dst,width,height);
-  else if(channels==2) Warp<unsigned char,2>(undistort_,src,dst,width,height);
-  else if(channels==3) Warp<unsigned char,3>(undistort_,src,dst,width,height);
-  else if(channels==4) Warp<unsigned char,4>(undistort_,src,dst,width,height);
+       if(channels==1) Warp<unsigned char,1>(undistort_,src,dst,width,height,threads_);
+  else if(channels==2) Warp<unsigned char,2>(undistort_,src,dst,width,height,threads_);
+  else if(channels==3) Warp<unsigned char,3>(undistort_,src,dst,width,height,threads_);
+  else if(channels==4) Warp<unsigned char,4>(undistort_,src,dst,width,height,threads_);
   //else assert("channels must be between 1 and 4");
 }
 
index b51b28a4bfb96596be570d788db610506b84a3e6..632922c38ff32d34b8915d791395558b708f11bd 100644 (file)
@@ -68,6 +68,9 @@ class CameraIntrinsics {
 
   void SetTangentialDistortion(double p1, double p2);
 
+  /// Set number of threads using for buffer distortion/undistortion
+  void SetThreads(int threads);
+
   /*!
       Apply camera intrinsics to the normalized point to get image coordinates.
 
@@ -153,6 +156,8 @@ class CameraIntrinsics {
 
   struct Grid *distort_;
   struct Grid *undistort_;
+
+  int threads_;
 };
 
 /// A human-readable representation of the camera intrinsic parameters.
index ebf29fde8ee7ad2c6f78611a341aae6b37483894..c8ad708f832c803c7cd8bf34e87f70ac4a1e0718 100644 (file)
@@ -144,6 +144,7 @@ void BKE_tracking_camera_get_reconstructed_interpolate(struct MovieTracking *tra
 struct MovieDistortion *BKE_tracking_distortion_new(void);
 void BKE_tracking_distortion_update(struct MovieDistortion *distortion, struct MovieTracking *tracking,
                                     int calibration_width, int calibration_height);
+void BKE_tracking_distortion_set_threads(struct MovieDistortion *distortion, int threads);
 struct MovieDistortion *BKE_tracking_distortion_copy(struct MovieDistortion *distortion);
 struct ImBuf *BKE_tracking_distortion_exec(struct MovieDistortion *distortion, struct MovieTracking *tracking,
                                            struct ImBuf *ibuf, int width, int height, float overscan, int undistort);
index ab42d86152d447ea13e3c1ea7e5373f940474445..1ad0bf1988c5bd723b136d557c190b3b5649dfe6 100644 (file)
@@ -1244,7 +1244,7 @@ static void movieclip_build_proxy_ibuf(MovieClip *clip, ImBuf *ibuf, int cfra, i
 
        scaleibuf = IMB_dupImBuf(ibuf);
 
-       IMB_scaleImBuf(scaleibuf, (short)rectx, (short)recty);
+       IMB_scaleImBuf_threaded(scaleibuf, (short)rectx, (short)recty);
 
        quality = clip->proxy.quality;
        scaleibuf->ftype = JPG | quality;
index 5f8e25af2eec0a893768d52fb21b63d58efb6b28..3100917cd0cc73dec354264188febdf4f8d70ad0 100644 (file)
@@ -1462,6 +1462,8 @@ MovieDistortion *BKE_tracking_distortion_new(void)
 
        distortion = MEM_callocN(sizeof(MovieDistortion), "BKE_tracking_distortion_create");
 
+       distortion->intrinsics = libmv_CameraIntrinsicsNewEmpty();
+
        return distortion;
 }
 
@@ -1474,12 +1476,7 @@ void BKE_tracking_distortion_update(MovieDistortion *distortion, MovieTracking *
        cameraIntrinscisOptionsFromTracking(&camera_intrinsics_options, tracking,
                                            calibration_width, calibration_height);
 
-       if (!distortion->intrinsics) {
-               distortion->intrinsics = libmv_CameraIntrinsicsNew(&camera_intrinsics_options);
-       }
-       else {
-               libmv_CameraIntrinsicsUpdate(distortion->intrinsics, &camera_intrinsics_options);
-       }
+       libmv_CameraIntrinsicsUpdate(distortion->intrinsics, &camera_intrinsics_options);
 #else
        (void) distortion;
        (void) tracking;
@@ -1488,6 +1485,16 @@ void BKE_tracking_distortion_update(MovieDistortion *distortion, MovieTracking *
 #endif
 }
 
+void BKE_tracking_distortion_set_threads(MovieDistortion *distortion, int threads)
+{
+#ifdef WITH_LIBMV
+       libmv_CameraIntrinsicsSetThreads(distortion->intrinsics, threads);
+#else
+       (void) distortion;
+       (void) threads;
+#endif
+}
+
 MovieDistortion *BKE_tracking_distortion_copy(MovieDistortion *distortion)
 {
        MovieDistortion *new_distortion;
index b53cf027c7831543d7ed0708c02f15641975d8e6..1938cd7b53dcfe68d78fc6bbbd73b6635f360545 100644 (file)
@@ -40,6 +40,7 @@
 #include "BLI_path_util.h"
 #include "BLI_math.h"
 #include "BLI_rect.h"
+#include "BLI_threads.h"
 
 #include "BLF_translation.h"
 
@@ -1004,8 +1005,11 @@ static void proxy_startjob(void *pjv, short *stop, short *do_update, float *prog
                }
        }
 
-       if (build_undistort_count)
+       if (build_undistort_count) {
+               int threads = BLI_system_thread_count();
                distortion = BKE_tracking_distortion_new();
+               BKE_tracking_distortion_set_threads(distortion, threads);
+       }
 
        for (cfra = sfra; cfra <= efra; cfra++) {
                if (clip->source != MCLIP_SRC_MOVIE)
index a19433dbd2ff42e97dc1c3be31f48223a7529b1f..48c14fc07f21a7af3d8c02a5671b96ecf9d072aa 100644 (file)
@@ -336,6 +336,12 @@ struct ImBuf *IMB_scaleImBuf(struct ImBuf *ibuf, unsigned int newx, unsigned int
  */
 struct ImBuf *IMB_scalefastImBuf(struct ImBuf *ibuf, unsigned int newx, unsigned int newy);
 
+/**
+ *
+ * \attention Defined in scaling.c
+ */
+void IMB_scaleImBuf_threaded(struct ImBuf *ibuf, unsigned int newx, unsigned int newy);
+
 /**
  *
  * \attention Defined in writeimage.c
index 1050d3f8715fd7a675fac027d3d91d907dcb28cc..51619e18980be706b8e4491b2d4e3136a1494aa5 100644 (file)
@@ -34,6 +34,7 @@
 
 #include "BLI_utildefines.h"
 #include "BLI_math_color.h"
+#include "BLI_math_interp.h"
 #include "MEM_guardedalloc.h"
 
 #include "imbuf.h"
@@ -1604,3 +1605,113 @@ struct ImBuf *IMB_scalefastImBuf(struct ImBuf *ibuf, unsigned int newx, unsigned
        return(ibuf);
 }
 
+/* ******** threaded scaling ******** */
+
+typedef struct ScaleTreadInitData {
+       ImBuf *ibuf;
+
+       unsigned int newx;
+       unsigned int newy;
+
+       unsigned char *byte_buffer;
+       float *float_buffer;
+} ScaleTreadInitData;
+
+typedef struct ScaleThreadData {
+       ImBuf *ibuf;
+
+       unsigned int newx;
+       unsigned int newy;
+
+       int start_line;
+       int tot_line;
+
+       unsigned char *byte_buffer;
+       float *float_buffer;
+} ScaleThreadData;
+
+static void scale_thread_init(void *data_v, int start_line, int tot_line, void *init_data_v)
+{
+       ScaleThreadData *data = (ScaleThreadData *) data_v;
+       ScaleTreadInitData *init_data = (ScaleTreadInitData *) init_data_v;
+
+       data->ibuf = init_data->ibuf;
+
+       data->newx = init_data->newx;
+       data->newy = init_data->newy;
+
+       data->start_line = start_line;
+       data->tot_line = tot_line;
+
+       data->byte_buffer = init_data->byte_buffer;
+       data->float_buffer = init_data->float_buffer;
+}
+
+static void *do_scale_thread(void *data_v)
+{
+       ScaleThreadData *data = (ScaleThreadData *) data_v;
+       ImBuf *ibuf = data->ibuf;
+       int i;
+       float factor_x = (float) ibuf->x / data->newx;
+       float factor_y = (float) ibuf->y / data->newy;
+
+       for (i = 0; i < data->tot_line; i++) {
+               int y = data->start_line + i;
+               int x;
+
+               for (x = 0; x < data->newx; x++) {
+                       float u = (float) x * factor_x;
+                       float v = (float) y * factor_y;
+                       int offset = y * data->newx + x;
+
+                       if (data->byte_buffer) {
+                               unsigned char *pixel = data->byte_buffer + 4 * offset;
+                               BLI_bilinear_interpolation_char((unsigned char *) ibuf->rect, pixel, ibuf->x, ibuf->y, 4, u, v);
+                       }
+
+                       if (data->float_buffer) {
+                               float *pixel = data->float_buffer + ibuf->channels * offset;
+                               BLI_bilinear_interpolation_fl(ibuf->rect_float, pixel, ibuf->x, ibuf->y, ibuf->channels, u, v);
+                       }
+               }
+       }
+
+       return NULL;
+}
+
+void IMB_scaleImBuf_threaded(ImBuf *ibuf, unsigned int newx, unsigned int newy)
+{
+       ScaleTreadInitData init_data = {0};
+
+       /* prepare initialization data */
+       init_data.ibuf = ibuf;
+
+       init_data.newx = newx;
+       init_data.newy = newy;
+
+       if (ibuf->rect)
+               init_data.byte_buffer = MEM_mallocN(4 * newx * newy * sizeof(char), "threaded scale byte buffer");
+
+       if (ibuf->rect_float)
+               init_data.float_buffer = MEM_mallocN(ibuf->channels * newx * newy * sizeof(float), "threaded scale float buffer");
+
+       /* actual scaling threads */
+       IMB_processor_apply_threaded(newy, sizeof(ScaleThreadData), &init_data,
+                                    scale_thread_init, do_scale_thread);
+
+       /* alter image buffer */
+       ibuf->x = newx;
+       ibuf->y = newy;
+
+       if (ibuf->rect) {
+               imb_freerectImBuf(ibuf);
+               ibuf->mall |= IB_rect;
+               ibuf->rect = (unsigned int *) init_data.byte_buffer;
+       }
+
+       if (ibuf->rect_float) {
+               imb_freerectfloatImBuf(ibuf);
+               ibuf->mall |= IB_rectfloat;
+               ibuf->rect_float = init_data.float_buffer;
+       }
+}